Programgen support for fnOutput forms in node (#7949) (#8434)

* Teach PCL about fnOutput forms

* Teach PCL about fnOutput forms

* Teach Node program gen to emit fnOutput forms

* TypeCheck fix

* AWS package bump

* Add tests

* CHANGELOG

* Temporarily skip non-Node affected tests

* Address PR feedback: restrict new form to Output args only

CHANGELOG_PENDING.md

@@ -28,6 +28,10 @@
 - [sdk/go] - Allow specifying Call failures from the provider.
   [#8424](https://github.com/pulumi/pulumi/pull/8424)
 
+- [codegen/nodejs] - Program generator now uses `fnOutput` forms where
+  appropriate, simplifying auto-generated examples.
+  [#8434](https://github.com/pulumi/pulumi/pull/8434)
+
 ### Bug Fixes
 
 - [engine] - Compute dependents correctly during targeted deletes.

pkg/codegen/hcl2/model/expression.go

@@ -1034,7 +1034,13 @@ func (x *FunctionCallExpression) Typecheck(typecheckOperands bool) hcl.Diagnosti
 	typecheckDiags := typecheckArgs(rng, x.Signature, x.Args...)
 	diagnostics = append(diagnostics, typecheckDiags...)
 
-	x.Signature.ReturnType = liftOperationType(x.Signature.ReturnType, x.Args...)
+	// Unless the function is already automatically using an
+	// Output-returning version, modify the signature to account
+	// for automatic lifting to Promise or Output.
+	_, isOutput := x.Signature.ReturnType.(*OutputType)
+	if !isOutput {
+		x.Signature.ReturnType = liftOperationType(x.Signature.ReturnType, x.Args...)
+	}
 	return diagnostics
 }
 

pkg/codegen/internal/test/helpers.go

@@ -363,7 +363,7 @@ func RunCommandWithOptions(
 type SchemaVersion = string
 
 const (
-	AwsSchema         SchemaVersion = "4.21.1"
+	AwsSchema         SchemaVersion = "4.26.0"
 	AzureNativeSchema SchemaVersion = "1.29.0"
 	AzureSchema       SchemaVersion = "4.18.0"
 	KubernetesSchema  SchemaVersion = "3.7.2"

pkg/codegen/internal/test/program_driver.go

@@ -50,6 +50,7 @@ var programTests = []programTest{
 		Name:        "aws-fargate",
 		Description: "AWS Fargate",
 		SkipCompile: codegen.NewStringSet("go"),
+		Skip:        codegen.NewStringSet("go", "python", "dotnet"),
 	},
 	{
 		Name:        "aws-s3-logging",
@@ -123,6 +124,11 @@ var programTests = []programTest{
 		//   TODO[pulumi/pulumi#8078]
 		//   TODO[pulumi/pulumi#8079]
 	},
+	{
+		Name:        "output-funcs-aws",
+		Description: "Output Versioned Functions",
+		Skip:        codegen.NewStringSet("go", "python", "dotnet"),
+	},
 }
 
 // Checks that a generated program is correct

pkg/codegen/internal/test/testdata/output-funcs-aws-pp/nodejs/output-funcs-aws.ts

@@ -0,0 +1,32 @@
+import * as pulumi from "@pulumi/pulumi";
+import * as aws from "@pulumi/aws";
+
+const aws_vpc = new aws.ec2.Vpc("aws_vpc", {
+    cidrBlock: "10.0.0.0/16",
+    instanceTenancy: "default",
+});
+const privateS3VpcEndpoint = new aws.ec2.VpcEndpoint("privateS3VpcEndpoint", {
+    vpcId: aws_vpc.id,
+    serviceName: "com.amazonaws.us-west-2.s3",
+});
+const privateS3PrefixList = aws.ec2.getPrefixListOutput({
+    prefixListId: privateS3VpcEndpoint.prefixListId,
+});
+const bar = new aws.ec2.NetworkAcl("bar", {vpcId: aws_vpc.id});
+const privateS3NetworkAclRule = new aws.ec2.NetworkAclRule("privateS3NetworkAclRule", {
+    networkAclId: bar.id,
+    ruleNumber: 200,
+    egress: false,
+    protocol: "tcp",
+    ruleAction: "allow",
+    cidrBlock: privateS3PrefixList.cidrBlocks[0],
+    fromPort: 443,
+    toPort: 443,
+});
+const amis = aws.ec2.getAmiIdsOutput({
+    owners: [bar.id],
+    filters: [{
+        name: bar.id,
+        values: ["pulumi*"],
+    }],
+});

pkg/codegen/internal/test/testdata/output-funcs-aws-pp/output-funcs-aws.pp

@@ -0,0 +1,36 @@
+resource aws_vpc "aws:ec2/vpc:Vpc" {
+  cidrBlock       = "10.0.0.0/16"
+  instanceTenancy = "default"
+}
+
+resource privateS3VpcEndpoint "aws:ec2/vpcEndpoint:VpcEndpoint" {
+  vpcId       = aws_vpc.id
+  serviceName = "com.amazonaws.us-west-2.s3"
+}
+
+privateS3PrefixList = invoke("aws:ec2:getPrefixList", {
+  prefixListId = privateS3VpcEndpoint.prefixListId
+})
+
+resource bar "aws:ec2/networkAcl:NetworkAcl" {
+  vpcId = aws_vpc.id
+}
+
+resource privateS3NetworkAclRule "aws:ec2/networkAclRule:NetworkAclRule" {
+  networkAclId = bar.id
+  ruleNumber   = 200
+  egress       = false
+  protocol     = "tcp"
+  ruleAction   = "allow"
+  cidrBlock    = privateS3PrefixList.cidrBlocks[0]
+  fromPort     = 443
+  toPort       = 443
+}
+
+# A contrived example to test that helper nested records ( `filters`
+# below) generate correctly when using output-versioned function
+# invoke forms.
+amis = invoke("aws:ec2:getAmiIds", {
+  owners = [bar.id]
+  filters = [{name=bar.id, values=["pulumi*"]}]
+})

pkg/codegen/nodejs/gen_program_expressions.go

@@ -328,8 +328,11 @@ func (g *generator) GenFunctionCallExpression(w io.Writer, expr *model.FunctionC
 		if module != "" {
 			module = "." + module
 		}
+		isOut := pcl.IsOutputVersionInvokeCall(expr)
 		name := fmt.Sprintf("%s%s.%s", makeValidIdentifier(pkg), module, fn)
-
+		if isOut {
+			name = fmt.Sprintf("%sOutput", name)
+		}
 		g.Fprintf(w, "%s(", name)
 		if len(expr.Args) >= 2 {
 			g.Fgenf(w, "%.v", expr.Args[1])

pkg/codegen/pcl/invoke.go

@@ -1,4 +1,4 @@
-// Copyright 2016-2020, Pulumi Corporation.
+// Copyright 2016-2021, Pulumi Corporation.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,9 +15,12 @@
 package pcl
 
 import (
+	"fmt"
+
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/hclsyntax"
 	"github.com/pulumi/pulumi/pkg/v3/codegen/hcl2/model"
+	"github.com/pulumi/pulumi/pkg/v3/codegen/schema"
 	"github.com/zclconf/go-cty/cty"
 )
 
@@ -42,46 +45,28 @@ func getInvokeToken(call *hclsyntax.FunctionCallExpr) (string, hcl.Range, bool)
 }
 
 func (b *binder) bindInvokeSignature(args []model.Expression) (model.StaticFunctionSignature, hcl.Diagnostics) {
-	signature := model.StaticFunctionSignature{
-		Parameters: []model.Parameter{
-			{
-				Name: "token",
-				Type: model.StringType,
-			},
-			{
-				Name: "args",
-				Type: model.NewOptionalType(model.DynamicType),
-			},
-			{
-				Name: "provider",
-				Type: model.NewOptionalType(model.StringType),
-			},
-		},
-		ReturnType: model.DynamicType,
-	}
-
 	if len(args) < 1 {
-		return signature, nil
+		return b.zeroSignature(), nil
 	}
 
 	template, ok := args[0].(*model.TemplateExpression)
 	if !ok || len(template.Parts) != 1 {
-		return signature, hcl.Diagnostics{tokenMustBeStringLiteral(args[0])}
+		return b.zeroSignature(), hcl.Diagnostics{tokenMustBeStringLiteral(args[0])}
 	}
 	lit, ok := template.Parts[0].(*model.LiteralValueExpression)
 	if !ok || model.StringType.ConversionFrom(lit.Type()) == model.NoConversion {
-		return signature, hcl.Diagnostics{tokenMustBeStringLiteral(args[0])}
+		return b.zeroSignature(), hcl.Diagnostics{tokenMustBeStringLiteral(args[0])}
 	}
 
 	token, tokenRange := lit.Value.AsString(), args[0].SyntaxNode().Range()
 	pkg, _, _, diagnostics := DecomposeToken(token, tokenRange)
 	if diagnostics.HasErrors() {
-		return signature, diagnostics
+		return b.zeroSignature(), diagnostics
 	}
 
 	pkgSchema, ok := b.options.packageCache.entries[pkg]
 	if !ok {
-		return signature, hcl.Diagnostics{unknownPackage(pkg, tokenRange)}
+		return b.zeroSignature(), hcl.Diagnostics{unknownPackage(pkg, tokenRange)}
 	}
 
 	fn, ok := pkgSchema.functions[token]
@@ -92,22 +77,153 @@ func (b *binder) bindInvokeSignature(args []model.Expression) (model.StaticFunct
 		}
 	}
 	if !ok {
-		return signature, hcl.Diagnostics{unknownFunction(token, tokenRange)}
+		return b.zeroSignature(), hcl.Diagnostics{unknownFunction(token, tokenRange)}
 	}
 
-	// Create args and result types for the schema.
-	if fn.Inputs == nil {
-		signature.Parameters[1].Type = model.NewOptionalType(model.NewObjectType(map[string]model.Type{}))
-	} else {
-		signature.Parameters[1].Type = b.schemaTypeToType(fn.Inputs)
+	sig, err := b.signatureForArgs(fn, args[1])
+	if err != nil {
+		diag := hcl.Diagnostics{errorf(tokenRange, "Invoke binding error: %v", err)}
+		return b.zeroSignature(), diag
 	}
 
-	if fn.Outputs == nil {
-		signature.ReturnType = model.NewObjectType(map[string]model.Type{})
-	} else {
-		signature.ReturnType = b.schemaTypeToType(fn.Outputs)
-	}
-	signature.ReturnType = model.NewPromiseType(signature.ReturnType)
-
-	return signature, nil
+	return sig, nil
+}
+
+func (b *binder) makeSignature(argsType, returnType model.Type) model.StaticFunctionSignature {
+	return model.StaticFunctionSignature{
+		Parameters: []model.Parameter{
+			{
+				Name: "token",
+				Type: model.StringType,
+			},
+			{
+				Name: "args",
+				Type: argsType,
+			},
+			{
+				Name: "provider",
+				Type: model.NewOptionalType(model.StringType),
+			},
+		},
+		ReturnType: returnType,
+	}
+}
+
+func (b *binder) zeroSignature() model.StaticFunctionSignature {
+	return b.makeSignature(model.NewOptionalType(model.DynamicType), model.DynamicType)
+}
+
+func (b *binder) signatureForArgs(fn *schema.Function, args model.Expression) (model.StaticFunctionSignature, error) {
+	if b.useOutputVersion(fn, args) {
+		return b.outputVersionSignature(fn)
+	}
+	return b.regularSignature(fn), nil
+}
+
+// Heuristic to decide when to use `fnOutput` form of a function. Will
+// conservatively prefer `false`. It only decides to return `true` if
+// doing so avoids the need to introduce an `apply` form to
+// accommodate `Output` args (`Promise` args do not count).
+func (b *binder) useOutputVersion(fn *schema.Function, args model.Expression) bool {
+	if !fn.NeedsOutputVersion() {
+		// No code emitted for an `fnOutput` form, impossible.
+		return false
+	}
+
+	outputFormParamType := b.schemaTypeToType(fn.Inputs.InputShape)
+	regularFormParamType := b.schemaTypeToType(fn.Inputs)
+	argsType := args.Type()
+
+	if regularFormParamType.ConversionFrom(argsType) == model.NoConversion &&
+		outputFormParamType.ConversionFrom(argsType) == model.SafeConversion &&
+		model.ContainsOutputs(argsType) {
+		return true
+	}
+
+	return false
+}
+
+func (b *binder) regularSignature(fn *schema.Function) model.StaticFunctionSignature {
+	var argsType model.Type
+	if fn.Inputs == nil {
+		argsType = model.NewOptionalType(model.NewObjectType(map[string]model.Type{}))
+	} else {
+		argsType = b.schemaTypeToType(fn.Inputs)
+	}
+
+	var returnType model.Type
+	if fn.Outputs == nil {
+		returnType = model.NewObjectType(map[string]model.Type{})
+	} else {
+		returnType = b.schemaTypeToType(fn.Outputs)
+	}
+
+	return b.makeSignature(argsType, model.NewPromiseType(returnType))
+}
+
+func (b *binder) outputVersionSignature(fn *schema.Function) (model.StaticFunctionSignature, error) {
+	if !fn.NeedsOutputVersion() {
+		return model.StaticFunctionSignature{}, fmt.Errorf("Function %s does not have an Output version", fn.Token)
+	}
+
+	// Given `fn.NeedsOutputVersion()==true`, can assume `fn.Inputs != nil`, `fn.Outputs != nil`.
+	argsType := b.schemaTypeToType(fn.Inputs.InputShape)
+	returnType := b.schemaTypeToType(fn.Outputs)
+	return b.makeSignature(argsType, model.NewOutputType(returnType)), nil
+}
+
+// Detects invoke calls that use an output version of a function.
+func IsOutputVersionInvokeCall(call *model.FunctionCallExpression) bool {
+	if call.Name == Invoke {
+		// Currently binder.bindInvokeSignature will assign
+		// either DynamicType, a Promise<T>, or an Output<T>
+		// for the return type of an invoke. Output<T> implies
+		// that an output version has been picked.
+		_, returnsOutput := call.Signature.ReturnType.(*model.OutputType)
+		return returnsOutput
+	}
+	return false
+}
+
+// Pattern matches to recognize `__convert(objCons(..))` pattern that
+// is used to annotate object constructors with appropriate nominal
+// types. If the expression matches, returns true followed by the
+// constructor expression and the appropriate type.
+func RecognizeTypedObjectCons(theExpr model.Expression) (bool, *model.ObjectConsExpression, model.Type) {
+	expr, isFunc := theExpr.(*model.FunctionCallExpression)
+	if !isFunc {
+		return false, nil, nil
+	}
+
+	if expr.Name != IntrinsicConvert {
+		return false, nil, nil
+	}
+
+	if len(expr.Args) != 1 {
+		return false, nil, nil
+	}
+
+	objCons, isObjCons := expr.Args[0].(*model.ObjectConsExpression)
+	if !isObjCons {
+		return false, nil, nil
+	}
+
+	return true, objCons, expr.Type()
+}
+
+// Pattern matches to recognize an encoded call to an output-versioned
+// invoke, such as `invoke(token, __convert(objCons(..)))`. If
+// matching, returns the `args` expression and its schema-bound type.
+func RecognizeOutputVersionedInvoke(
+	expr *model.FunctionCallExpression,
+) (bool, *model.ObjectConsExpression, model.Type) {
+	if !IsOutputVersionInvokeCall(expr) {
+		return false, nil, nil
+	}
+
+	if len(expr.Args) < 2 {
+		return false, nil, nil
+	}
+
+	return RecognizeTypedObjectCons(expr.Args[1])
 }