Go program gen: resource range, readDir, template strings, etc (#4818)

This commit is contained in:
Evan Boyle 2020-06-15 23:00:02 -07:00 committed by GitHub
parent f9074f6bcb
commit 2d61852e4f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 325 additions and 47 deletions

View file

@ -3,8 +3,11 @@ CHANGELOG
## HEAD (Unreleased)
- Go program gen improvements (resource range, readDir, fileArchive)
[#4818](https://github.com/pulumi/pulumi/pull/4818)
- Set default config namespace for Get/Try/Require methods in Go SDK.
[4802](https://github.com/pulumi/pulumi/pull/4802)
[#4802](https://github.com/pulumi/pulumi/pull/4802)
- Improve typing for Go SDK secret config values
[#4800](https://github.com/pulumi/pulumi/pull/4800)

View file

@ -19,10 +19,12 @@ import (
type generator struct {
// The formatter to use when generating code.
*format.Formatter
program *hcl2.Program
diagnostics hcl.Diagnostics
jsonTempSpiller *jsonSpiller
ternaryTempSpiller *tempSpiller
program *hcl2.Program
diagnostics hcl.Diagnostics
jsonTempSpiller *jsonSpiller
ternaryTempSpiller *tempSpiller
readDirTempSpiller *readDirSpiller
scopeTraversalRoots codegen.StringSet
}
func GenerateProgram(program *hcl2.Program) (map[string][]byte, hcl.Diagnostics, error) {
@ -30,9 +32,11 @@ func GenerateProgram(program *hcl2.Program) (map[string][]byte, hcl.Diagnostics,
nodes := hcl2.Linearize(program)
g := &generator{
program: program,
jsonTempSpiller: &jsonSpiller{},
ternaryTempSpiller: &tempSpiller{},
program: program,
jsonTempSpiller: &jsonSpiller{},
ternaryTempSpiller: &tempSpiller{},
readDirTempSpiller: &readDirSpiller{},
scopeTraversalRoots: codegen.NewStringSet(),
}
g.Formatter = format.NewFormatter(g)
@ -40,6 +44,10 @@ func GenerateProgram(program *hcl2.Program) (map[string][]byte, hcl.Diagnostics,
var index bytes.Buffer
g.genPreamble(&index, program)
for _, n := range nodes {
g.collectScopeRoots(n)
}
for _, n := range nodes {
g.genNode(&index, n)
}
@ -58,6 +66,16 @@ func GenerateProgram(program *hcl2.Program) (map[string][]byte, hcl.Diagnostics,
return files, g.diagnostics, nil
}
func (g *generator) collectScopeRoots(n hcl2.Node) {
diags := n.VisitExpressions(nil, func(n model.Expression) (model.Expression, hcl.Diagnostics) {
if st, ok := n.(*model.ScopeTraversalExpression); ok {
g.scopeTraversalRoots.Add(st.RootName)
}
return n, nil
})
contract.Assert(len(diags) == 0)
}
// genPreamble generates package decl, imports, and opens the main func
func (g *generator) genPreamble(w io.Writer, program *hcl2.Program) {
g.Fprint(w, "package main\n\n")
@ -114,6 +132,11 @@ func (g *generator) collectImports(w io.Writer, program *hcl2.Program) (codegen.
stdImports.Add(fnPkg)
}
}
if t, ok := n.(*model.TemplateExpression); ok {
if len(t.Parts) > 1 {
stdImports.Add("fmt")
}
}
return n, nil
})
contract.Assert(len(diags) == 0)
@ -159,21 +182,41 @@ func (g *generator) genResource(w io.Writer, r *hcl2.Resource) {
g.genTemps(w, temps)
}
g.Fgenf(w, "%s, err := %s.New%s(ctx, \"%[1]s\", ", resName, mod, typ)
if len(r.Inputs) > 0 {
g.Fgenf(w, "&%s.%sArgs{\n", mod, typ)
for _, attr := range r.Inputs {
g.Fgenf(w, "%s: ", strings.Title(attr.Name))
g.Fgenf(w, "%.v,\n", attr.Value)
instantiate := func(varName, resourceName string) {
if g.scopeTraversalRoots.Has(varName) {
g.Fgenf(w, "%s, err := %s.New%s(ctx, %s, ", varName, mod, typ, resourceName)
} else {
g.Fgenf(w, "_, err = %s.New%s(ctx, %s, ", mod, typ, resourceName)
}
g.Fgenf(w, "})\n")
} else {
g.Fgenf(w, "nil)\n")
if len(r.Inputs) > 0 {
g.Fgenf(w, "&%s.%sArgs{\n", mod, typ)
for _, attr := range r.Inputs {
g.Fgenf(w, "%s: ", strings.Title(attr.Name))
g.Fgenf(w, "%.v,\n", attr.Value)
}
g.Fgenf(w, "})\n")
} else {
g.Fgenf(w, "nil)\n")
}
g.Fgenf(w, "if err != nil {\n")
g.Fgenf(w, "return err\n")
g.Fgenf(w, "}\n")
}
if r.Options != nil && r.Options.Range != nil {
rangeType := model.ResolveOutputs(r.Options.Range.Type())
rangeExpr, temps := g.lowerExpression(r.Options.Range, rangeType, false)
g.genTemps(w, temps)
g.Fgenf(w, "for i0, val0 := range %.v {\n", rangeExpr)
instantiate("_", fmt.Sprintf("\"%s-\"+ string(i0)", resName))
g.Fgenf(w, "}\n")
} else {
instantiate(resName, fmt.Sprintf("\"%s\"", resName))
}
g.Fgenf(w, "if err != nil {\n")
g.Fgenf(w, "return err\n")
g.Fgenf(w, "}\n")
}
@ -183,8 +226,30 @@ func (g *generator) genOutputAssignment(w io.Writer, v *hcl2.OutputVariable) {
g.genTemps(w, temps)
g.Fgenf(w, "ctx.Export(\"%s\", %.3v)\n", v.Name(), expr)
}
func (g *generator) genTemps(w io.Writer, temps []interface{}) {
singleReturn := ""
g.genTempsMultiReturn(w, temps, singleReturn)
}
func (g *generator) genTempsMultiReturn(w io.Writer, temps []interface{}, zeroValueType string) {
genZeroValueDecl := false
if zeroValueType != "" {
for _, t := range temps {
switch t.(type) {
case *jsonTemp, *readDirTemp:
genZeroValueDecl = true
default:
}
}
if genZeroValueDecl {
// TODO add entropy to var name
// currently only used inside anonymous functions (no scope collisions)
g.Fgenf(w, "var _zero %s\n", zeroValueType)
}
}
for _, t := range temps {
switch t := t.(type) {
case *ternaryTemp:
@ -202,9 +267,32 @@ func (g *generator) genTemps(w io.Writer, temps []interface{}) {
args := stripInputs(t.Value.Args[0])
g.Fgenf(w, "%.v)\n", args)
g.Fgenf(w, "if err != nil {\n")
g.Fgenf(w, "return err\n")
if genZeroValueDecl {
g.Fgenf(w, "return _zero, err\n")
} else {
g.Fgenf(w, "return err\n")
}
g.Fgenf(w, "}\n")
g.Fgenf(w, "%s := string(%s)\n", t.Name, bytesVar)
case *readDirTemp:
tmpSuffix := strings.Split(t.Name, "files")[1]
g.Fgenf(w, "%s, err := ioutil.ReadDir(%.v)\n", t.Name, t.Value.Args[0])
g.Fgenf(w, "if err != nil {\n")
if genZeroValueDecl {
g.Fgenf(w, "return _zero, err\n")
} else {
g.Fgenf(w, "return err\n")
}
g.Fgenf(w, "}\n")
namesVar := fmt.Sprintf("fileNames%s", tmpSuffix)
g.Fgenf(w, "%s := make([]string, len(%s))\n", namesVar, t.Name)
iVar := fmt.Sprintf("i%s", tmpSuffix)
valVar := fmt.Sprintf("val%s", tmpSuffix)
g.Fgenf(w, "for %s, %s := range %s {\n", iVar, valVar, t.Name)
g.Fgenf(w, "%s[%s] = %s.Name()\n", namesVar, iVar, valVar)
g.Fgenf(w, "}\n")
default:
contract.Failf("unexpected temp type: %v", t)
}
}
}
@ -213,17 +301,21 @@ func (g *generator) genLocalVariable(w io.Writer, v *hcl2.LocalVariable) {
isInput := false
expr, temps := g.lowerExpression(v.Definition.Value, v.Type(), isInput)
g.genTemps(w, temps)
name := v.Name()
if !g.scopeTraversalRoots.Has(name) {
name = "_"
}
switch expr := expr.(type) {
case *model.FunctionCallExpression:
switch expr.Name {
case hcl2.Invoke:
g.Fgenf(w, "%s, err := %.3v;\n", v.Name(), expr)
g.Fgenf(w, "%s, err := %.3v;\n", name, expr)
g.Fgenf(w, "if err != nil {\n")
g.Fgenf(w, "return err\n")
g.Fgenf(w, "}\n")
}
default:
g.Fgenf(w, "%s := %.3v;\n", v.Name(), expr)
g.Fgenf(w, "%s := %.3v;\n", name, expr)
}

View file

@ -73,7 +73,11 @@ func (g *generator) GenAnonymousFunctionExpression(w io.Writer, expr *model.Anon
isInput := isInputty(expr.Signature.ReturnType)
retType := argumentTypeName(nil, expr.Signature.ReturnType, isInput)
g.Fgenf(w, ") (%s, error) {\n", retType)
g.Fgenf(w, "return %v, nil", expr.Body)
body, temps := g.lowerExpression(expr.Body, expr.Signature.ReturnType, isInput)
g.genTempsMultiReturn(w, temps, retType)
g.Fgenf(w, "return %v, nil", body)
g.Fgenf(w, "\n}")
}
@ -171,8 +175,7 @@ func (g *generator) GenFunctionCallExpression(w io.Writer, expr *model.FunctionC
g.genNYI(w, "call %v", expr.Name)
// g.Fgenf(w, "new FileArchive(%.v)", expr.Args[0])
case "fileAsset":
g.genNYI(w, "call %v", expr.Name)
// g.Fgenf(w, "new FileAsset(%.v)", expr.Args[0])
g.Fgenf(w, "pulumi.NewFileAsset(%.v)", expr.Args[0])
case hcl2.Invoke:
_, module, fn, diags := functionName(expr.Args[0])
contract.Assert(len(diags) == 0)
@ -200,14 +203,13 @@ func (g *generator) GenFunctionCallExpression(w io.Writer, expr *model.FunctionC
case "readFile":
g.genNYI(w, "ReadFile")
case "readDir":
// TODO
g.genNYI(w, "call %v", expr.Name)
// C# for reference
// g.Fgenf(w, "Directory.GetFiles(%.v).Select(Path.GetFileName)", expr.Args[0])
contract.Failf("unlowered toJSON function expression @ %v", expr.SyntaxNode().Range())
case "split":
g.Fgenf(w, "%.20v.Split(%v)", expr.Args[1], expr.Args[0])
case "toJSON":
contract.Failf("unlowered toJSON function expression @ %v", expr.SyntaxNode().Range())
case "mimeType":
g.Fgenf(w, "mime.TypeByExtension(path.Ext(%.v))", expr.Args[0])
default:
g.genNYI(w, "call %v", expr.Name)
}
@ -373,8 +375,21 @@ func (g *generator) genScopeTraversalExpression(w io.Writer, expr *model.ScopeTr
if isInput {
g.Fgenf(w, "%s(", argumentTypeName(expr, expr.Type(), isInput))
}
g.Fgen(w, rootName)
g.genRelativeTraversal(w, expr.Traversal.SimpleSplit().Rel, expr.Parts, objType)
if rootName == "range" {
part := expr.Traversal[1].(hcl.TraverseAttr).Name
switch part {
case "value":
g.Fgenf(w, "val0")
case "key":
g.Fgenf(w, "key0")
default:
contract.Failf("unexpected traversal on range expression: %s", part)
}
} else {
g.Fgen(w, rootName)
g.genRelativeTraversal(w, expr.Traversal.SimpleSplit().Rel, expr.Parts, objType)
}
if isInput {
g.Fgenf(w, ")")
@ -395,9 +410,15 @@ func (g *generator) GenTemplateExpression(w io.Writer, expr *model.TemplateExpre
g.GenLiteralValueExpression(w, lit)
return
}
} else {
fmtMaker := make([]string, len(expr.Parts)+1)
fmtStr := strings.Join(fmtMaker, "%v")
g.Fgenf(w, "fmt.Sprintf(\"%s\"", fmtStr)
for _, v := range expr.Parts {
g.Fgenf(w, ", %.v", v)
}
g.Fgenf(w, ")")
}
g.genNYI(w, "TODO multi part template expressions")
}
// GenTemplateJoinExpression generates code for a TemplateJoinExpression.
@ -608,10 +629,11 @@ func (nameInfo) Format(name string) string {
// lowerExpression amends the expression with intrinsics for C# generation.
func (g *generator) lowerExpression(expr model.Expression, typ model.Type, isInput bool) (
model.Expression, []interface{}) {
expr, tTemps, ternDiags := g.rewriteTernaries(expr, g.ternaryTempSpiller)
expr, jTemps, jsonDiags := g.rewriteToJSON(expr, g.jsonTempSpiller)
expr, diags := hcl2.RewriteApplies(expr, nameInfo(0), false /*TODO*/)
expr = hcl2.RewriteConversions(expr, typ)
expr, tTemps, ternDiags := g.rewriteTernaries(expr, g.ternaryTempSpiller)
expr, jTemps, jsonDiags := g.rewriteToJSON(expr, g.jsonTempSpiller)
expr, rTemps, readDirDiags := g.rewriteReadDir(expr, g.readDirTempSpiller)
if isInput {
expr = rewriteInputs(expr)
@ -623,8 +645,12 @@ func (g *generator) lowerExpression(expr model.Expression, typ model.Type, isInp
for _, t := range jTemps {
temps = append(temps, t)
}
for _, t := range rTemps {
temps = append(temps, t)
}
diags = append(diags, ternDiags...)
diags = append(diags, jsonDiags...)
diags = append(diags, readDirDiags...)
contract.Assert(len(diags) == 0)
return expr, temps
}
@ -646,7 +672,10 @@ func (g *generator) genApply(w io.Writer, expr *model.FunctionCallExpression) {
isInput := false
retType := argumentTypeName(nil, then.Signature.ReturnType, isInput)
// TODO account for outputs in other namespaces like aws
typeAssertion := fmt.Sprintf(".(pulumi.%sOutput)", Title(retType))
typeAssertion := fmt.Sprintf(".(%sOutput)", retType)
if !strings.HasPrefix(retType, "pulumi.") {
typeAssertion = fmt.Sprintf(".(pulumi.%sOutput)", Title(retType))
}
if len(applyArgs) == 1 {
// If we only have a single output, just generate a normal `.Apply`
@ -744,7 +773,9 @@ func functionName(tokenArg model.Expression) (string, string, string, hcl.Diagno
}
var functionPackages = map[string][]string{
"toJSON": {"encoding/json"},
"toJSON": {"encoding/json"},
"readDir": {"io/ioutil"},
"mimeType": {"mime", "path"},
}
func (g *generator) genFunctionPackages(x *model.FunctionCallExpression) []string {

View file

@ -58,7 +58,8 @@ func modifyInputs(
return x
}
switch expr.Name {
case "mimeType":
return modf(x)
case hcl2.IntrinsicConvert:
switch rt := expr.Signature.ReturnType.(type) {
case *model.UnionType:

View file

@ -0,0 +1,70 @@
package gen
import (
"fmt"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/pulumi/pulumi/pkg/v2/codegen/hcl2/model"
"github.com/pulumi/pulumi/pkg/v2/codegen/hcl2/syntax"
)
type readDirTemp struct {
Name string
Value *model.FunctionCallExpression
}
func (rt *readDirTemp) Type() model.Type {
return rt.Value.Type()
}
func (rt *readDirTemp) Traverse(traverser hcl.Traverser) (model.Traversable, hcl.Diagnostics) {
return rt.Type().Traverse(traverser)
}
func (rt *readDirTemp) SyntaxNode() hclsyntax.Node {
return syntax.None
}
type readDirSpiller struct {
temps []*readDirTemp
count int
}
func (rs *readDirSpiller) spillExpression(x model.Expression) (model.Expression, hcl.Diagnostics) {
var temp *readDirTemp
scopeName := ""
switch x := x.(type) {
case *model.FunctionCallExpression:
switch x.Name {
case "readDir":
scopeName = fmt.Sprintf("fileNames%d", rs.count)
temp = &readDirTemp{
Name: fmt.Sprintf("files%d", rs.count),
Value: x,
}
rs.temps = append(rs.temps, temp)
rs.count++
default:
return x, nil
}
default:
return x, nil
}
return &model.ScopeTraversalExpression{
RootName: scopeName,
Traversal: hcl.Traversal{hcl.TraverseRoot{Name: ""}},
Parts: []model.Traversable{temp},
}, nil
}
func (g *generator) rewriteReadDir(
x model.Expression,
spiller *readDirSpiller,
) (model.Expression, []*readDirTemp, hcl.Diagnostics) {
spiller.temps = nil
x, diags := model.VisitExpression(x, spiller.spillExpression, nil)
return x, spiller.temps, diags
}

View file

@ -8,6 +8,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/pulumi/pulumi/pkg/v2/codegen"
"github.com/pulumi/pulumi/pkg/v2/codegen/hcl2"
"github.com/pulumi/pulumi/pkg/v2/codegen/hcl2/model/format"
"github.com/pulumi/pulumi/pkg/v2/codegen/hcl2/syntax"
@ -28,6 +29,7 @@ func TestGenProgram(t *testing.T) {
}
// TODO: include all test files
if filepath.Base(f.Name()) != "aws-s3-logging.pp" &&
filepath.Base(f.Name()) != "aws-s3-folder.pp" &&
filepath.Base(f.Name()) != "aws-fargate.pp" {
continue
}
@ -63,7 +65,7 @@ func TestGenProgram(t *testing.T) {
files, diags, err := GenerateProgram(program)
assert.NoError(t, err)
if diags.HasErrors() {
t.Fatalf("failed to bind program: %v", diags)
t.Fatalf("failed to generate program: %v", diags)
}
assert.Equal(t, string(expected), string(files["main.go"]))
})
@ -116,9 +118,11 @@ func newTestGenerator(t *testing.T, testFile string) *generator {
}
g := &generator{
program: program,
jsonTempSpiller: &jsonSpiller{},
ternaryTempSpiller: &tempSpiller{},
program: program,
jsonTempSpiller: &jsonSpiller{},
ternaryTempSpiller: &tempSpiller{},
readDirTempSpiller: &readDirSpiller{},
scopeTraversalRoots: codegen.NewStringSet(),
}
g.Formatter = format.NewFormatter(g)
return g

View file

@ -77,7 +77,7 @@ func main() {
if err != nil {
return err
}
taskExecRolePolicyAttachment, err := iam.NewRolePolicyAttachment(ctx, "taskExecRolePolicyAttachment", &iam.RolePolicyAttachmentArgs{
_, err = iam.NewRolePolicyAttachment(ctx, "taskExecRolePolicyAttachment", &iam.RolePolicyAttachmentArgs{
Role: taskExecRole.Name,
PolicyArn: pulumi.String("arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"),
})
@ -146,7 +146,7 @@ func main() {
if err != nil {
return err
}
appService, err := ecs.NewService(ctx, "appService", &ecs.ServiceArgs{
_, err = ecs.NewService(ctx, "appService", &ecs.ServiceArgs{
Cluster: cluster.Arn,
DesiredCount: pulumi.Int(5),
LaunchType: pulumi.String("FARGATE"),

View file

@ -0,0 +1,77 @@
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"mime"
"path"
"github.com/pulumi/pulumi-aws/sdk/v2/go/aws/s3"
"github.com/pulumi/pulumi/sdk/v2/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
siteBucket, err := s3.NewBucket(ctx, "siteBucket", &s3.BucketArgs{
Website: &s3.BucketWebsiteArgs{
IndexDocument: pulumi.String("index.html"),
},
})
if err != nil {
return err
}
siteDir := "www"
files0, err := ioutil.ReadDir(siteDir)
if err != nil {
return err
}
fileNames0 := make([]string, len(files0))
for i0, val0 := range files0 {
fileNames0[i0] = val0.Name()
}
for i0, val0 := range fileNames0 {
_, err = s3.NewBucketObject(ctx, "files-"+string(i0), &s3.BucketObjectArgs{
Bucket: siteBucket.ID(),
Key: pulumi.String(val0),
Source: pulumi.NewFileAsset(fmt.Sprintf("%v%v%v", siteDir, "/", val0)),
ContentType: pulumi.String(mime.TypeByExtension(path.Ext(val0))),
})
if err != nil {
return err
}
}
_, err = s3.NewBucketPolicy(ctx, "bucketPolicy", &s3.BucketPolicyArgs{
Bucket: siteBucket.ID(),
Policy: siteBucket.ID().ApplyT(func(id string) (pulumi.String, error) {
var _zero pulumi.String
tmpJSON0, err := json.Marshal(map[string]interface{}{
"Version": "2012-10-17",
"Statement": []map[string]interface{}{
map[string]interface{}{
"Effect": "Allow",
"Principal": "*",
"Action": []string{
"s3:GetObject",
},
"Resource": []string{
fmt.Sprintf("%v%v%v", "arn:aws:s3:::", id, "/*"),
},
},
},
})
if err != nil {
return _zero, err
}
json0 := string(tmpJSON0)
return pulumi.String(json0), nil
}).(pulumi.StringOutput),
})
if err != nil {
return err
}
ctx.Export("bucketName", siteBucket.Bucket)
ctx.Export("websiteUrl", siteBucket.WebsiteEndpoint)
return nil
})
}