From 2e8bbcc9ddc2e2fdba3590450afbe20f656d0d77 Mon Sep 17 00:00:00 2001 From: joeduffy Date: Sat, 9 Jun 2018 09:11:35 -0700 Subject: [PATCH] Add output marshaling and improve input marsalling This change primarily does two things: * Adds output marshaling. * Adds tests for roundtripping inputs to outputs. It also * Fixes a bug in the verification of asset archives. * Change input types to simply `interface{}` and `map[string]interface{}`. There is no need for wrapper types. This is more idiomatic. * Reject output properties upon marshaling failure. * Don't support time.Time as a marshaling concept. This was getting too cute. It's not clear what its marshaling format ought to be. --- sdk/go/pulumi-language-go/main.go | 1 - sdk/go/pulumi/asset/asset.go | 10 +-- sdk/go/pulumi/context.go | 22 +++--- sdk/go/pulumi/properties.go | 33 --------- sdk/go/pulumi/rpc.go | 115 ++++++++++++++++++++++++++---- sdk/go/pulumi/rpc_test.go | 88 +++++++++++++++++++++++ 6 files changed, 208 insertions(+), 61 deletions(-) create mode 100644 sdk/go/pulumi/rpc_test.go diff --git a/sdk/go/pulumi-language-go/main.go b/sdk/go/pulumi-language-go/main.go index 325ddb7e0..7c3de3369 100644 --- a/sdk/go/pulumi-language-go/main.go +++ b/sdk/go/pulumi-language-go/main.go @@ -93,7 +93,6 @@ func (host *goLanguageHost) GetRequiredPlugins(ctx context.Context, // RPC endpoint for LanguageRuntimeServer::Run func (host *goLanguageHost) Run(ctx context.Context, req *pulumirpc.RunRequest) (*pulumirpc.RunResponse, error) { - // Create the environment we'll use to run the process. This is how we pass the RunInfo to the actual // Go program runtime, to avoid needing any sort of program interface other than just a main entrypoint. env, err := host.constructEnv(req) diff --git a/sdk/go/pulumi/asset/asset.go b/sdk/go/pulumi/asset/asset.go index 02da0b1a4..e544dd2f2 100644 --- a/sdk/go/pulumi/asset/asset.go +++ b/sdk/go/pulumi/asset/asset.go @@ -72,21 +72,21 @@ type archive struct { // NewAssetArchive creates a new archive from an in-memory collection of named assets or other archives. func NewAssetArchive(assets map[string]interface{}) Archive { for k, a := range assets { - if _, ok := a.(*Asset); !ok { - if _, ok2 := a.(*Archive); !ok2 { + if _, ok := a.(Asset); !ok { + if _, ok2 := a.(Archive); !ok2 { contract.Failf( - "expected asset map to contain *Assets and/or *Archives; %s is %v", k, reflect.TypeOf(a)) + "expected asset map to contain Assets and/or Archives; %s is %v", k, reflect.TypeOf(a)) } } } return &archive{assets: assets} } -func NewPathArchive(path string) Archive { +func NewFileArchive(path string) Archive { return &archive{path: path} } -func NewURIArchive(uri string) Archive { +func NewRemoteArchive(uri string) Archive { return &archive{uri: uri} } diff --git a/sdk/go/pulumi/context.go b/sdk/go/pulumi/context.go index 16f32cdac..e9274904a 100644 --- a/sdk/go/pulumi/context.go +++ b/sdk/go/pulumi/context.go @@ -33,7 +33,7 @@ type Context struct { ctx context.Context info RunInfo stackR URN - exports Inputs + exports map[string]interface{} monitor pulumirpc.ResourceMonitorClient monitorConn *grpc.ClientConn engine pulumirpc.EngineClient @@ -73,7 +73,7 @@ func NewContext(ctx context.Context, info RunInfo) (*Context, error) { return &Context{ ctx: ctx, info: info, - exports: make(Inputs), + exports: make(map[string]interface{}), monitorConn: monitorConn, monitor: pulumirpc.NewResourceMonitorClient(monitorConn), engineConn: engineConn, @@ -108,7 +108,7 @@ func (ctx *Context) Parallel() int { return ctx.info.Parallel } func (ctx *Context) DryRun() bool { return ctx.info.DryRun } // Invoke will invoke a provider's function, identified by its token tok. -func (ctx *Context) Invoke(tok string, args Inputs) (Outputs, error) { +func (ctx *Context) Invoke(tok string, args map[string]interface{}) (Outputs, error) { // TODO(joe): implement this. return nil, errors.New("Invoke not yet implemented") } @@ -116,7 +116,7 @@ func (ctx *Context) Invoke(tok string, args Inputs) (Outputs, error) { // ReadResource reads an existing custom resource's state from the resource monitor. Note that resources read in this // way will not be part of the resulting stack's state, as they are presumed to belong to another. func (ctx *Context) ReadResource( - t, name string, id ID, state Inputs, opts ...ResourceOpt) (*ResourceState, error) { + t, name string, id ID, state map[string]interface{}, opts ...ResourceOpt) (*ResourceState, error) { if t == "" { return nil, errors.New("resource type argument cannot be empty") } else if name == "" { @@ -132,7 +132,7 @@ func (ctx *Context) ReadResource( // the "name" part to use in creating a stable and globally unique URN for the object. state contains the goal state // for the resource object and opts contains optional settings that govern the way the resource is created. func (ctx *Context) RegisterResource( - t, name string, custom bool, props Inputs, opts ...ResourceOpt) (*ResourceState, error) { + t, name string, custom bool, props map[string]interface{}, opts ...ResourceOpt) (*ResourceState, error) { if t == "" { return nil, errors.New("resource type argument cannot be empty") } else if name == "" { @@ -218,8 +218,12 @@ func (ctx *Context) RegisterResource( resolveID(ID(resp.Id)) } for _, key := range keys { - // TODO(joe): check for missing keys, etc. - resolveState[key](outprops[key]) + out, err := unmarshalOutput(outprops[key]) + if err != nil { + rejectState[key](err) + } else { + resolveState[key](out) + } } } @@ -329,11 +333,11 @@ type ResourceState struct { } // RegisterResourceOutputs completes the resource registration, attaching an optional set of computed outputs. -func (ctx *Context) RegisterResourceOutputs(urn URN, outs Inputs) error { +func (ctx *Context) RegisterResourceOutputs(urn URN, outs map[string]interface{}) error { return nil } // Export registers a key and value pair with the current context's stack. -func (ctx *Context) Export(name string, value Input) { +func (ctx *Context) Export(name string, value interface{}) { ctx.exports[name] = value } diff --git a/sdk/go/pulumi/properties.go b/sdk/go/pulumi/properties.go index a0129238e..92f5d8091 100644 --- a/sdk/go/pulumi/properties.go +++ b/sdk/go/pulumi/properties.go @@ -15,22 +15,11 @@ package pulumi import ( - "time" - "github.com/spf13/cast" "github.com/pulumi/pulumi/sdk/go/pulumi/asset" ) -// Input is an input property for a resource. It is a discriminated union of either a value or another resource's -// output value, which will make the receiving resource dependent on the resource from which the output came. -type Input interface{} - -// Inputs is a map of property name to value, one for each resource input property. Each value can be a prompt, -// JSON serializable primitive -- bool, string, int, array, or map -- or it can be an *Output, in which case the -// input property will carry dependency information from the resource to which the output belongs. -type Inputs map[string]interface{} - // Output helps encode the relationship between resources in a Pulumi application. Specifically an output property // holds onto a value and the resource it came from. An output value can then be provided when constructing new // resources, allowing that new resource to know both the value as well as the resource the value came from. This @@ -261,15 +250,6 @@ func (out *Output) String() (string, error) { return cast.ToString(v), nil } -// Time retrives the underlying value for this output property as a time. -func (out *Output) Time() (time.Time, error) { - v, err := out.Value() - if err != nil { - return time.Time{}, err - } - return cast.ToTime(v), nil -} - // Uuint retrives the underlying value for this output property as a uint. func (out *Output) Uint() (uint, error) { v, err := out.Value() @@ -496,19 +476,6 @@ func (out *StringOutput) Apply(applier func(string) (interface{}, error)) *Outpu }) } -// TimeOutput is an Output that is typed to return number values. -type TimeOutput Output - -// Value returns the underlying number value. -func (out *TimeOutput) Value() (time.Time, error) { return (*Output)(out).Time() } - -// Apply applies a transformation to the number value when it is available. -func (out *TimeOutput) Apply(applier func(time.Time) (interface{}, error)) *Output { - return (*Output)(out).Apply(func(v interface{}) (interface{}, error) { - return applier(cast.ToTime(v)) - }) -} - // UintOutput is an Output that is typed to return uint values. type UintOutput Output diff --git a/sdk/go/pulumi/rpc.go b/sdk/go/pulumi/rpc.go index 38cb797d4..bfdacee71 100644 --- a/sdk/go/pulumi/rpc.go +++ b/sdk/go/pulumi/rpc.go @@ -20,6 +20,7 @@ import ( structpb "github.com/golang/protobuf/ptypes/struct" "github.com/pkg/errors" + "github.com/spf13/cast" "github.com/pulumi/pulumi/pkg/resource" "github.com/pulumi/pulumi/pkg/resource/plugin" @@ -27,7 +28,7 @@ import ( ) // marshalInputs turns resource property inputs into a gRPC struct suitable for marshaling. -func marshalInputs(props Inputs) ([]string, *structpb.Struct, []URN, error) { +func marshalInputs(props map[string]interface{}) ([]string, *structpb.Struct, []URN, error) { var keys []string for key := range props { keys = append(keys, key) @@ -75,15 +76,8 @@ func marshalInput(v interface{}) (interface{}, []Resource, error) { // Next, look for some well known types. switch t := v.(type) { - case bool, int, uint, int32, uint32, int64, uint64, float32, float64, string: + case bool, int, uint, int8, uint8, int16, uint16, int32, uint32, int64, uint64, float32, float64, string: return t, nil, nil - case CustomResource: - // Resources aren't serializable; instead, serialize a reference to ID, tracking as a dependency.a - e, d, err := marshalInput(t.ID()) - if err != nil { - return nil, nil, err - } - return e, append([]Resource{t}, d...), nil case asset.Asset: return map[string]interface{}{ rpcTokenSpecialSigKey: rpcTokenSpecialAssetSig, @@ -122,6 +116,13 @@ func marshalInput(v interface{}) (interface{}, []Resource, error) { return nil, nil, err } return e, append(t.Deps(), d...), err + case CustomResource: + // Resources aren't serializable; instead, serialize a reference to ID, tracking as a dependency.a + e, d, err := marshalInput(t.ID()) + if err != nil { + return nil, nil, err + } + return e, append([]Resource{t}, d...), nil } // Finally, handle the usual primitives (numbers, strings, arrays, maps, ...) @@ -143,7 +144,7 @@ func marshalInput(v interface{}) (interface{}, []Resource, error) { return arr, deps, nil case reflect.Map: // For maps, only support string-based keys, and recurse into the values. - var obj map[string]interface{} + obj := make(map[string]interface{}) var deps []Resource for _, key := range rv.MapKeys() { k, ok := key.Interface().(string) @@ -152,7 +153,7 @@ func marshalInput(v interface{}) (interface{}, []Resource, error) { errors.Errorf("expected map keys to be strings; got %v", reflect.TypeOf(key.Interface())) } value := rv.MapIndex(key) - v, d, err := marshalInput(value) + v, d, err := marshalInput(value.Interface()) if err != nil { return nil, nil, err } @@ -162,7 +163,7 @@ func marshalInput(v interface{}) (interface{}, []Resource, error) { } return obj, deps, nil case reflect.Ptr: - // For pointerss, recurse into the underlying value. + // For pointers, recurse into the underlying value. if rv.IsNil() { return nil, nil, nil } @@ -171,5 +172,93 @@ func marshalInput(v interface{}) (interface{}, []Resource, error) { return marshalInput(rv.String()) } - return nil, nil, errors.Errorf("unrecognized input property type: %v", reflect.TypeOf(v)) + return nil, nil, errors.Errorf("unrecognized input property type: %v (%v)", v, reflect.TypeOf(v)) +} + +// unmarshalOutputs unmarshals all the outputs into a simple map. +func unmarshalOutputs(outs *structpb.Struct) (map[string]interface{}, error) { + outprops, err := plugin.UnmarshalProperties(outs, plugin.MarshalOptions{}) + if err != nil { + return nil, err + } + + result := make(map[string]interface{}) + for k, v := range outprops.Mappable() { + result[k], err = unmarshalOutput(v) + if err != nil { + return nil, err + } + } + return result, nil +} + +// unmarshalOutput unmarshals a single output variable into its runtime representation. For the most part, this just +// returns the raw value. In a small number of cases, we need to change a type. +func unmarshalOutput(v interface{}) (interface{}, error) { + // In the case of assets and archives, turn these into real asset and archive structures. + if m, ok := v.(map[string]interface{}); ok { + if m[rpcTokenSpecialSigKey] == rpcTokenSpecialAssetSig { + if path := m["path"]; path != nil { + return asset.NewFileAsset(cast.ToString(path)), nil + } else if text := m["text"]; text != nil { + return asset.NewStringAsset(cast.ToString(text)), nil + } else if uri := m["uri"]; uri != nil { + return asset.NewRemoteAsset(cast.ToString(uri)), nil + } + return nil, errors.New("expected asset to be one of File, String, or Remote; got none") + } else if m[rpcTokenSpecialSigKey] == rpcTokenSpecialArchiveSig { + if assets := m["assets"]; assets != nil { + as := make(map[string]interface{}) + for k, v := range assets.(map[string]interface{}) { + a, err := unmarshalOutput(v) + if err != nil { + return nil, err + } + as[k] = a + } + return asset.NewAssetArchive(as), nil + } else if path := m["path"]; path != nil { + return asset.NewFileArchive(cast.ToString(path)), nil + } else if uri := m["uri"]; uri != nil { + return asset.NewRemoteArchive(cast.ToString(uri)), nil + } + return nil, errors.New("expected asset to be one of File, String, or Remote; got none") + } + } + + // For arrays and maps, just make sure to transform them deeply. + rv := reflect.ValueOf(v) + switch rk := rv.Type().Kind(); rk { + case reflect.Array, reflect.Slice: + // If an array or a slice, create a new array by recursing into elements. + var arr []interface{} + for i := 0; i < rv.Len(); i++ { + elem := rv.Index(i) + e, err := unmarshalOutput(elem.Interface()) + if err != nil { + return nil, err + } + arr = append(arr, e) + } + return arr, nil + case reflect.Map: + // For maps, only support string-based keys, and recurse into the values. + var obj map[string]interface{} + for _, key := range rv.MapKeys() { + k, ok := key.Interface().(string) + if !ok { + return nil, errors.Errorf("expected map keys to be strings; got %v", reflect.TypeOf(key.Interface())) + } + value := rv.MapIndex(key) + v, err := unmarshalOutput(value) + if err != nil { + return nil, err + } + + obj[k] = v + } + return obj, nil + } + + return v, nil } diff --git a/sdk/go/pulumi/rpc_test.go b/sdk/go/pulumi/rpc_test.go new file mode 100644 index 000000000..3e3d487c4 --- /dev/null +++ b/sdk/go/pulumi/rpc_test.go @@ -0,0 +1,88 @@ +// Copyright 2016-2018, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pulumi + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/pulumi/pulumi/sdk/go/pulumi/asset" +) + +// TestMarshalRoundtrip ensures that marshaling a complex structure to and from its on-the-wire gRPC format succeeds. +func TestMarshalRoundtrip(t *testing.T) { + // Create interesting inputs. + out, resolve, _ := NewOutput(nil) + resolve("outputty") + input := map[string]interface{}{ + "s": "a string", + "a": true, + "b": 42, + "cStringAsset": asset.NewStringAsset("put a lime in the coconut"), + "cFileAsset": asset.NewFileAsset("foo.txt"), + "cRemoteAsset": asset.NewRemoteAsset("https://pulumi.com/fake/asset.txt"), + "dAssetArchive": asset.NewAssetArchive(map[string]interface{}{ + "subAsset": asset.NewFileAsset("bar.txt"), + "subArchive": asset.NewFileArchive("bar.zip"), + }), + "dFileArchive": asset.NewFileArchive("foo.zip"), + "dRemoteArchive": asset.NewRemoteArchive("https://pulumi.com/fake/archive.zip"), + "e": out, + "fArray": []interface{}{0, 1.3, "x", false}, + "fMap": map[string]interface{}{ + "x": "y", + "y": 999.9, + "z": false, + }, + } + + // Marshal those inputs. + _, m, deps, err := marshalInputs(input) + if !assert.Nil(t, err) { + assert.Equal(t, 0, len(deps)) + + // Now just unmarshal and ensure the resulting map matches. + res, err := unmarshalOutputs(m) + if !assert.Nil(t, err) { + if !assert.NotNil(t, res) { + assert.Equal(t, "a string", res["s"]) + assert.Equal(t, true, res["a"]) + assert.Equal(t, 42, res["b"]) + assert.Equal(t, "put a lime in the coconut", res["cStringAsset"].(asset.Asset).Text()) + assert.Equal(t, "foo.txt", res["cFileAsset"].(asset.Asset).Path()) + assert.Equal(t, "https://pulumi.com/fake/asset.txt", res["cRemoteAsset"].(asset.Asset).URI()) + ar := res["dAssetArchive"].(asset.Archive).Assets() + assert.Equal(t, 2, len(ar)) + assert.Equal(t, "bar.txt", ar["subAsset"].(asset.Asset).Path()) + assert.Equal(t, "bar.zip", ar["subrchive"].(asset.Archive).Path()) + assert.Equal(t, "foo.zip", res["dFileArchive"].(asset.Archive).Path()) + assert.Equal(t, "https://pulumi.com/fake/archive.zip", res["dRemoteArchive"].(asset.Archive).URI()) + assert.Equal(t, "outputty", res["e"]) + aa := res["fArray"].([]interface{}) + assert.Equal(t, 4, len(aa)) + assert.Equal(t, 0, aa[0]) + assert.Equal(t, 1.3, aa[1]) + assert.Equal(t, "x", aa[2]) + assert.Equal(t, false, aa[3]) + am := res["fMap"].(map[string]interface{}) + assert.Equal(t, 3, len(am)) + assert.Equal(t, "y", am["x"]) + assert.Equal(t, 999.9, am["y"]) + assert.Equal(t, false, am["z"]) + } + } + } +}