maxmustermann/pulumi
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Initial support for (un)marshaling output values (#7861)

Browse source 
This change expands the definition of `resource.Output` in the Go SDK with additional information about the output, i.e. dependencies and secretness, and adds support in the core Go RPC code for (un)marshaling output values.

Output values are marshaled as special objects ala archives, assets, and resource refs and are unmarshaled as `resource.Output` values.

Subsequent PRs will add:
 - A monitor feature for output values, which will initially be disabled by default but available to turn on via an envvar
 - Support for (un)marshaling output values in each language SDKs
 - A way for providers to indicate support for receiving output values
 - E2E tests
 - Turn the monitor feature on by default (w/ env var to disable) (Note: the current plan is to initially scope this to only be used when marshaling inputs to a multi-language component)
This commit is contained in:
Justin Van Patten 2021-09-13 09:05:31 -07:00 committed by GitHub
parent e696fb6c50
commit 0c0684af5c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 1625 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
CHANGELOG_PENDING.md
View file

@ -1,7 +1,10 @@
### Improvements
- [sdk] Improve error messages for (un)marshalling properties
- [sdk/go] - Improve error messages for (un)marshalling properties.
[#7936](https://github.com/pulumi/pulumi/pull/7936)
- [sdk/go] - Initial support for (un)marshalling output values.
[#7861](https://github.com/pulumi/pulumi/pull/7861)
### Bug Fixes

1
pkg/go.sum
View file

@ -1141,6 +1141,7 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

1
sdk/go.mod
View file

@ -47,5 +47,6 @@ require (
gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
gopkg.in/src-d/go-git.v4 v4.13.1
gopkg.in/yaml.v2 v2.2.8
pgregory.net/rapid v0.4.7
sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0
)

2
sdk/go.sum
View file

@ -344,5 +344,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g=
pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU=
sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 h1:ucqkfpjg9WzSUubAO62csmucvxl4/JeW3F4I4909XkM=
sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=

103
sdk/go/common/resource/plugin/rpc.go
View file

@ -1,4 +1,4 @@
// Copyright 2016-2018, Pulumi Corporation.
// Copyright 2016-2021, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -39,6 +39,7 @@ type MarshalOptions struct {
RejectAssets bool // true if we should return errors on Asset and Archive values.
KeepResources bool // true if we are keeping resoures (otherwise we return raw urn).
SkipInternalKeys bool // true to skip internal property keys (keys that start with "__") in the resulting map.
KeepOutputValues bool // true if we are keeping output values.
}
const (
@ -71,7 +72,7 @@ func MarshalProperties(props resource.PropertyMap, opts MarshalOptions) (*struct
for _, key := range props.StableKeys() {
v := props[key]
logging.V(9).Infof("Marshaling property for RPC[%s]: %s=%v", opts.Label, key, v)
if v.IsOutput() {
if v.IsOutput() && !v.OutputValue().Known && !opts.KeepOutputValues {
logging.V(9).Infof("Skipping output property for RPC[%s]: %v", opts.Label, key)
} else if opts.SkipNulls && v.IsNull() {
logging.V(9).Infof("Skipping null property for RPC[%s]: %s (as requested)", opts.Label, key)
@ -150,12 +151,35 @@ func MarshalPropertyValue(key resource.PropertyKey, v resource.PropertyValue,
}
return nil, nil // return nil and the caller will ignore it.
} else if v.IsOutput() {
// Note that at the moment we don't differentiate between computed and output properties on the wire. As
// a result, they will show up as computed on the other end. This distinction isn't currently interesting.
if opts.KeepUnknowns {
return marshalUnknownProperty(v.OutputValue().Element, opts), nil
if !opts.KeepOutputValues {
result := v.OutputValue().Element
if !v.OutputValue().Known {
// Unknown outputs are marshaled the same as Computed.
result = resource.MakeComputed(result)
}
if v.OutputValue().Secret {
result = resource.MakeSecret(result)
}
return MarshalPropertyValue(key, result, opts)
}
return nil, nil // return nil and the caller will ignore it.
obj := resource.PropertyMap{
resource.SigKey: resource.NewStringProperty(resource.OutputValueSig),
}
if v.OutputValue().Known {
obj["value"] = v.OutputValue().Element
}
if v.OutputValue().Secret {
obj["secret"] = resource.NewBoolProperty(v.OutputValue().Secret)
}
if len(v.OutputValue().Dependencies) > 0 {
deps := make([]resource.PropertyValue, len(v.OutputValue().Dependencies))
for i, dep := range v.OutputValue().Dependencies {
deps[i] = resource.NewStringProperty(string(dep))
}
obj["dependencies"] = resource.NewArrayProperty(deps)
}
output := resource.NewObjectProperty(obj)
return MarshalPropertyValue(key, output, opts)
} else if v.IsSecret() {
if !opts.KeepSecrets {
logging.V(5).Infof("marshalling secret value as raw value as opts.KeepSecrets is false")
@ -366,12 +390,7 @@ func UnmarshalPropertyValue(key resource.PropertyKey, v *structpb.Value,
if !ok {
return nil, fmt.Errorf("malformed RPC secret: missing value for %q", key)
}
if !opts.KeepSecrets {
logging.V(5).Infof("unmarshalling secret as raw value, as opts.KeepSecrets is false")
return &value, nil
}
s := resource.MakeSecret(value)
return &s, nil
return unmarshalSecretPropertyValue(value, opts), nil
case resource.ResourceReferenceSig:
urn, ok := obj["urn"]
if !ok {
@ -425,6 +444,55 @@ func UnmarshalPropertyValue(key resource.PropertyKey, v *structpb.Value,
ref = resource.MakeComponentResourceReference(resource.URN(urn.StringValue()), packageVersion)
}
return &ref, nil
case resource.OutputValueSig:
value, known := obj["value"]
var secret bool
if secretProp, ok := obj["secret"]; ok {
if !secretProp.IsBool() {
return nil, fmt.Errorf("malformed output value for %q: secret not a bool", key)
}
secret = secretProp.BoolValue()
}
if !opts.KeepOutputValues {
result := &value
if !known {
result, err = UnmarshalPropertyValue(key, &structpb.Value{
Kind: &structpb.Value_StringValue{StringValue: UnknownStringValue},
}, opts)
if err != nil {
return nil, err
}
}
if secret && result != nil {
result = unmarshalSecretPropertyValue(*result, opts)
}
return result, nil
}
var dependencies []resource.URN
if dependenciesProp, ok := obj["dependencies"]; ok {
if !dependenciesProp.IsArray() {
return nil, fmt.Errorf("malformed output value for %q: dependencies not an array", key)
}
dependencies = make([]resource.URN, len(dependenciesProp.ArrayValue()))
for i, dep := range dependenciesProp.ArrayValue() {
if !dep.IsString() {
return nil, fmt.Errorf(
"malformed output value for %q: element in dependencies not a string", key)
}
dependencies[i] = resource.URN(dep.StringValue())
}
}
output := resource.NewOutputProperty(resource.Output{
Element: value,
Known: known,
Secret: secret,
Dependencies: dependencies,
})
return &output, nil
default:
return nil, fmt.Errorf("unrecognized signature '%v' in property map for %q", sig, key)
}
@ -461,6 +529,15 @@ func unmarshalUnknownPropertyValue(s string, opts MarshalOptions) (resource.Prop
return resource.PropertyValue{}, false
}
func unmarshalSecretPropertyValue(v resource.PropertyValue, opts MarshalOptions) *resource.PropertyValue {
if !opts.KeepSecrets {
logging.V(5).Infof("unmarshalling secret as raw value, as opts.KeepSecrets is false")
return &v
}
s := resource.MakeSecret(v)
return &s
}
// MarshalNull marshals a nil to its protobuf form.
func MarshalNull(opts MarshalOptions) *structpb.Value {
return &structpb.Value{

86
sdk/go/common/resource/plugin/rpc_rapid_test.go Normal file
View file

@ -0,0 +1,86 @@
// Copyright 2016-2021, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package plugin
import (
"testing"
"github.com/stretchr/testify/assert"
"pgregory.net/rapid"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
)
func urnGen() *rapid.Generator {
return rapid.StringMatching(`urn:pulumi:a::b::c:d:e::[abcd][123]`).
Map(func(x string) resource.URN { return resource.URN(x) })
}
// Generates PropertyValue values.
func propertyValueGen() *rapid.Generator {
return rapid.Just(resource.NewNullProperty())
}
// Generates Output values.
func outputGen() *rapid.Generator {
propertyValueG := propertyValueGen()
urnsG := rapid.SliceOf(urnGen())
return rapid.Custom(func(t *rapid.T) resource.Output {
element := propertyValueG.Draw(t, "element").(resource.PropertyValue)
known := rapid.Bool().Draw(t, "known").(bool)
secret := rapid.Bool().Draw(t, "secret").(bool)
deps := urnsG.Draw(t, "dependencies").([]resource.URN)
return resource.Output{
Element: element,
Known: known,
Secret: secret,
Dependencies: deps,
}
})
}
func normOutput(pv *resource.PropertyValue) {
if pv.IsOutput() {
out := pv.OutputValue()
if len(out.Dependencies) == 0 && out.Dependencies != nil {
out.Dependencies = nil
}
pv.V = out
}
}
func TestOutputValueTurnaround(t *testing.T) {
rapid.Check(t, func(t *rapid.T) {
out := outputGen().Draw(t, "output").(resource.Output)
v := resource.NewOutputProperty(out)
opts := MarshalOptions{KeepOutputValues: true}
pb, err := MarshalPropertyValue("", v, opts)
assert.NoError(t, err)
if err != nil {
t.FailNow()
}
v2, err := UnmarshalPropertyValue("", pb, opts)
assert.NoError(t, err)
if err != nil {
t.FailNow()
}
assert.NotNil(t, v2)
normOutput(&v)
normOutput(v2)
assert.Equal(t, v, *v2)
})
}

1217
sdk/go/common/resource/plugin/rpc_test.go
View file

File diff suppressed because it is too large Load diff

37
sdk/go/common/resource/properties.go
View file

@ -1,4 +1,4 @@
// Copyright 2016-2018, Pulumi Corporation.
// Copyright 2016-2021, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -84,7 +84,10 @@ type Computed struct {
// encountered, it means the resource has not yet been created, and so the output value is unavailable. Note that an
// output property is a special case of computed, but carries additional semantic meaning.
type Output struct {
Element PropertyValue // the eventual value (type) of the output property.
Element PropertyValue // the value of this output if it is resolved.
Known bool `json:"-"` // true if this output's value is known.
Secret bool `json:"-"` // true if this output's value is secret.
Dependencies []URN `json:"-"` // the dependencies associated with this output.
}
// Secret indicates that the underlying value should be persisted securely.
@ -364,12 +367,15 @@ func NewPropertyValueRepl(v interface{},
// HasValue returns true if a value is semantically meaningful.
func (v PropertyValue) HasValue() bool {
return !v.IsNull() && !v.IsOutput()
if v.IsOutput() {
return v.OutputValue().Known
}
return !v.IsNull()
}
// ContainsUnknowns returns true if the property value contains at least one unknown (deeply).
func (v PropertyValue) ContainsUnknowns() bool {
if v.IsComputed() || v.IsOutput() {
if v.IsComputed() || (v.IsOutput() && !v.OutputValue().Known) {
return true
} else if v.IsArray() {
for _, e := range v.ArrayValue() {
@ -392,7 +398,7 @@ func (v PropertyValue) ContainsSecrets() bool {
} else if v.IsComputed() {
return v.Input().Element.ContainsSecrets()
} else if v.IsOutput() {
return v.OutputValue().Element.ContainsSecrets()
return v.OutputValue().Secret || v.OutputValue().Element.ContainsSecrets()
} else if v.IsArray() {
for _, e := range v.ArrayValue() {
if e.ContainsSecrets() {
@ -530,7 +536,12 @@ func (v PropertyValue) TypeString() string {
} else if v.IsComputed() {
return "output<" + v.Input().Element.TypeString() + ">"
} else if v.IsOutput() {
return "output<" + v.OutputValue().Element.TypeString() + ">"
if !v.OutputValue().Known {
return MakeComputed(v.OutputValue().Element).TypeString()
} else if v.OutputValue().Secret {
return MakeSecret(v.OutputValue().Element).TypeString()
}
return v.OutputValue().Element.TypeString()
} else if v.IsSecret() {
return "secret<" + v.SecretValue().Element.TypeString() + ">"
} else if v.IsResourceReference() {
@ -588,9 +599,16 @@ func (v PropertyValue) MapRepl(replk func(string) (string, bool),
// String implements the fmt.Stringer interface to add slightly more information to the output.
func (v PropertyValue) String() string {
if v.IsComputed() || v.IsOutput() {
// For computed and output properties, show their type followed by an empty object string.
if v.IsComputed() {
// For computed properties, show the type followed by an empty object string.
return fmt.Sprintf("%v{}", v.TypeString())
} else if v.IsOutput() {
if !v.OutputValue().Known {
return MakeComputed(v.OutputValue().Element).String()
} else if v.OutputValue().Secret {
return MakeSecret(v.OutputValue().Element).String()
}
return v.OutputValue().Element.String()
}
// For all others, just display the underlying property value.
return fmt.Sprintf("{%v}", v.V)
@ -620,6 +638,9 @@ const SecretSig = "1b47061264138c4ac30d75fd1eb44270"
// ResourceReferenceSig is the unique resource reference signature.
const ResourceReferenceSig = "5cf8f73096256a8f31e491e813e4eb8e"
// OutputValueSig is the unique output value signature.
const OutputValueSig = "d0e6a833031e9bbcd3f4e8bde6ca49a4"
// IsInternalPropertyKey returns true if the given property key is an internal key that should not be displayed to
// users.
func IsInternalPropertyKey(key PropertyKey) bool {

197
sdk/go/common/resource/properties_test.go
View file

@ -1,4 +1,4 @@
// Copyright 2016-2018, Pulumi Corporation.
// Copyright 2016-2021, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -170,3 +170,198 @@ func TestSecretUnknown(t *testing.T) {
assert.True(t, c.ContainsUnknowns())
assert.True(t, co.ContainsUnknowns())
}
func TestTypeString(t *testing.T) {
tests := []struct {
prop PropertyValue
expected string
}{
{
prop: MakeComputed(NewStringProperty("")),
expected: "output<string>",
},
{
prop: MakeSecret(NewStringProperty("")),
expected: "secret<string>",
},
{
prop: MakeOutput(NewStringProperty("")),
expected: "output<string>",
},
{
prop: NewOutputProperty(Output{
Element: NewStringProperty(""),
Known: true,
}),
expected: "string",
},
{
prop: NewOutputProperty(Output{
Element: NewStringProperty(""),
Known: true,
Secret: true,
}),
expected: "secret<string>",
},
}
for _, tt := range tests {
t.Run(tt.expected, func(t *testing.T) {
assert.Equal(t, tt.expected, tt.prop.TypeString())
})
}
}
func TestString(t *testing.T) {
tests := []struct {
prop PropertyValue
expected string
}{
{
prop: MakeComputed(NewStringProperty("")),
expected: "output<string>{}",
},
{
prop: MakeSecret(NewStringProperty("shh")),
expected: "{&{{shh}}}",
},
{
prop: MakeOutput(NewStringProperty("")),
expected: "output<string>{}",
},
{
prop: NewOutputProperty(Output{
Element: NewStringProperty("hello"),
Known: true,
}),
expected: "{hello}",
},
{
prop: NewOutputProperty(Output{
Element: NewStringProperty("shh"),
Known: true,
Secret: true,
}),
expected: "{&{{shh}}}",
},
}
for _, tt := range tests {
t.Run(tt.expected, func(t *testing.T) {
assert.Equal(t, tt.expected, tt.prop.String())
})
}
}
func TestContainsUnknowns(t *testing.T) {
tests := []struct {
name string
prop PropertyValue
expected bool
}{
{
name: "computed unknown",
prop: MakeComputed(NewStringProperty("")),
expected: true,
},
{
name: "output unknown",
prop: MakeOutput(NewStringProperty("")),
expected: true,
},
{
name: "output known",
prop: NewOutputProperty(Output{
Element: NewStringProperty(""),
Known: true,
}),
expected: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expected, tt.prop.ContainsUnknowns())
})
}
}
func TestContainsSecrets(t *testing.T) {
tests := []struct {
name string
prop PropertyValue
expected bool
}{
{
name: "secret",
prop: MakeSecret(NewStringProperty("")),
expected: true,
},
{
name: "output unknown",
prop: MakeOutput(NewStringProperty("")),
expected: false,
},
{
name: "output unknown containing secret",
prop: MakeOutput(MakeSecret(NewStringProperty(""))),
expected: true,
},
{
name: "output unknown secret",
prop: NewOutputProperty(Output{
Element: NewStringProperty(""),
Secret: true,
}),
expected: true,
},
{
name: "output known secret",
prop: NewOutputProperty(Output{
Element: NewStringProperty(""),
Known: true,
Secret: true,
}),
expected: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expected, tt.prop.ContainsSecrets())
})
}
}
func TestHasValue(t *testing.T) {
tests := []struct {
name string
prop PropertyValue
expected bool
}{
{
name: "null",
prop: NewNullProperty(),
expected: false,
},
{
name: "string",
prop: NewStringProperty(""),
expected: true,
},
{
name: "output unknown",
prop: MakeOutput(NewStringProperty("")),
expected: false,
},
{
name: "output known",
prop: NewOutputProperty(Output{
Element: NewStringProperty(""),
Known: true,
}),
expected: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expected, tt.prop.HasValue())
})
}
}

1
tests/go.sum
View file

@ -1128,6 +1128,7 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=