[sdk/go] Transitive component dependencies. (#7737)

Implement Node/.NET-style dependency semantics for component resources. Depending on a component implicitly depends on all of the component's children. The exact set of children depends on exactly when the component resource is observed. Part of #7542.
2021-08-11 21:51:23 -05:00 · 2021-08-11 21:51:23 -05:00 · 2d70324b56
parent 7361e719dc
commit 2d70324b56
6 changed files with 266 additions and 68 deletions
--- a/CHANGELOG_PENDING.md
+++ b/CHANGELOG_PENDING.md
@ -1,4 +1,8 @@
 ### Improvements

+- [sdk/go] - Depending on a component now depends on the transitive closure of its
+  child resources.
+  [#7732](https://github.com/pulumi/pulumi/pull/7732)
+
 ### Bug Fixes

--- a/sdk/go/pulumi/context.go
+++ b/sdk/go/pulumi/context.go
@ -587,7 +587,7 @@ func (ctx *Context) ReadResource(
 		}

 		// Prepare the inputs for an impending operation.
-		inputs, err = ctx.prepareResourceInputs(props, t, options, res, false)
+		inputs, err = ctx.prepareResourceInputs(resource, props, t, options, res, false)
 		if err != nil {
 			return
 		}
@ -703,6 +703,9 @@ func (ctx *Context) registerResource(
 	}

 	_, custom := resource.(CustomResource)
+	if !custom && remote {
+		resource.markRemoteComponent()
+	}

 	if _, isProvider := resource.(ProviderResource); isProvider && !strings.HasPrefix(t, "pulumi:providers:") {
 		return errors.New("provider resource type must begin with \"pulumi:providers:\"")
@ -767,7 +770,7 @@ func (ctx *Context) registerResource(
 		}()

 		// Prepare the inputs for an impending operation.
-		inputs, err = ctx.prepareResourceInputs(props, t, options, resState, remote)
+		inputs, err = ctx.prepareResourceInputs(resource, props, t, options, resState, remote)
 		if err != nil {
 			return
 		}
@ -1150,12 +1153,12 @@ type resourceInputs struct {
 }

 // prepareResourceInputs prepares the inputs for a resource operation, shared between read and register.
-func (ctx *Context) prepareResourceInputs(props Input, t string, opts *resourceOptions, resource *resourceState,
-	remote bool) (*resourceInputs, error) {
+func (ctx *Context) prepareResourceInputs(res Resource, props Input, t string, opts *resourceOptions,
+	state *resourceState, remote bool) (*resourceInputs, error) {

 	// Get the parent and dependency URNs from the options, in addition to the protection bit.  If there wasn't an
 	// explicit parent, and a root stack resource exists, we will automatically parent to that.
-	resOpts, err := ctx.getOpts(t, resource.provider, opts, remote)
+	resOpts, err := ctx.getOpts(res, t, state.provider, opts, remote)
 	if err != nil {
 		return nil, fmt.Errorf("resolving options: %w", err)
 	}
@ -1180,15 +1183,11 @@ func (ctx *Context) prepareResourceInputs(props Input, t string, opts *resourceO
 	// Convert the property dependencies map for RPC and remove duplicates.
 	rpcPropertyDeps := make(map[string]*pulumirpc.RegisterResourceRequest_PropertyDependencies)
 	for k, deps := range propertyDeps {
-		sort.Slice(deps, func(i, j int) bool { return deps[i] < deps[j] })
-
-		urns := make([]string, 0, len(deps))
+		urns := make([]string, len(deps))
 		for i, d := range deps {
-			if i > 0 && urns[i-1] == string(d) {
-				continue
-			}
-			urns = append(urns, string(d))
+			urns[i] = string(d)
 		}
+		sort.Strings(urns)

 		rpcPropertyDeps[k] = &pulumirpc.RegisterResourceRequest_PropertyDependencies{
 			Urns: urns,
@ -1197,18 +1196,18 @@ func (ctx *Context) prepareResourceInputs(props Input, t string, opts *resourceO

 	// Merge all dependencies with what we got earlier from property marshaling, and remove duplicates.
 	var deps []string
-	depMap := make(map[URN]bool)
+	depSet := urnSet{}
 	for _, dep := range append(resOpts.depURNs, rpcDeps...) {
-		if _, has := depMap[dep]; !has {
+		if !depSet.has(dep) {
 			deps = append(deps, string(dep))
-			depMap[dep] = true
+			depSet.add(dep)
 		}
 	}
 	sort.Strings(deps)

 	// Await alias URNs
-	aliases := make([]string, len(resource.aliases))
-	for i, alias := range resource.aliases {
+	aliases := make([]string, len(state.aliases))
+	for i, alias := range state.aliases {
 		urn, _, _, err := alias.awaitURN(context.Background())
 		if err != nil {
 			return nil, fmt.Errorf("error waiting for alias URN to resolve: %w", err)
@ -1231,7 +1230,7 @@ func (ctx *Context) prepareResourceInputs(props Input, t string, opts *resourceO
 		ignoreChanges:           resOpts.ignoreChanges,
 		aliases:                 aliases,
 		additionalSecretOutputs: resOpts.additionalSecretOutputs,
-		version:                 resource.version,
+		version:                 state.version,
 	}, nil
 }

@ -1260,7 +1259,7 @@ type resourceOpts struct {

 // getOpts returns a set of resource options from an array of them. This includes the parent URN, any dependency URNs,
 // a boolean indicating whether the resource is to be protected, and the URN and ID of the resource's provider, if any.
-func (ctx *Context) getOpts(t string, provider ProviderResource, opts *resourceOptions, remote bool,
+func (ctx *Context) getOpts(res Resource, t string, provider ProviderResource, opts *resourceOptions, remote bool,
 ) (resourceOpts, error) {

 	var importID ID
@ -1274,6 +1273,8 @@ func (ctx *Context) getOpts(t string, provider ProviderResource, opts *resourceO

 	var parentURN URN
 	if opts.Parent != nil {
+		opts.Parent.addChild(res)
+
 		urn, _, _, err := opts.Parent.URN().awaitURN(context.TODO())
 		if err != nil {
 			return resourceOpts{}, err
@ -1283,14 +1284,13 @@ func (ctx *Context) getOpts(t string, provider ProviderResource, opts *resourceO

 	var depURNs []URN
 	if opts.DependsOn != nil {
-		depURNs = make([]URN, len(opts.DependsOn))
-		for i, r := range opts.DependsOn {
-			urn, _, _, err := r.URN().awaitURN(context.TODO())
-			if err != nil {
+		depSet := urnSet{}
+		for _, r := range opts.DependsOn {
+			if err := addDependency(context.TODO(), depSet, r); err != nil {
 				return resourceOpts{}, err
 			}
-			depURNs[i] = urn
 		}
+		depURNs = depSet.values()
 	}

 	var providerRef string
--- a/sdk/go/pulumi/resource.go
+++ b/sdk/go/pulumi/resource.go
@ -16,6 +16,7 @@ package pulumi

 import (
 	"reflect"
+	"sync"

 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
 )
@ -33,50 +34,74 @@ var providerResourceStateType = reflect.TypeOf(ProviderResourceState{})

 // ResourceState is the base
 type ResourceState struct {
+	m sync.RWMutex
+
 	urn URNOutput `pulumi:"urn"`

-	providers map[string]ProviderResource
-
-	provider ProviderResource
-
-	version string
-
-	aliases []URNOutput
-
-	name string
-
+	children        resourceSet
+	providers       map[string]ProviderResource
+	provider        ProviderResource
+	version         string
+	aliases         []URNOutput
+	name            string
 	transformations []ResourceTransformation
+
+	remoteComponent bool
 }

-func (s ResourceState) URN() URNOutput {
+func (s *ResourceState) URN() URNOutput {
 	return s.urn
 }

-func (s ResourceState) GetProvider(token string) ProviderResource {
+func (s *ResourceState) GetProvider(token string) ProviderResource {
 	return s.providers[getPackage(token)]
 }

-func (s ResourceState) getProviders() map[string]ProviderResource {
+func (s *ResourceState) getChildren() []Resource {
+	s.m.RLock()
+	defer s.m.RUnlock()
+
+	var children []Resource
+	if len(s.children) != 0 {
+		children = make([]Resource, 0, len(s.children))
+		for r := range s.children {
+			children = append(children, r)
+		}
+	}
+	return children
+}
+
+func (s *ResourceState) addChild(r Resource) {
+	s.m.Lock()
+	defer s.m.Unlock()
+
+	if s.children == nil {
+		s.children = resourceSet{}
+	}
+	s.children.add(r)
+}
+
+func (s *ResourceState) getProviders() map[string]ProviderResource {
 	return s.providers
 }

-func (s ResourceState) getProvider() ProviderResource {
+func (s *ResourceState) getProvider() ProviderResource {
 	return s.provider
 }

-func (s ResourceState) getVersion() string {
+func (s *ResourceState) getVersion() string {
 	return s.version
 }

-func (s ResourceState) getAliases() []URNOutput {
+func (s *ResourceState) getAliases() []URNOutput {
 	return s.aliases
 }

-func (s ResourceState) getName() string {
+func (s *ResourceState) getName() string {
 	return s.name
 }

-func (s ResourceState) getTransformations() []ResourceTransformation {
+func (s *ResourceState) getTransformations() []ResourceTransformation {
 	return s.transformations
 }

@ -84,12 +109,23 @@ func (s *ResourceState) addTransformation(t ResourceTransformation) {
 	s.transformations = append(s.transformations, t)
 }

-func (ResourceState) isResource() {}
+func (s *ResourceState) markRemoteComponent() {
+	s.remoteComponent = true
+}
+
+func (s *ResourceState) isRemoteComponent() bool {
+	return s.remoteComponent
+}
+
+func (*ResourceState) isResource() {}

 func (ctx *Context) newDependencyResource(urn URN) Resource {
 	var res ResourceState
 	res.urn.OutputState = ctx.newOutputState(res.urn.ElementType(), &res)
 	res.urn.resolve(urn, true, false, nil)
+
+	// For the purposes of dependency management, dependency resources are treated like remote components.
+	res.remoteComponent = true
 	return &res
 }

@ -99,11 +135,11 @@ type CustomResourceState struct {
 	id IDOutput `pulumi:"id"`
 }

-func (s CustomResourceState) ID() IDOutput {
+func (s *CustomResourceState) ID() IDOutput {
 	return s.id
 }

-func (CustomResourceState) isCustomResource() {}
+func (*CustomResourceState) isCustomResource() {}

 func (ctx *Context) newDependencyCustomResource(urn URN, id ID) CustomResource {
 	var res CustomResourceState
@ -120,7 +156,7 @@ type ProviderResourceState struct {
 	pkg string
 }

-func (s ProviderResourceState) getPackage() string {
+func (s *ProviderResourceState) getPackage() string {
 	return s.pkg
 }

@ -139,6 +175,12 @@ type Resource interface {
 	// URN is this resource's stable logical URN used to distinctly address it before, during, and after deployments.
 	URN() URNOutput

+	// getChildren returns the resource's children.
+	getChildren() []Resource
+
+	// addChild adds a child to the resource.
+	addChild(r Resource)
+
 	// getProviders returns the provider map for this resource.
 	getProviders() map[string]ProviderResource

@ -162,6 +204,12 @@ type Resource interface {

 	// addTransformation adds a single transformation to the resource.
 	addTransformation(t ResourceTransformation)
+
+	// markRemoteComponent marks this resource as a remote component resource.
+	markRemoteComponent()
+
+	// isRemoteComponent returns true if this is not a local (i.e. in-process) component resource.
+	isRemoteComponent() bool
 }

 // CustomResource is a cloud resource whose create, read, update, and delete (CRUD) operations are managed by performing
--- a/sdk/go/pulumi/resource_set.go
+++ b/sdk/go/pulumi/resource_set.go
@ -0,0 +1,20 @@
+package pulumi
+
+type resourceSet map[Resource]struct{}
+
+func (s resourceSet) add(r Resource) {
+	s[r] = struct{}{}
+}
+
+func (s resourceSet) any() bool {
+	return len(s) > 0
+}
+
+func (s resourceSet) delete(r Resource) {
+	delete(s, r)
+}
+
+func (s resourceSet) has(r Resource) bool {
+	_, ok := s[r]
+	return ok
+}
--- a/sdk/go/pulumi/rpc.go
+++ b/sdk/go/pulumi/rpc.go
@ -63,14 +63,96 @@ func mapStructTypes(from, to reflect.Type) func(reflect.Value, int) (reflect.Str
 	}
 }

+type urnSet map[URN]struct{}
+
+func (s urnSet) add(v URN) {
+	s[v] = struct{}{}
+}
+
+func (s urnSet) has(v URN) bool {
+	_, ok := s[v]
+	return ok
+}
+
+func (s urnSet) union(other urnSet) {
+	for v := range other {
+		s.add(v)
+	}
+}
+
+func (s urnSet) values() []URN {
+	values := make([]URN, 0, len(s))
+	for v := range s {
+		values = append(values, v)
+	}
+	return values
+}
+
+// addDependency adds a dependency on the given resource to the set of deps.
+//
+// The behavior of this method depends on whether or not the resource is a custom resource, a local component resource,
+// or a remote component resource:
+//
+// - Custom resources are added directly to the set, as they are "real" nodes in the dependency graph.
+// - Local component resources act as aggregations of their descendents. Rather than adding the component resource
+//   itself, each child resource is added as a dependency.
+// - Remote component resources are added directly to the set, as they naturally act as aggregations of their children
+//   with respect to dependencies: the construction of a remote component always waits on the construction of its
+//   children.
+//
+// In other words, if we had:
+//
+//				  Comp1
+//			  /     |     \
+//		  Cust1   Comp2  Remote1
+//				  /   \       \
+//			  Cust2   Cust3  Comp3
+//			  /                 \
+//		  Cust4                Cust5
+//
+// Then the transitively reachable resources of Comp1 will be [Cust1, Cust2, Cust3, Remote1].
+// It will *not* include:
+// * Cust4 because it is a child of a custom resource
+// * Comp2 because it is a non-remote component resoruce
+// * Comp3 and Cust5 because Comp3 is a child of a remote component resource
+func addDependency(ctx context.Context, deps urnSet, res Resource) error {
+	if _, custom := res.(CustomResource); !custom {
+		for _, child := range res.getChildren() {
+			if err := addDependency(ctx, deps, child); err != nil {
+				return err
+			}
+		}
+		if !res.isRemoteComponent() {
+			return nil
+		}
+	}
+
+	urn, _, _, err := res.URN().awaitURN(ctx)
+	if err != nil {
+		return err
+	}
+	deps.add(urn)
+	return nil
+}
+
+// expandDependencies expands the given slice of Resources into a set of URNs.
+func expandDependencies(ctx context.Context, deps []Resource) (urnSet, error) {
+	urns := urnSet{}
+	for _, r := range deps {
+		if err := addDependency(ctx, urns, r); err != nil {
+			return nil, err
+		}
+	}
+	return urns, nil
+}
+
 // marshalInputs turns resource property inputs into a map suitable for marshaling.
 func marshalInputs(props Input) (resource.PropertyMap, map[string][]URN, []URN, error) {
-	var depURNs []URN
-	depset := map[URN]bool{}
+	deps := urnSet{}
 	pmap, pdeps := resource.PropertyMap{}, map[string][]URN{}

 	if props == nil {
-		return pmap, pdeps, depURNs, nil
+		return pmap, pdeps, nil, nil
 	}

 	marshalProperty := func(pname string, pv interface{}, pt reflect.Type) error {
@ -81,24 +163,14 @@ func marshalInputs(props Input) (resource.PropertyMap, map[string][]URN, []URN,
 		}

 		// Record all dependencies accumulated from reading this property.
-		var deps []URN
-		pdepset := map[URN]bool{}
-		for _, dep := range resourceDeps {
-			depURN, _, _, err := dep.URN().awaitURN(context.TODO())
-			if err != nil {
-				return err
-			}
-			if !pdepset[depURN] {
-				deps = append(deps, depURN)
-				pdepset[depURN] = true
-			}
-			if !depset[depURN] {
-				depURNs = append(depURNs, depURN)
-				depset[depURN] = true
-			}
+		allDeps, err := expandDependencies(context.TODO(), resourceDeps)
+		if err != nil {
+			return err
 		}
-		if len(deps) > 0 {
-			pdeps[pname] = deps
+		deps.union(allDeps)
+
+		if len(allDeps) > 0 {
+			pdeps[pname] = allDeps.values()
 		}

 		if !v.IsNull() || len(deps) > 0 {
@ -110,7 +182,7 @@ func marshalInputs(props Input) (resource.PropertyMap, map[string][]URN, []URN,
 	pv := reflect.ValueOf(props)
 	if pv.Kind() == reflect.Ptr {
 		if pv.IsNil() {
-			return pmap, pdeps, depURNs, nil
+			return pmap, pdeps, nil, nil
 		}
 		pv = pv.Elem()
 	}
@ -157,7 +229,7 @@ func marshalInputs(props Input) (resource.PropertyMap, map[string][]URN, []URN,
 		return nil, nil, nil, fmt.Errorf("cannot marshal Input that is not a struct or map, saw type %s", pt.String())
 	}

-	return pmap, pdeps, depURNs, nil
+	return pmap, pdeps, deps.values(), nil
 }

 // `gosec` thinks these are credentials, but they are not.
--- a/sdk/go/pulumi/rpc_test.go
+++ b/sdk/go/pulumi/rpc_test.go
@ -22,6 +22,7 @@ import (
 	"testing"

 	"github.com/blang/semver"
+	structpb "github.com/golang/protobuf/ptypes/struct"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
 	"github.com/stretchr/testify/assert"
@ -829,3 +830,56 @@ func TestInvalidArchive(t *testing.T) {
 	_, _, err = marshalInput(d, archiveType, true)
 	assert.Error(t, err)
 }
+
+func TestDependsOnComponent(t *testing.T) {
+	ctx, err := NewContext(context.Background(), RunInfo{})
+	assert.Nil(t, err)
+
+	registerResource := func(name string, res Resource, opts *resourceOptions) (Resource, []string) {
+		state := ctx.makeResourceState("", "", res, nil, nil, "", nil, nil)
+		state.resolve(ctx, nil, nil, name, "", &structpb.Struct{}, nil)
+
+		if opts == nil {
+			opts = &resourceOptions{}
+		}
+
+		inputs, err := ctx.prepareResourceInputs(res, Map{}, "", opts, state, false)
+		require.NoError(t, err)
+
+		return res, inputs.deps
+	}
+
+	newResource := func(name string, opts *resourceOptions) (Resource, []string) {
+		var res testResource
+		return registerResource(name, &res, opts)
+	}
+
+	newComponent := func(name string, opts *resourceOptions) (Resource, []string) {
+		var res simpleComponentResource
+		return registerResource(name, &res, opts)
+	}
+
+	resA, _ := newResource("resA", nil)
+	comp1, _ := newComponent("comp1", nil)
+	resB, _ := newResource("resB", &resourceOptions{Parent: comp1})
+	newResource("resC", &resourceOptions{Parent: resB})
+	comp2, _ := newComponent("comp2", &resourceOptions{Parent: comp1})
+
+	resD, deps := newResource("resD", &resourceOptions{DependsOn: []Resource{resA}, Parent: comp2})
+	assert.Equal(t, []string{"resA"}, deps)
+
+	_, deps = newResource("resE", &resourceOptions{DependsOn: []Resource{resD}, Parent: comp2})
+	assert.Equal(t, []string{"resD"}, deps)
+
+	_, deps = newResource("resF", &resourceOptions{DependsOn: []Resource{resA}})
+	assert.Equal(t, []string{"resA"}, deps)
+
+	resG, deps := newResource("resG", &resourceOptions{DependsOn: []Resource{comp1}})
+	assert.Equal(t, []string{"resB", "resD", "resE"}, deps)
+
+	_, deps = newResource("resH", &resourceOptions{DependsOn: []Resource{comp2}})
+	assert.Equal(t, []string{"resD", "resE"}, deps)
+
+	_, deps = newResource("resI", &resourceOptions{DependsOn: []Resource{resG}})
+	assert.Equal(t, []string{"resG"}, deps)
+}