Add support for refreshing specific targets. (#3225)

CHANGELOG.md

@@ -7,6 +7,8 @@ CHANGELOG
   [#3238](https://github.com/pulumi/pulumi/pull/3238)
 - Fix parsing of GitLab urls with subgroups.
   [#3239](https://github.com/pulumi/pulumi/pull/3239)
+- `pulumi refresh` can now be scoped to refresh a subset of resources by adding a `--target urn` or
+  `-t urn` argument.  Multiple resources can be specified using `-t urn1 -t urn2`.
 
 ## 1.1.0 (2019-09-11)
 

cmd/refresh.go

@@ -23,6 +23,7 @@ import (
 	"github.com/pulumi/pulumi/pkg/backend"
 	"github.com/pulumi/pulumi/pkg/backend/display"
 	"github.com/pulumi/pulumi/pkg/engine"
+	"github.com/pulumi/pulumi/pkg/resource"
 	"github.com/pulumi/pulumi/pkg/util/cmdutil"
 	"github.com/pulumi/pulumi/pkg/util/result"
 )
@@ -43,6 +44,7 @@ func newRefreshCmd() *cobra.Command {
 	var skipPreview bool
 	var suppressOutputs bool
 	var yes bool
+	var targets *[]string
 
 	var cmd = &cobra.Command{
 		Use:   "refresh",
@@ -110,10 +112,16 @@ func newRefreshCmd() *cobra.Command {
 				return result.FromError(errors.Wrap(err, "getting stack configuration"))
 			}
 
+			targetUrns := []resource.URN{}
+			for _, t := range *targets {
+				targetUrns = append(targetUrns, resource.URN(t))
+			}
+
 			opts.Engine = engine.UpdateOptions{
-				Parallel:      parallel,
+				Parallel:       parallel,
-				Debug:         debug,
+				Debug:          debug,
-				UseLegacyDiff: useLegacyDiff(),
+				UseLegacyDiff:  useLegacyDiff(),
+				RefreshTargets: targetUrns,
 			}
 
 			changes, res := s.Refresh(commandContext(), backend.UpdateOperation{
@@ -156,6 +164,10 @@ func newRefreshCmd() *cobra.Command {
 		&message, "message", "m", "",
 		"Optional message to associate with the update operation")
 
+	targets = cmd.PersistentFlags().StringArrayP(
+		"target", "t", []string{},
+		"Specify a single resource URN to refresh. Multiple resource can be specified using: --target urn1 --target urn2")
+
 	// Flags for engine.UpdateOptions.
 	cmd.PersistentFlags().BoolVar(
 		&diffDisplay, "diff", false,

go.mod

@@ -28,6 +28,7 @@ require (
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
 	github.com/mitchellh/copystructure v1.0.0
 	github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
+	github.com/mxschmitt/golang-combinations v1.0.0
 	github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663
 	github.com/onsi/ginkgo v1.7.0 // indirect
 	github.com/onsi/gomega v1.4.3 // indirect

go.sum

@@ -312,6 +312,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
 github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY=
 github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxschmitt/golang-combinations v1.0.0 h1:NFoO7CSP8MUcFlHpe1YdewKwMa15dgDbaqkVLC5DUPI=
+github.com/mxschmitt/golang-combinations v1.0.0/go.mod h1:RbMhWvfCelHR6WROvT2bVfxJvZHoEvBj71SKe+H0MYU=
 github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663 h1:Ri1EhipkbhWsffPJ3IPlrb4SkTOPa2PfRXp3jchBczw=
 github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=

pkg/engine/lifecycle_test.go

@@ -45,6 +45,8 @@ import (
 	"github.com/pulumi/pulumi/pkg/util/result"
 	"github.com/pulumi/pulumi/pkg/util/rpcutil/rpcerror"
 	"github.com/pulumi/pulumi/pkg/workspace"
+
+	combinations "github.com/mxschmitt/golang-combinations"
 )
 
 type JournalEntryKind int
@@ -1554,15 +1556,54 @@ func TestRefreshWithDelete(t *testing.T) {
 	}
 }
 
+func pickURN(urns []resource.URN, target string) resource.URN {
+	switch target {
+	case "resA":
+		return urns[0]
+	case "resB":
+		return urns[1]
+	case "resC":
+		return urns[2]
+	default:
+		panic("Invalid target: " + target)
+	}
+}
+
 // Tests that dependencies are correctly rewritten when refresh removes deleted resources.
 func TestRefreshDeleteDependencies(t *testing.T) {
+	names := []string{"resA", "resB", "resC"}
+
+	// Try refreshing a stack with every combination of the three above resources as a target to
+	// refresh.
+	subsets := combinations.All(names)
+
+	// combinations.All doesn't return the empty set.  So explicitly test that case (i.e. test no
+	// targets specified)
+	validateRefreshDeleteCombination(t, names, []string{})
+
+	for _, subset := range subsets {
+		validateRefreshDeleteCombination(t, names, subset)
+	}
+}
+
+func validateRefreshDeleteCombination(t *testing.T, names []string, targets []string) {
 	p := &TestPlan{}
 
 	const resType = "pkgA:m:typA"
 
-	urnA := p.NewURN(resType, "resA", "")
+	urnA := p.NewURN(resType, names[0], "")
-	urnB := p.NewURN(resType, "resB", "")
+	urnB := p.NewURN(resType, names[1], "")
-	urnC := p.NewURN(resType, "resC", "")
+	urnC := p.NewURN(resType, names[2], "")
+	urns := []resource.URN{urnA, urnB, urnC}
+
+	refreshTargets := []resource.URN{}
+
+	t.Logf("Refreshing targets: %v", targets)
+	for _, target := range targets {
+		refreshTargets = append(p.Options.RefreshTargets, pickURN(urns, target))
+	}
+
+	p.Options.RefreshTargets = refreshTargets
 
 	newResource := func(urn resource.URN, id resource.ID, delete bool, dependencies ...resource.URN) *resource.State {
 		return &resource.State{
@@ -1610,7 +1651,28 @@ func TestRefreshDeleteDependencies(t *testing.T) {
 
 	p.Options.host = deploytest.NewPluginHost(nil, nil, nil, loaders...)
 
-	p.Steps = []TestStep{{Op: Refresh}}
+	p.Steps = []TestStep{
+		{
+			Op: Refresh,
+			Validate: func(project workspace.Project, target deploy.Target, j *Journal,
+				_ []Event, res result.Result) result.Result {
+
+				// Should see only refreshes.
+				for _, entry := range j.Entries {
+					if len(refreshTargets) > 0 {
+						// should only see changes to urns we explicitly asked to change
+						assert.Containsf(t, refreshTargets, entry.Step.URN(),
+							"Refreshed a resource that wasn't a target: %v", entry.Step.URN())
+					}
+
+					assert.Equal(t, deploy.OpRefresh, entry.Step.Op())
+				}
+
+				return res
+			},
+		},
+	}
+
 	snap := p.Run(t, old)
 
 	provURN := p.NewProviderURN("pkgA", "default", "")
@@ -1625,36 +1687,80 @@ func TestRefreshDeleteDependencies(t *testing.T) {
 			t.Fatalf("unexpected resource %v", urn)
 		}
 
-		switch r.ID {
+		if len(refreshTargets) == 0 || containsURN(refreshTargets, urnA) {
-		case "1":
+			// 'A' was deleted, so we should see the impact downstream.
-			// A::0 was deleted, so B's dependency list should be empty.
+
-			assert.Equal(t, urnB, r.URN)
+			switch r.ID {
-			assert.Empty(t, r.Dependencies)
+			case "1":
-		case "2":
+				// A::0 was deleted, so B's dependency list should be empty.
-			// A::0 was deleted, so C's dependency list should only contain B.
+				assert.Equal(t, urnB, r.URN)
-			assert.Equal(t, urnC, r.URN)
+				assert.Empty(t, r.Dependencies)
-			assert.Equal(t, []resource.URN{urnB}, r.Dependencies)
+			case "2":
-		case "3":
+				// A::0 was deleted, so C's dependency list should only contain B.
-			// A::3 should not have changed.
+				assert.Equal(t, urnC, r.URN)
-			assert.Equal(t, oldResources[3], r)
+				assert.Equal(t, []resource.URN{urnB}, r.Dependencies)
-		case "5":
+			case "3":
-			// A::4 was deleted but A::3 was still refernceable by C, so C should not have changed.
+				// A::3 should not have changed.
-			assert.Equal(t, oldResources[5], r)
+				assert.Equal(t, oldResources[3], r)
-		default:
+			case "5":
-			t.Fatalf("unexepcted resource %v::%v", r.URN, r.ID)
+				// A::4 was deleted but A::3 was still refernceable by C, so C should not have changed.
+				assert.Equal(t, oldResources[5], r)
+			default:
+				t.Fatalf("Unexpected changed resource when refreshing %v: %v::%v", refreshTargets, r.URN, r.ID)
+			}
+		} else {
+			// A was not deleted. So nothing should be impacted.
+			id, err := strconv.Atoi(r.ID.String())
+			assert.NoError(t, err)
+			assert.Equal(t, oldResources[id], r)
 		}
 	}
 }
 
+func containsURN(urns []resource.URN, urn resource.URN) bool {
+	for _, val := range urns {
+		if val == urn {
+			return true
+		}
+	}
+
+	return false
+}
+
 // Tests basic refresh functionality.
 func TestRefreshBasics(t *testing.T) {
+	names := []string{"resA", "resB", "resC"}
+
+	// Try refreshing a stack with every combination of the three above resources as a target to
+	// refresh.
+	subsets := combinations.All(names)
+
+	// combinations.All doesn't return the empty set.  So explicitly test that case (i.e. test no
+	// targets specified)
+	validateRefreshBasicsCombination(t, names, []string{})
+
+	for _, subset := range subsets {
+		validateRefreshBasicsCombination(t, names, subset)
+	}
+}
+
+func validateRefreshBasicsCombination(t *testing.T, names []string, targets []string) {
 	p := &TestPlan{}
 
 	const resType = "pkgA:m:typA"
 
-	urnA := p.NewURN(resType, "resA", "")
+	urnA := p.NewURN(resType, names[0], "")
-	urnB := p.NewURN(resType, "resB", "")
+	urnB := p.NewURN(resType, names[1], "")
-	urnC := p.NewURN(resType, "resC", "")
+	urnC := p.NewURN(resType, names[2], "")
+	urns := []resource.URN{urnA, urnB, urnC}
+
+	refreshTargets := []resource.URN{}
+
+	for _, target := range targets {
+		refreshTargets = append(p.Options.RefreshTargets, pickURN(urns, target))
+	}
+
+	p.Options.RefreshTargets = refreshTargets
 
 	newResource := func(urn resource.URN, id resource.ID, delete bool, dependencies ...resource.URN) *resource.State {
 		return &resource.State{
@@ -1722,6 +1828,12 @@ func TestRefreshBasics(t *testing.T) {
 
 			// Should see only refreshes.
 			for _, entry := range j.Entries {
+				if len(refreshTargets) > 0 {
+					// should only see changes to urns we explicitly asked to change
+					assert.Containsf(t, refreshTargets, entry.Step.URN(),
+						"Refreshed a resource that wasn't a target: %v", entry.Step.URN())
+				}
+
 				assert.Equal(t, deploy.OpRefresh, entry.Step.Op())
 				resultOp := entry.Step.(*deploy.RefreshStep).ResultOp()
 

pkg/engine/plan.go

@@ -181,6 +181,7 @@ func (planResult *planResult) Walk(cancelCtx *Context, events deploy.Events, pre
 			Parallel:          planResult.Options.Parallel,
 			Refresh:           planResult.Options.Refresh,
 			RefreshOnly:       planResult.Options.isRefresh,
+			RefreshTargets:    planResult.Options.RefreshTargets,
 			TrustDependencies: planResult.Options.trustDependencies,
 			UseLegacyDiff:     planResult.Options.UseLegacyDiff,
 		}

pkg/engine/update.go

@@ -62,6 +62,9 @@ type UpdateOptions struct {
 	// true if the plan should refresh before executing.
 	Refresh bool
 
+	// Specific resources to refresh during a refresh operation.
+	RefreshTargets []resource.URN
+
 	// true if the engine should use legacy diffing behavior during an update.
 	UseLegacyDiff bool
 

pkg/resource/deploy/plan.go

@@ -47,12 +47,13 @@ type BackendClient interface {
 
 // Options controls the planning and deployment process.
 type Options struct {
-	Events            Events // an optional events callback interface.
+	Events            Events         // an optional events callback interface.
-	Parallel          int    // the degree of parallelism for resource operations (<=1 for serial).
+	Parallel          int            // the degree of parallelism for resource operations (<=1 for serial).
-	Refresh           bool   // whether or not to refresh before executing the plan.
+	Refresh           bool           // whether or not to refresh before executing the plan.
-	RefreshOnly       bool   // whether or not to exit after refreshing.
+	RefreshOnly       bool           // whether or not to exit after refreshing.
-	TrustDependencies bool   // whether or not to trust the resource dependency graph.
+	RefreshTargets    []resource.URN // The specific resources to refresh during a refresh op.
-	UseLegacyDiff     bool   // whether or not to use legacy diffing behavior.
+	TrustDependencies bool           // whether or not to trust the resource dependency graph.
+	UseLegacyDiff     bool           // whether or not to use legacy diffing behavior.
 }
 
 // DegreeOfParallelism returns the degree of parallelism that should be used during the

pkg/resource/deploy/plan_executor.go

@@ -290,10 +290,19 @@ func (pe *planExecutor) refresh(callerCtx context.Context, opts Options, preview
 		return nil
 	}
 
-	// Create a refresh step for each resource in the old snapshot.
+	// If the user did not provide any --target's, create a refresh step for each resource in the
-	steps := make([]Step, len(prev.Resources))
+	// old snapshot.  If they did provider --target's then only create refresh steps for those
-	for i := range prev.Resources {
+	// specific targets.
-		steps[i] = NewRefreshStep(pe.plan, prev.Resources[i], nil)
+	steps := []Step{}
+	initialResources := []*resource.State{}
+	resourceToStep := map[*resource.State]Step{}
+	for _, res := range prev.Resources {
+		initialResources = append(initialResources, res)
+		if shouldRefresh(opts, res) {
+			step := NewRefreshStep(pe.plan, res, nil)
+			steps = append(steps, step)
+			resourceToStep[res] = step
+		}
 	}
 
 	// Fire up a worker pool and issue each refresh in turn.
@@ -303,39 +312,54 @@ func (pe *planExecutor) refresh(callerCtx context.Context, opts Options, preview
 	stepExec.SignalCompletion()
 	stepExec.WaitForCompletion()
 
-	// Rebuild this plan's map of old resources and dependency graph, stripping out any deleted resources and repairing
+	// Rebuild this plan's map of old resources and dependency graph, stripping out any deleted
-	// dependency lists as necessary. Note that this updates the base snapshot _in memory_, so it is critical that any
+	// resources and repairing dependency lists as necessary. Note that this updates the base
-	// components that use the snapshot refer to the same instance and avoid reading it concurrently with this rebuild.
+	// snapshot _in memory_, so it is critical that any components that use the snapshot refer to
+	// the same instance and avoid reading it concurrently with this rebuild.
 	//
-	// The process of repairing dependency lists is a bit subtle. Because multiple physical resources may share a URN,
+	// The process of repairing dependency lists is a bit subtle. Because multiple physical
-	// the ability of a particular URN to be referenced in a dependency list can change based on the dependent
+	// resources may share a URN, the ability of a particular URN to be referenced in a dependency
-	// resource's position in the resource list. For example, consider the following list of resources, where each
+	// list can change based on the dependent resource's position in the resource list. For example,
-	// resource is a (URN, ID, Dependencies) tuple:
+	// consider the following list of resources, where each resource is a (URN, ID, Dependencies)
+	// tuple:
 	//
 	//     [ (A, 0, []), (B, 0, [A]), (A, 1, []), (A, 2, []), (C, 0, [A]) ]
 	//
-	// Let `(A, 0, [])` and `(A, 2, [])` be deleted by the refresh. This produces the following intermediate list
+	// Let `(A, 0, [])` and `(A, 2, [])` be deleted by the refresh. This produces the following
-	// before dependency lists are repaired:
+	// intermediate list before dependency lists are repaired:
 	//
 	//     [ (B, 0, [A]), (A, 1, []), (C, 0, [A]) ]
 	//
-	// In order to repair the dependency lists, we iterate over the intermediate resource list, keeping track of which
+	// In order to repair the dependency lists, we iterate over the intermediate resource list,
-	// URNs refer to at least one physical resource at each point in the list, and remove any dependencies that refer
+	// keeping track of which URNs refer to at least one physical resource at each point in the
-	// to URNs that do not refer to any physical resources. This process produces the following final list:
+	// list, and remove any dependencies that refer to URNs that do not refer to any physical
+	// resources. This process produces the following final list:
 	//
 	//     [ (B, 0, []), (A, 1, []), (C, 0, [A]) ]
 	//
-	// Note that the correctness of this process depends on the fact that the list of resources is a topological sort
+	// Note that the correctness of this process depends on the fact that the list of resources is a
-	// of its corresponding dependency graph, so a resource always appears in the list after any resources on which it
+	// topological sort of its corresponding dependency graph, so a resource always appears in the
-	// may depend.
+	// list after any resources on which it may depend.
-	resources := make([]*resource.State, 0, len(prev.Resources))
+	resources := []*resource.State{}
 	referenceable := make(map[resource.URN]bool)
 	olds := make(map[resource.URN]*resource.State)
-	for _, s := range steps {
+	for _, s := range initialResources {
-		new := s.New()
+		var old, new *resource.State
+		if step, has := resourceToStep[s]; has {
+			// We produces a refresh step for this specific resource.  Use the new information about
+			// its dependencies during the update.
+			old = step.Old()
+			new = step.New()
+		} else {
+			// We didn't do anything with this resource.  However, we still may want to update its
+			// dependencies.  So use this resource itself as the 'new' one to update.
+			old = s
+			new = s
+		}
+
 		if new == nil {
-			contract.Assert(s.Old().Custom)
+			contract.Assert(old.Custom)
-			contract.Assert(!providers.IsProviderType(s.Old().Type))
+			contract.Assert(!providers.IsProviderType(old.Type))
 			continue
 		}
 
@@ -359,6 +383,7 @@ func (pe *planExecutor) refresh(callerCtx context.Context, opts Options, preview
 			olds[new.URN] = new
 		}
 	}
+
 	pe.plan.prev.Resources = resources
 	pe.plan.olds, pe.plan.depGraph = olds, graph.NewDependencyGraph(resources)
 
@@ -375,3 +400,18 @@ func (pe *planExecutor) refresh(callerCtx context.Context, opts Options, preview
 	}
 	return nil
 }
+
+func shouldRefresh(opts Options, res *resource.State) bool {
+	if len(opts.RefreshTargets) == 0 {
+		return true
+	}
+
+	//var found = false
+	for _, urn := range opts.RefreshTargets {
+		if urn == res.URN {
+			return true
+		}
+	}
+
+	return false
+}