maxmustermann/pulumi
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add support for refreshing specific targets. (#3225)

Browse source
This commit is contained in:
CyrusNajmabadi 2019-09-17 18:14:10 -07:00 committed by GitHub
parent 5e3cc50f4b
commit f788eb8fc1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 231 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
CHANGELOG.md
View file

@ -7,6 +7,8 @@ CHANGELOG
[#3238](https://github.com/pulumi/pulumi/pull/3238)
- Fix parsing of GitLab urls with subgroups.
[#3239](https://github.com/pulumi/pulumi/pull/3239)
- `pulumi refresh` can now be scoped to refresh a subset of resources by adding a `--target urn` or
`-t urn` argument. Multiple resources can be specified using `-t urn1 -t urn2`.
## 1.1.0 (2019-09-11)

18
cmd/refresh.go
View file

@ -23,6 +23,7 @@ import (
"github.com/pulumi/pulumi/pkg/backend"
"github.com/pulumi/pulumi/pkg/backend/display"
"github.com/pulumi/pulumi/pkg/engine"
"github.com/pulumi/pulumi/pkg/resource"
"github.com/pulumi/pulumi/pkg/util/cmdutil"
"github.com/pulumi/pulumi/pkg/util/result"
)
@ -43,6 +44,7 @@ func newRefreshCmd() *cobra.Command {
var skipPreview bool
var suppressOutputs bool
var yes bool
var targets *[]string
var cmd = &cobra.Command{
Use: "refresh",
@ -110,10 +112,16 @@ func newRefreshCmd() *cobra.Command {
return result.FromError(errors.Wrap(err, "getting stack configuration"))
}
targetUrns := []resource.URN{}
for _, t := range *targets {
targetUrns = append(targetUrns, resource.URN(t))
}
opts.Engine = engine.UpdateOptions{
Parallel: parallel,
Debug: debug,
UseLegacyDiff: useLegacyDiff(),
Parallel: parallel,
Debug: debug,
UseLegacyDiff: useLegacyDiff(),
RefreshTargets: targetUrns,
}
changes, res := s.Refresh(commandContext(), backend.UpdateOperation{
@ -156,6 +164,10 @@ func newRefreshCmd() *cobra.Command {
&message, "message", "m", "",
"Optional message to associate with the update operation")
targets = cmd.PersistentFlags().StringArrayP(
"target", "t", []string{},
"Specify a single resource URN to refresh. Multiple resource can be specified using: --target urn1 --target urn2")
// Flags for engine.UpdateOptions.
cmd.PersistentFlags().BoolVar(
&diffDisplay, "diff", false,

1
go.mod
View file

@ -28,6 +28,7 @@ require (
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
github.com/mitchellh/copystructure v1.0.0
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
github.com/mxschmitt/golang-combinations v1.0.0
github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663
github.com/onsi/ginkgo v1.7.0 // indirect
github.com/onsi/gomega v1.4.3 // indirect

2
go.sum
View file

@ -312,6 +312,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY=
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mxschmitt/golang-combinations v1.0.0 h1:NFoO7CSP8MUcFlHpe1YdewKwMa15dgDbaqkVLC5DUPI=
github.com/mxschmitt/golang-combinations v1.0.0/go.mod h1:RbMhWvfCelHR6WROvT2bVfxJvZHoEvBj71SKe+H0MYU=
github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663 h1:Ri1EhipkbhWsffPJ3IPlrb4SkTOPa2PfRXp3jchBczw=
github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=

160
pkg/engine/lifecycle_test.go
View file

@ -45,6 +45,8 @@ import (
"github.com/pulumi/pulumi/pkg/util/result"
"github.com/pulumi/pulumi/pkg/util/rpcutil/rpcerror"
"github.com/pulumi/pulumi/pkg/workspace"
combinations "github.com/mxschmitt/golang-combinations"
)
type JournalEntryKind int
@ -1554,15 +1556,54 @@ func TestRefreshWithDelete(t *testing.T) {
}
}
func pickURN(urns []resource.URN, target string) resource.URN {
switch target {
case "resA":
return urns[0]
case "resB":
return urns[1]
case "resC":
return urns[2]
default:
panic("Invalid target: " + target)
}
}
// Tests that dependencies are correctly rewritten when refresh removes deleted resources.
func TestRefreshDeleteDependencies(t *testing.T) {
names := []string{"resA", "resB", "resC"}
// Try refreshing a stack with every combination of the three above resources as a target to
// refresh.
subsets := combinations.All(names)
// combinations.All doesn't return the empty set. So explicitly test that case (i.e. test no
// targets specified)
validateRefreshDeleteCombination(t, names, []string{})
for _, subset := range subsets {
validateRefreshDeleteCombination(t, names, subset)
}
}
func validateRefreshDeleteCombination(t *testing.T, names []string, targets []string) {
p := &TestPlan{}
const resType = "pkgA:m:typA"
urnA := p.NewURN(resType, "resA", "")
urnB := p.NewURN(resType, "resB", "")
urnC := p.NewURN(resType, "resC", "")
urnA := p.NewURN(resType, names[0], "")
urnB := p.NewURN(resType, names[1], "")
urnC := p.NewURN(resType, names[2], "")
urns := []resource.URN{urnA, urnB, urnC}
refreshTargets := []resource.URN{}
t.Logf("Refreshing targets: %v", targets)
for _, target := range targets {
refreshTargets = append(p.Options.RefreshTargets, pickURN(urns, target))
}
p.Options.RefreshTargets = refreshTargets
newResource := func(urn resource.URN, id resource.ID, delete bool, dependencies ...resource.URN) *resource.State {
return &resource.State{
@ -1610,7 +1651,28 @@ func TestRefreshDeleteDependencies(t *testing.T) {
p.Options.host = deploytest.NewPluginHost(nil, nil, nil, loaders...)
p.Steps = []TestStep{{Op: Refresh}}
p.Steps = []TestStep{
{
Op: Refresh,
Validate: func(project workspace.Project, target deploy.Target, j *Journal,
_ []Event, res result.Result) result.Result {
// Should see only refreshes.
for _, entry := range j.Entries {
if len(refreshTargets) > 0 {
// should only see changes to urns we explicitly asked to change
assert.Containsf(t, refreshTargets, entry.Step.URN(),
"Refreshed a resource that wasn't a target: %v", entry.Step.URN())
}
assert.Equal(t, deploy.OpRefresh, entry.Step.Op())
}
return res
},
},
}
snap := p.Run(t, old)
provURN := p.NewProviderURN("pkgA", "default", "")
@ -1625,36 +1687,80 @@ func TestRefreshDeleteDependencies(t *testing.T) {
t.Fatalf("unexpected resource %v", urn)
}
switch r.ID {
case "1":
// A::0 was deleted, so B's dependency list should be empty.
assert.Equal(t, urnB, r.URN)
assert.Empty(t, r.Dependencies)
case "2":
// A::0 was deleted, so C's dependency list should only contain B.
assert.Equal(t, urnC, r.URN)
assert.Equal(t, []resource.URN{urnB}, r.Dependencies)
case "3":
// A::3 should not have changed.
assert.Equal(t, oldResources[3], r)
case "5":
// A::4 was deleted but A::3 was still refernceable by C, so C should not have changed.
assert.Equal(t, oldResources[5], r)
default:
t.Fatalf("unexepcted resource %v::%v", r.URN, r.ID)
if len(refreshTargets) == 0 || containsURN(refreshTargets, urnA) {
// 'A' was deleted, so we should see the impact downstream.
switch r.ID {
case "1":
// A::0 was deleted, so B's dependency list should be empty.
assert.Equal(t, urnB, r.URN)
assert.Empty(t, r.Dependencies)
case "2":
// A::0 was deleted, so C's dependency list should only contain B.
assert.Equal(t, urnC, r.URN)
assert.Equal(t, []resource.URN{urnB}, r.Dependencies)
case "3":
// A::3 should not have changed.
assert.Equal(t, oldResources[3], r)
case "5":
// A::4 was deleted but A::3 was still refernceable by C, so C should not have changed.
assert.Equal(t, oldResources[5], r)
default:
t.Fatalf("Unexpected changed resource when refreshing %v: %v::%v", refreshTargets, r.URN, r.ID)
}
} else {
// A was not deleted. So nothing should be impacted.
id, err := strconv.Atoi(r.ID.String())
assert.NoError(t, err)
assert.Equal(t, oldResources[id], r)
}
}
}
func containsURN(urns []resource.URN, urn resource.URN) bool {
for _, val := range urns {
if val == urn {
return true
}
}
return false
}
// Tests basic refresh functionality.
func TestRefreshBasics(t *testing.T) {
names := []string{"resA", "resB", "resC"}
// Try refreshing a stack with every combination of the three above resources as a target to
// refresh.
subsets := combinations.All(names)
// combinations.All doesn't return the empty set. So explicitly test that case (i.e. test no
// targets specified)
validateRefreshBasicsCombination(t, names, []string{})
for _, subset := range subsets {
validateRefreshBasicsCombination(t, names, subset)
}
}
func validateRefreshBasicsCombination(t *testing.T, names []string, targets []string) {
p := &TestPlan{}
const resType = "pkgA:m:typA"
urnA := p.NewURN(resType, "resA", "")
urnB := p.NewURN(resType, "resB", "")
urnC := p.NewURN(resType, "resC", "")
urnA := p.NewURN(resType, names[0], "")
urnB := p.NewURN(resType, names[1], "")
urnC := p.NewURN(resType, names[2], "")
urns := []resource.URN{urnA, urnB, urnC}
refreshTargets := []resource.URN{}
for _, target := range targets {
refreshTargets = append(p.Options.RefreshTargets, pickURN(urns, target))
}
p.Options.RefreshTargets = refreshTargets
newResource := func(urn resource.URN, id resource.ID, delete bool, dependencies ...resource.URN) *resource.State {
return &resource.State{
@ -1722,6 +1828,12 @@ func TestRefreshBasics(t *testing.T) {
// Should see only refreshes.
for _, entry := range j.Entries {
if len(refreshTargets) > 0 {
// should only see changes to urns we explicitly asked to change
assert.Containsf(t, refreshTargets, entry.Step.URN(),
"Refreshed a resource that wasn't a target: %v", entry.Step.URN())
}
assert.Equal(t, deploy.OpRefresh, entry.Step.Op())
resultOp := entry.Step.(*deploy.RefreshStep).ResultOp()

1
pkg/engine/plan.go
View file

@ -181,6 +181,7 @@ func (planResult *planResult) Walk(cancelCtx *Context, events deploy.Events, pre
Parallel: planResult.Options.Parallel,
Refresh: planResult.Options.Refresh,
RefreshOnly: planResult.Options.isRefresh,
RefreshTargets: planResult.Options.RefreshTargets,
TrustDependencies: planResult.Options.trustDependencies,
UseLegacyDiff: planResult.Options.UseLegacyDiff,
}

3
pkg/engine/update.go
View file

@ -62,6 +62,9 @@ type UpdateOptions struct {
// true if the plan should refresh before executing.
Refresh bool
// Specific resources to refresh during a refresh operation.
RefreshTargets []resource.URN
// true if the engine should use legacy diffing behavior during an update.
UseLegacyDiff bool

13
pkg/resource/deploy/plan.go
View file

@ -47,12 +47,13 @@ type BackendClient interface {
// Options controls the planning and deployment process.
type Options struct {
Events Events // an optional events callback interface.
Parallel int // the degree of parallelism for resource operations (<=1 for serial).
Refresh bool // whether or not to refresh before executing the plan.
RefreshOnly bool // whether or not to exit after refreshing.
TrustDependencies bool // whether or not to trust the resource dependency graph.
UseLegacyDiff bool // whether or not to use legacy diffing behavior.
Events Events // an optional events callback interface.
Parallel int // the degree of parallelism for resource operations (<=1 for serial).
Refresh bool // whether or not to refresh before executing the plan.
RefreshOnly bool // whether or not to exit after refreshing.
RefreshTargets []resource.URN // The specific resources to refresh during a refresh op.
TrustDependencies bool // whether or not to trust the resource dependency graph.
UseLegacyDiff bool // whether or not to use legacy diffing behavior.
}
// DegreeOfParallelism returns the degree of parallelism that should be used during the

88
pkg/resource/deploy/plan_executor.go
View file

@ -290,10 +290,19 @@ func (pe *planExecutor) refresh(callerCtx context.Context, opts Options, preview
return nil
}
// Create a refresh step for each resource in the old snapshot.
steps := make([]Step, len(prev.Resources))
for i := range prev.Resources {
steps[i] = NewRefreshStep(pe.plan, prev.Resources[i], nil)
// If the user did not provide any --target's, create a refresh step for each resource in the
// old snapshot. If they did provider --target's then only create refresh steps for those
// specific targets.
steps := []Step{}
initialResources := []*resource.State{}
resourceToStep := map[*resource.State]Step{}
for _, res := range prev.Resources {
initialResources = append(initialResources, res)
if shouldRefresh(opts, res) {
step := NewRefreshStep(pe.plan, res, nil)
steps = append(steps, step)
resourceToStep[res] = step
}
}
// Fire up a worker pool and issue each refresh in turn.
@ -303,39 +312,54 @@ func (pe *planExecutor) refresh(callerCtx context.Context, opts Options, preview
stepExec.SignalCompletion()
stepExec.WaitForCompletion()
// Rebuild this plan's map of old resources and dependency graph, stripping out any deleted resources and repairing
// dependency lists as necessary. Note that this updates the base snapshot _in memory_, so it is critical that any
// components that use the snapshot refer to the same instance and avoid reading it concurrently with this rebuild.
// Rebuild this plan's map of old resources and dependency graph, stripping out any deleted
// resources and repairing dependency lists as necessary. Note that this updates the base
// snapshot _in memory_, so it is critical that any components that use the snapshot refer to
// the same instance and avoid reading it concurrently with this rebuild.
//
// The process of repairing dependency lists is a bit subtle. Because multiple physical resources may share a URN,
// the ability of a particular URN to be referenced in a dependency list can change based on the dependent
// resource's position in the resource list. For example, consider the following list of resources, where each
// resource is a (URN, ID, Dependencies) tuple:
// The process of repairing dependency lists is a bit subtle. Because multiple physical
// resources may share a URN, the ability of a particular URN to be referenced in a dependency
// list can change based on the dependent resource's position in the resource list. For example,
// consider the following list of resources, where each resource is a (URN, ID, Dependencies)
// tuple:
//
// [ (A, 0, []), (B, 0, [A]), (A, 1, []), (A, 2, []), (C, 0, [A]) ]
//
// Let `(A, 0, [])` and `(A, 2, [])` be deleted by the refresh. This produces the following intermediate list
// before dependency lists are repaired:
// Let `(A, 0, [])` and `(A, 2, [])` be deleted by the refresh. This produces the following
// intermediate list before dependency lists are repaired:
//
// [ (B, 0, [A]), (A, 1, []), (C, 0, [A]) ]
//
// In order to repair the dependency lists, we iterate over the intermediate resource list, keeping track of which
// URNs refer to at least one physical resource at each point in the list, and remove any dependencies that refer
// to URNs that do not refer to any physical resources. This process produces the following final list:
// In order to repair the dependency lists, we iterate over the intermediate resource list,
// keeping track of which URNs refer to at least one physical resource at each point in the
// list, and remove any dependencies that refer to URNs that do not refer to any physical
// resources. This process produces the following final list:
//
// [ (B, 0, []), (A, 1, []), (C, 0, [A]) ]
//
// Note that the correctness of this process depends on the fact that the list of resources is a topological sort
// of its corresponding dependency graph, so a resource always appears in the list after any resources on which it
// may depend.
resources := make([]*resource.State, 0, len(prev.Resources))
// Note that the correctness of this process depends on the fact that the list of resources is a
// topological sort of its corresponding dependency graph, so a resource always appears in the
// list after any resources on which it may depend.
resources := []*resource.State{}
referenceable := make(map[resource.URN]bool)
olds := make(map[resource.URN]*resource.State)
for _, s := range steps {
new := s.New()
for _, s := range initialResources {
var old, new *resource.State
if step, has := resourceToStep[s]; has {
// We produces a refresh step for this specific resource. Use the new information about
// its dependencies during the update.
old = step.Old()
new = step.New()
} else {
// We didn't do anything with this resource. However, we still may want to update its
// dependencies. So use this resource itself as the 'new' one to update.
old = s
new = s
}
if new == nil {
contract.Assert(s.Old().Custom)
contract.Assert(!providers.IsProviderType(s.Old().Type))
contract.Assert(old.Custom)
contract.Assert(!providers.IsProviderType(old.Type))
continue
}
@ -359,6 +383,7 @@ func (pe *planExecutor) refresh(callerCtx context.Context, opts Options, preview
olds[new.URN] = new
}
}
pe.plan.prev.Resources = resources
pe.plan.olds, pe.plan.depGraph = olds, graph.NewDependencyGraph(resources)
@ -375,3 +400,18 @@ func (pe *planExecutor) refresh(callerCtx context.Context, opts Options, preview
}
return nil
}
func shouldRefresh(opts Options, res *resource.State) bool {
if len(opts.RefreshTargets) == 0 {
return true
}
//var found = false
for _, urn := range opts.RefreshTargets {
if urn == res.URN {
return true
}
}
return false
}