Implement the --exclude-protected feature (#8359)
* Implement the --exclude-protected feature This piggybacks on the same machinery used by the --target flag. By examining the stack, we find a list of all resources managed by Pulumi (in that stack). We then form them into a DAG, and mark all resources as either protected or unprotected. A resource is protected it has the `Protect` flag set or is has a child with the `protect` flag set. It is unprotected otherwise. We then pass the urns of unprotected resources to the update options passed to the destroy operation in the same way that `--target` does. * Update changelog * Handle providers correctly * Add integration test * Protect dependencies of protected resources * Handle --exclude-protected in separate function * Simplify implementation via DependencyGraph * Add TransitiveDependenciesOf * Cleanup unused functions * Gate printed message behind !jsonDisplay * Ensure provider is not `""` * Clean up documentation (and some code)
This commit is contained in:
parent
10ceee406e
commit
554660b23a
|
@ -1,8 +1,11 @@
|
|||
### Improvements
|
||||
* Adds CI detector for Buildkite [#7933](https://github.com/pulumi/pulumi/pull/7933)
|
||||
|
||||
- [CLI] Adding the ability to use `pulumi org set [name]` to set a default org
|
||||
to use when creating a stacks in the Pulumi Service backend or Self -hosted Service
|
||||
- [cli] - Add `--exclude-protected` flag to `pulumi destroy`.
|
||||
[#8359](https://github.com/pulumi/pulumi/pull/8359)
|
||||
|
||||
- [cli] Adding the ability to use `pulumi org set [name]` to set a default org
|
||||
to use when creating a stacks in the Pulumi Service backend or self-hosted Service
|
||||
[#8352](https://github.com/pulumi/pulumi/pull/8352)
|
||||
|
||||
- [schema] Add IsOverlay option to disable codegen for particular types
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// Copyright 2016-2018, Pulumi Corporation.
|
||||
// Copyright 2016-2021, Pulumi Corporation.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
@ -24,8 +24,10 @@ import (
|
|||
"github.com/pulumi/pulumi/pkg/v3/backend"
|
||||
"github.com/pulumi/pulumi/pkg/v3/backend/display"
|
||||
"github.com/pulumi/pulumi/pkg/v3/engine"
|
||||
"github.com/pulumi/pulumi/pkg/v3/resource/graph"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/common/util/result"
|
||||
)
|
||||
|
||||
|
@ -52,6 +54,7 @@ func newDestroyCmd() *cobra.Command {
|
|||
var yes bool
|
||||
var targets *[]string
|
||||
var targetDependents bool
|
||||
var excludeProtected bool
|
||||
|
||||
var cmd = &cobra.Command{
|
||||
Use: "destroy",
|
||||
|
@ -149,6 +152,29 @@ func newDestroyCmd() *cobra.Command {
|
|||
if err != nil {
|
||||
return result.FromError(err)
|
||||
}
|
||||
|
||||
if targets != nil && len(*targets) > 0 && excludeProtected {
|
||||
return result.FromError(errors.New("You cannot specify --target and --exclude-protected"))
|
||||
}
|
||||
|
||||
var protectedCount int
|
||||
if excludeProtected {
|
||||
contract.Assert(len(targetUrns) == 0)
|
||||
targetUrns, protectedCount, err = handleExcludeProtected(s)
|
||||
if err != nil {
|
||||
return result.FromError(err)
|
||||
} else if protectedCount > 0 && len(targetUrns) == 0 {
|
||||
if !jsonDisplay {
|
||||
fmt.Printf("There were no unprotected resources to destroy. There are still %d"+
|
||||
" protected resources associated with this stack.\n", protectedCount)
|
||||
}
|
||||
// We need to return now. Otherwise the update will conclude
|
||||
// we tried to destroy everything and error for trying to
|
||||
// destroy a protected resource.
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
opts.Engine = engine.UpdateOptions{
|
||||
Parallel: parallel,
|
||||
Debug: debug,
|
||||
|
@ -170,8 +196,10 @@ func newDestroyCmd() *cobra.Command {
|
|||
SecretsManager: sm,
|
||||
Scopes: cancellationScopes,
|
||||
})
|
||||
|
||||
if res == nil && len(*targets) == 0 && !jsonDisplay {
|
||||
if res == nil && protectedCount > 0 && !jsonDisplay {
|
||||
fmt.Printf("All unprotected resources were destroyed. There are still %d protected resources"+
|
||||
" associated with this stack.\n", protectedCount)
|
||||
} else if res == nil && len(*targets) == 0 && !jsonDisplay {
|
||||
fmt.Printf("The resources in the stack have been deleted, but the history and configuration "+
|
||||
"associated with the stack are still maintained. \nIf you want to remove the stack "+
|
||||
"completely, run 'pulumi stack rm %s'.\n", s.Ref())
|
||||
|
@ -202,6 +230,8 @@ func newDestroyCmd() *cobra.Command {
|
|||
cmd.PersistentFlags().BoolVar(
|
||||
&targetDependents, "target-dependents", false,
|
||||
"Allows destroying of dependent targets discovered but not specified in --target list")
|
||||
cmd.PersistentFlags().BoolVar(&excludeProtected, "exclude-protected", false, "Do not destroy protected resources."+
|
||||
" Destroy all other resources.")
|
||||
|
||||
// Flags for engine.UpdateOptions.
|
||||
cmd.PersistentFlags().BoolVar(
|
||||
|
@ -257,3 +287,52 @@ func newDestroyCmd() *cobra.Command {
|
|||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// seperateProtected returns a list or unprotected and protected resources respectively. This allows
|
||||
// us to safely destroy all resources in the unprotected list without invalidating any resource in
|
||||
// the protected list. Protection is contravarient: A < B where A: Protected => B: Protected, A < B
|
||||
// where B: Protected !=> A: Protected.
|
||||
//
|
||||
// A
|
||||
// B: Parent = A
|
||||
// C: Parent = A, Protect = True
|
||||
// D: Parent = C
|
||||
//
|
||||
// -->
|
||||
//
|
||||
// Unprotected: B, D
|
||||
// Protected: A, C
|
||||
//
|
||||
// We rely on the fact that `resources` is topologically sorted with respect to its dependencies.
|
||||
// This function understands that providers live outside this topological sort.
|
||||
func seperateProtected(resources []*resource.State) (
|
||||
/*unprotected*/ []*resource.State /*protected*/, []*resource.State) {
|
||||
dg := graph.NewDependencyGraph(resources)
|
||||
transitiveProtected := graph.ResourceSet{}
|
||||
for _, r := range resources {
|
||||
if r.Protect {
|
||||
rProtected := dg.TransitiveDependenciesOf(r)
|
||||
rProtected[r] = true
|
||||
transitiveProtected.UnionWith(rProtected)
|
||||
}
|
||||
}
|
||||
allResources := graph.NewResourceSetFromArray(resources)
|
||||
return allResources.SetMinus(transitiveProtected).ToArray(), transitiveProtected.ToArray()
|
||||
}
|
||||
|
||||
// Returns the number of protected resources that remain. Appends all unprotected resources to `targetUrns`.
|
||||
func handleExcludeProtected(s backend.Stack) ([]resource.URN, int, error) {
|
||||
// Get snapshot
|
||||
snapshot, err := s.Snapshot(commandContext())
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
} else if snapshot == nil {
|
||||
return nil, 0, errors.New("Failed to find the stack snapshot. Are you in a stack?")
|
||||
}
|
||||
unprotected, protected := seperateProtected(snapshot.Resources)
|
||||
targetUrns := []resource.URN{}
|
||||
for _, r := range unprotected {
|
||||
targetUrns = append(targetUrns, r.URN)
|
||||
}
|
||||
return targetUrns, len(protected), nil
|
||||
}
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// Copyright 2016-2018, Pulumi Corporation. All rights reserved.
|
||||
// Copyright 2016-2021, Pulumi Corporation. All rights reserved.
|
||||
|
||||
package graph
|
||||
|
||||
|
@ -123,6 +123,67 @@ func (dg *DependencyGraph) DependenciesOf(res *resource.State) ResourceSet {
|
|||
return set
|
||||
}
|
||||
|
||||
// `TransitiveDependenciesOf` calculates the set of resources that `r` depends
|
||||
// on, directly or indirectly. This includes as a `Parent`, a member of r's
|
||||
// `Dependencies` list or as a provider.
|
||||
//
|
||||
// This function is linear in the number of resources in the `DependencyGraph`.
|
||||
func (dg *DependencyGraph) TransitiveDependenciesOf(r *resource.State) ResourceSet {
|
||||
dependentProviders := make(map[resource.URN]struct{})
|
||||
|
||||
urns := make(map[resource.URN]*node, len(dg.resources))
|
||||
for _, r := range dg.resources {
|
||||
urns[r.URN] = &node{resource: r}
|
||||
}
|
||||
|
||||
// Linearity is due to short circuiting in the traversal.
|
||||
markAsDependency(r.URN, urns, dependentProviders)
|
||||
|
||||
// This will only trigger if (urn, node) is a provider. The check is implicit
|
||||
// in the set lookup.
|
||||
for urn := range urns {
|
||||
if _, ok := dependentProviders[urn]; ok {
|
||||
markAsDependency(urn, urns, dependentProviders)
|
||||
}
|
||||
}
|
||||
|
||||
dependencies := ResourceSet{}
|
||||
for _, r := range urns {
|
||||
if r.marked {
|
||||
dependencies[r.resource] = true
|
||||
}
|
||||
}
|
||||
// We don't want to include `r` as it's own dependency.
|
||||
delete(dependencies, r)
|
||||
return dependencies
|
||||
|
||||
}
|
||||
|
||||
// Mark a resource and its parents as a dependency. This is a helper function for `TransitiveDependenciesOf`.
|
||||
func markAsDependency(urn resource.URN, urns map[resource.URN]*node, dependedProviders map[resource.URN]struct{}) {
|
||||
r := urns[urn]
|
||||
for {
|
||||
r.marked = true
|
||||
if r.resource.Provider != "" {
|
||||
ref, err := providers.ParseReference(r.resource.Provider)
|
||||
contract.AssertNoError(err)
|
||||
dependedProviders[ref.URN()] = struct{}{}
|
||||
}
|
||||
for _, dep := range r.resource.Dependencies {
|
||||
markAsDependency(dep, urns, dependedProviders)
|
||||
}
|
||||
|
||||
// If p is already marked, we don't need to continue to traverse. All
|
||||
// nodes above p will have already been marked. This is a property of
|
||||
// `resources` being topologically sorted.
|
||||
if p, ok := urns[r.resource.Parent]; ok && !p.marked {
|
||||
r = p
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NewDependencyGraph creates a new DependencyGraph from a list of resources.
|
||||
// The resources should be in topological order with respect to their dependencies, including
|
||||
// parents appearing before children.
|
||||
|
@ -143,3 +204,9 @@ func NewDependencyGraph(resources []*resource.State) *DependencyGraph {
|
|||
|
||||
return &DependencyGraph{index, resources, childrenOf}
|
||||
}
|
||||
|
||||
// A node in a graph.
|
||||
type node struct {
|
||||
marked bool
|
||||
resource *resource.State
|
||||
}
|
||||
|
|
|
@ -202,6 +202,24 @@ func TestRapidDependingOnOrdered(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestRapidTransitiveDependenciesOf(t *testing.T) {
|
||||
graphCheck(t, func(t *rapid.T, universe []*resource.State) {
|
||||
expectedInTDepsOf := transitively(universe)(expectedDependenciesOf)
|
||||
dg := NewDependencyGraph(universe)
|
||||
for _, a := range universe {
|
||||
tda := dg.TransitiveDependenciesOf(a)
|
||||
for _, b := range universe {
|
||||
assert.Equalf(t,
|
||||
expectedInTDepsOf(a, b),
|
||||
tda[b],
|
||||
"Mismatch on a=%v, b=%b",
|
||||
a.URN,
|
||||
b.URN)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Generators --------------------------------------------------------------------------------------
|
||||
|
||||
// Generates ordered values of type `[]ResourceState` that:
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// Copyright 2016-2018, Pulumi Corporation. All rights reserved.
|
||||
// Copyright 2016-2021, Pulumi Corporation. All rights reserved.
|
||||
|
||||
package graph
|
||||
|
||||
|
@ -229,3 +229,29 @@ func TestDependenciesOfRemoteComponentsNoCycle(t *testing.T) {
|
|||
assert.True(t, rDependencies[parent])
|
||||
assert.False(t, rDependencies[child])
|
||||
}
|
||||
|
||||
func TestTransitiveDependenciesOf(t *testing.T) {
|
||||
aws := NewProviderResource("aws", "default", "0")
|
||||
parent := NewResource("parent", aws)
|
||||
greatUncle := NewResource("greatUncle", aws)
|
||||
uncle := NewResource("r", aws)
|
||||
uncle.Parent = greatUncle.URN
|
||||
child := NewResource("child", aws, uncle.URN)
|
||||
child.Parent = parent.URN
|
||||
baby := NewResource("baby", aws)
|
||||
baby.Parent = child.URN
|
||||
|
||||
dg := NewDependencyGraph([]*resource.State{
|
||||
aws,
|
||||
parent,
|
||||
greatUncle,
|
||||
uncle,
|
||||
child,
|
||||
baby,
|
||||
})
|
||||
// <(relation)- as an alias for depends on via relation
|
||||
// baby <(Parent)- child <(Dependency)- uncle <(Parent)- greatUncle <(Provider)- aws
|
||||
set := dg.TransitiveDependenciesOf(baby)
|
||||
assert.True(t, set[aws], "everything should depend on the provider")
|
||||
assert.True(t, set[greatUncle], "child depends on greatUncle")
|
||||
}
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// Copyright 2016-2018, Pulumi Corporation.
|
||||
// Copyright 2016-2021, Pulumi Corporation.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
@ -16,6 +16,7 @@ package graph
|
|||
|
||||
import (
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// ResourceSet represents a set of Resources.
|
||||
|
@ -32,3 +33,62 @@ func (s ResourceSet) Intersect(other ResourceSet) ResourceSet {
|
|||
|
||||
return newSet
|
||||
}
|
||||
|
||||
// Returns the contents of the set as an array of resources. To ensure
|
||||
// determinism, they are sorted by urn.
|
||||
func (s ResourceSet) ToArray() []*resource.State {
|
||||
arr := make([]*resource.State, len(s))
|
||||
i := 0
|
||||
for r := range s {
|
||||
arr[i] = r
|
||||
i++
|
||||
}
|
||||
sort.Slice(arr, func(i, j int) bool {
|
||||
return arr[i].URN < arr[j].URN
|
||||
})
|
||||
return arr
|
||||
}
|
||||
|
||||
// Produces a set from an array.
|
||||
func NewResourceSetFromArray(arr []*resource.State) ResourceSet {
|
||||
s := ResourceSet{}
|
||||
for _, r := range arr {
|
||||
s[r] = true
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Produces a shallow copy of `s`.
|
||||
func CopyResourceSet(s ResourceSet) ResourceSet {
|
||||
result := ResourceSet{}
|
||||
for k, v := range s {
|
||||
result[k] = v
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// Computes s - other. Input sets are unchanged. If `other[k] = false`, then `k`
|
||||
// will not be removed from `s`.
|
||||
func (s ResourceSet) SetMinus(other ResourceSet) ResourceSet {
|
||||
result := CopyResourceSet(s)
|
||||
for k, v := range other {
|
||||
if v {
|
||||
delete(result, k)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// Produces a new set with elements from both sets. The original sets are unchanged.
|
||||
func (s ResourceSet) Union(other ResourceSet) ResourceSet {
|
||||
result := CopyResourceSet(s)
|
||||
return result.UnionWith(other)
|
||||
}
|
||||
|
||||
// Alters `s` to include elements of `other`.
|
||||
func (s ResourceSet) UnionWith(other ResourceSet) ResourceSet {
|
||||
for k, v := range other {
|
||||
s[k] = v || s[k]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// Copyright 2016-2018, Pulumi Corporation.
|
||||
// Copyright 2016-2021, Pulumi Corporation.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
@ -37,3 +37,40 @@ func TestIntersect(t *testing.T) {
|
|||
assert.True(t, setC[b])
|
||||
assert.False(t, setC[c])
|
||||
}
|
||||
|
||||
func TestUnion(t *testing.T) {
|
||||
a := NewResource("a", nil)
|
||||
b := NewResource("b", nil)
|
||||
c := NewResource("c", nil)
|
||||
|
||||
setA := make(ResourceSet)
|
||||
setA[a] = true
|
||||
setA[c] = true
|
||||
|
||||
setB := make(ResourceSet)
|
||||
setB[b] = true
|
||||
|
||||
setC := setA.Union(setB)
|
||||
assert.True(t, setC[a])
|
||||
assert.True(t, setC[b])
|
||||
assert.True(t, setC[c])
|
||||
}
|
||||
|
||||
func TestSetMinus(t *testing.T) {
|
||||
a := NewResource("a", nil)
|
||||
b := NewResource("b", nil)
|
||||
c := NewResource("c", nil)
|
||||
|
||||
setA := make(ResourceSet)
|
||||
setA[a] = true
|
||||
setA[b] = true
|
||||
|
||||
setB := make(ResourceSet)
|
||||
setB[b] = true
|
||||
setB[c] = true
|
||||
|
||||
setC := setA.SetMinus(setB)
|
||||
assert.True(t, setC[a])
|
||||
assert.False(t, setC[b])
|
||||
assert.False(t, setC[c])
|
||||
}
|
||||
|
|
3
tests/integration/exclude_protected/Pulumi.yaml
Normal file
3
tests/integration/exclude_protected/Pulumi.yaml
Normal file
|
@ -0,0 +1,3 @@
|
|||
name: exclude-protected
|
||||
runtime: nodejs
|
||||
description: A minimal AWS TypeScript Pulumi program
|
35
tests/integration/exclude_protected/index.ts
Normal file
35
tests/integration/exclude_protected/index.ts
Normal file
|
@ -0,0 +1,35 @@
|
|||
import * as pulumi from "@pulumi/pulumi";
|
||||
|
||||
|
||||
class Resource extends pulumi.ComponentResource {
|
||||
constructor(name: string, _?: {}, opts?: pulumi.ComponentResourceOptions) {
|
||||
super("my:module:Resource", name, {}, opts);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
const bucket1 = new Resource("my-bucket", {}, { protect: true });
|
||||
// Because `protect` is explicitly set to false, we will delete this.
|
||||
new Resource("my-bucket-child", {}, { protect: false, parent: bucket1 });
|
||||
new Resource("my-bucket-child-protected", {}, { protect: true, parent: bucket1 });
|
||||
|
||||
const bucket2 = new Resource("my-2bucket", {}, { protect: false });
|
||||
new Resource("my-2bucket-child", {}, { protect: false, parent: bucket2 });
|
||||
new Resource("my-2bucket-protected-child", {}, { protect: true, parent: bucket2 });
|
||||
|
||||
const p = new Resource("provided-bucket", {}, { protect: true })
|
||||
// Inherits protected status from `p`. This is protected in the state, and is thus safe.
|
||||
new Resource("provided-bucket-child", {}, { parent: p })
|
||||
new Resource("provided-bucket-child-unprotected", {}, { parent: p, protect: false })
|
||||
|
||||
// If possible, we should do a test with providers, that looks something like
|
||||
// this. Doing a provider test with component resources is problematic because
|
||||
// `ComponentResources` don't have CRUD operations.
|
||||
//
|
||||
// import * as aws from "@pulumi/aws";
|
||||
// new aws.s3.Bucket("provider-unprotected", {}, { provider: prov })
|
||||
// const p = new aws.s3.Bucket("provided-bucket", {}, { provider: prov, protect: true })
|
||||
// // Inherits protected status from `p`. This is protected in the state, and is thus safe.
|
||||
// new aws.s3.Bucket("provided-bucket-child", {}, { parent: p })
|
||||
// new aws.s3.Bucket("provided-bucket-child-unprotected", {}, { parent: p, protect: false })
|
9
tests/integration/exclude_protected/package.json
Normal file
9
tests/integration/exclude_protected/package.json
Normal file
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"name": "test-protect",
|
||||
"devDependencies": {
|
||||
"@types/node": "^10.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@pulumi/pulumi": "^3.0.0"
|
||||
}
|
||||
}
|
|
@ -943,3 +943,29 @@ func TestJSONOutputWithStreamingPreview(t *testing.T) {
|
|||
},
|
||||
})
|
||||
}
|
||||
|
||||
func TestExcludeProtected(t *testing.T) {
|
||||
e := ptesting.NewEnvironment(t)
|
||||
defer func() {
|
||||
if !t.Failed() {
|
||||
e.DeleteEnvironment()
|
||||
}
|
||||
}()
|
||||
|
||||
e.ImportDirectory("exclude_protected")
|
||||
|
||||
e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL())
|
||||
|
||||
e.RunCommand("pulumi", "stack", "init", "dev")
|
||||
|
||||
e.RunCommand("yarn", "link", "@pulumi/pulumi")
|
||||
e.RunCommand("yarn", "install")
|
||||
|
||||
e.RunCommand("pulumi", "up", "--skip-preview", "--yes")
|
||||
|
||||
stdout, _ := e.RunCommand("pulumi", "destroy", "--skip-preview", "--yes", "--exclude-protected")
|
||||
assert.Contains(t, stdout, "All unprotected resources were destroyed. There are still 7 protected resources")
|
||||
// We run the command again, but this time there are not unprotected resources to destroy.
|
||||
stdout, _ = e.RunCommand("pulumi", "destroy", "--skip-preview", "--yes", "--exclude-protected")
|
||||
assert.Contains(t, stdout, "There were no unprotected resources to destroy. There are still 7")
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue