Implement the --exclude-protected feature (#8359)

* Implement the --exclude-protected feature

This piggybacks on the same machinery used by the --target flag. By
examining the stack, we find a list of all resources managed by
Pulumi (in that stack). We then form them into a DAG, and mark all
resources as either protected or unprotected.

A resource is protected it has the `Protect` flag set or is has a child
with the `protect` flag set. It is unprotected otherwise.

We then pass the urns of unprotected resources to the update options
passed to the destroy operation in the same way that `--target` does.

* Update changelog

* Handle providers correctly

* Add integration test

* Protect dependencies of protected resources

* Handle --exclude-protected in separate function

* Simplify implementation via DependencyGraph

* Add TransitiveDependenciesOf

* Cleanup unused functions

* Gate printed message behind !jsonDisplay

* Ensure provider is not `""`

* Clean up documentation (and some code)

CHANGELOG_PENDING.md

@@ -1,8 +1,11 @@
 ### Improvements
 * Adds CI detector for Buildkite [#7933](https://github.com/pulumi/pulumi/pull/7933)
 
-- [CLI] Adding the ability to use `pulumi org set [name]` to set a default org
-  to use when creating a stacks in the Pulumi Service backend or Self -hosted Service
+- [cli] - Add `--exclude-protected` flag to `pulumi destroy`.
+  [#8359](https://github.com/pulumi/pulumi/pull/8359)
+
+- [cli] Adding the ability to use `pulumi org set [name]` to set a default org
+  to use when creating a stacks in the Pulumi Service backend or self-hosted Service
   [#8352](https://github.com/pulumi/pulumi/pull/8352)
 
 - [schema] Add IsOverlay option to disable codegen for particular types

pkg/cmd/pulumi/destroy.go

@@ -1,4 +1,4 @@
-// Copyright 2016-2018, Pulumi Corporation.
+// Copyright 2016-2021, Pulumi Corporation.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -24,8 +24,10 @@ import (
 	"github.com/pulumi/pulumi/pkg/v3/backend"
 	"github.com/pulumi/pulumi/pkg/v3/backend/display"
 	"github.com/pulumi/pulumi/pkg/v3/engine"
+	"github.com/pulumi/pulumi/pkg/v3/resource/graph"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/result"
 )
 
@@ -52,6 +54,7 @@ func newDestroyCmd() *cobra.Command {
 	var yes bool
 	var targets *[]string
 	var targetDependents bool
+	var excludeProtected bool
 
 	var cmd = &cobra.Command{
 		Use:        "destroy",
@@ -149,6 +152,29 @@ func newDestroyCmd() *cobra.Command {
 			if err != nil {
 				return result.FromError(err)
 			}
+
+			if targets != nil && len(*targets) > 0 && excludeProtected {
+				return result.FromError(errors.New("You cannot specify --target and --exclude-protected"))
+			}
+
+			var protectedCount int
+			if excludeProtected {
+				contract.Assert(len(targetUrns) == 0)
+				targetUrns, protectedCount, err = handleExcludeProtected(s)
+				if err != nil {
+					return result.FromError(err)
+				} else if protectedCount > 0 && len(targetUrns) == 0 {
+					if !jsonDisplay {
+						fmt.Printf("There were no unprotected resources to destroy. There are still %d"+
+							" protected resources associated with this stack.\n", protectedCount)
+					}
+					// We need to return now. Otherwise the update will conclude
+					// we tried to destroy everything and error for trying to
+					// destroy a protected resource.
+					return nil
+				}
+			}
+
 			opts.Engine = engine.UpdateOptions{
 				Parallel:                  parallel,
 				Debug:                     debug,
@@ -170,8 +196,10 @@ func newDestroyCmd() *cobra.Command {
 				SecretsManager:     sm,
 				Scopes:             cancellationScopes,
 			})
-
-			if res == nil && len(*targets) == 0 && !jsonDisplay {
+			if res == nil && protectedCount > 0 && !jsonDisplay {
+				fmt.Printf("All unprotected resources were destroyed. There are still %d protected resources"+
+					" associated with this stack.\n", protectedCount)
+			} else if res == nil && len(*targets) == 0 && !jsonDisplay {
 				fmt.Printf("The resources in the stack have been deleted, but the history and configuration "+
 					"associated with the stack are still maintained. \nIf you want to remove the stack "+
 					"completely, run 'pulumi stack rm %s'.\n", s.Ref())
@@ -202,6 +230,8 @@ func newDestroyCmd() *cobra.Command {
 	cmd.PersistentFlags().BoolVar(
 		&targetDependents, "target-dependents", false,
 		"Allows destroying of dependent targets discovered but not specified in --target list")
+	cmd.PersistentFlags().BoolVar(&excludeProtected, "exclude-protected", false, "Do not destroy protected resources."+
+		" Destroy all other resources.")
 
 	// Flags for engine.UpdateOptions.
 	cmd.PersistentFlags().BoolVar(
@@ -257,3 +287,52 @@ func newDestroyCmd() *cobra.Command {
 
 	return cmd
 }
+
+// seperateProtected returns a list or unprotected and protected resources respectively. This allows
+// us to safely destroy all resources in the unprotected list without invalidating any resource in
+// the protected list. Protection is contravarient: A < B where A: Protected => B: Protected, A < B
+// where B: Protected !=> A: Protected.
+//
+// A
+// B: Parent = A
+// C: Parent = A, Protect = True
+// D: Parent = C
+//
+// -->
+//
+// Unprotected: B, D
+// Protected: A, C
+//
+// We rely on the fact that `resources` is topologically sorted with respect to its dependencies.
+// This function understands that providers live outside this topological sort.
+func seperateProtected(resources []*resource.State) (
+	/*unprotected*/ []*resource.State /*protected*/, []*resource.State) {
+	dg := graph.NewDependencyGraph(resources)
+	transitiveProtected := graph.ResourceSet{}
+	for _, r := range resources {
+		if r.Protect {
+			rProtected := dg.TransitiveDependenciesOf(r)
+			rProtected[r] = true
+			transitiveProtected.UnionWith(rProtected)
+		}
+	}
+	allResources := graph.NewResourceSetFromArray(resources)
+	return allResources.SetMinus(transitiveProtected).ToArray(), transitiveProtected.ToArray()
+}
+
+// Returns the number of protected resources that remain. Appends all unprotected resources to `targetUrns`.
+func handleExcludeProtected(s backend.Stack) ([]resource.URN, int, error) {
+	// Get snapshot
+	snapshot, err := s.Snapshot(commandContext())
+	if err != nil {
+		return nil, 0, err
+	} else if snapshot == nil {
+		return nil, 0, errors.New("Failed to find the stack snapshot. Are you in a stack?")
+	}
+	unprotected, protected := seperateProtected(snapshot.Resources)
+	targetUrns := []resource.URN{}
+	for _, r := range unprotected {
+		targetUrns = append(targetUrns, r.URN)
+	}
+	return targetUrns, len(protected), nil
+}

pkg/resource/graph/dependency_graph.go

@@ -1,4 +1,4 @@
-// Copyright 2016-2018, Pulumi Corporation.  All rights reserved.
+// Copyright 2016-2021, Pulumi Corporation.  All rights reserved.
 
 package graph
 
@@ -123,6 +123,67 @@ func (dg *DependencyGraph) DependenciesOf(res *resource.State) ResourceSet {
 	return set
 }
 
+// `TransitiveDependenciesOf` calculates the set of resources that `r` depends
+// on, directly or indirectly. This includes as a `Parent`, a member of r's
+// `Dependencies` list or as a provider.
+//
+// This function is linear in the number of resources in the `DependencyGraph`.
+func (dg *DependencyGraph) TransitiveDependenciesOf(r *resource.State) ResourceSet {
+	dependentProviders := make(map[resource.URN]struct{})
+
+	urns := make(map[resource.URN]*node, len(dg.resources))
+	for _, r := range dg.resources {
+		urns[r.URN] = &node{resource: r}
+	}
+
+	// Linearity is due to short circuiting in the traversal.
+	markAsDependency(r.URN, urns, dependentProviders)
+
+	// This will only trigger if (urn, node) is a provider. The check is implicit
+	// in the set lookup.
+	for urn := range urns {
+		if _, ok := dependentProviders[urn]; ok {
+			markAsDependency(urn, urns, dependentProviders)
+		}
+	}
+
+	dependencies := ResourceSet{}
+	for _, r := range urns {
+		if r.marked {
+			dependencies[r.resource] = true
+		}
+	}
+	// We don't want to include `r` as it's own dependency.
+	delete(dependencies, r)
+	return dependencies
+
+}
+
+// Mark a resource and its parents as a dependency. This is a helper function for `TransitiveDependenciesOf`.
+func markAsDependency(urn resource.URN, urns map[resource.URN]*node, dependedProviders map[resource.URN]struct{}) {
+	r := urns[urn]
+	for {
+		r.marked = true
+		if r.resource.Provider != "" {
+			ref, err := providers.ParseReference(r.resource.Provider)
+			contract.AssertNoError(err)
+			dependedProviders[ref.URN()] = struct{}{}
+		}
+		for _, dep := range r.resource.Dependencies {
+			markAsDependency(dep, urns, dependedProviders)
+		}
+
+		// If p is already marked, we don't need to continue to traverse. All
+		// nodes above p will have already been marked. This is a property of
+		// `resources` being topologically sorted.
+		if p, ok := urns[r.resource.Parent]; ok && !p.marked {
+			r = p
+		} else {
+			break
+		}
+	}
+}
+
 // NewDependencyGraph creates a new DependencyGraph from a list of resources.
 // The resources should be in topological order with respect to their dependencies, including
 // parents appearing before children.
@@ -143,3 +204,9 @@ func NewDependencyGraph(resources []*resource.State) *DependencyGraph {
 
 	return &DependencyGraph{index, resources, childrenOf}
 }
+
+// A node in a graph.
+type node struct {
+	marked   bool
+	resource *resource.State
+}

pkg/resource/graph/dependency_graph_rapid_test.go

@@ -202,6 +202,24 @@ func TestRapidDependingOnOrdered(t *testing.T) {
 	}
 }
 
+func TestRapidTransitiveDependenciesOf(t *testing.T) {
+	graphCheck(t, func(t *rapid.T, universe []*resource.State) {
+		expectedInTDepsOf := transitively(universe)(expectedDependenciesOf)
+		dg := NewDependencyGraph(universe)
+		for _, a := range universe {
+			tda := dg.TransitiveDependenciesOf(a)
+			for _, b := range universe {
+				assert.Equalf(t,
+					expectedInTDepsOf(a, b),
+					tda[b],
+					"Mismatch on a=%v, b=%b",
+					a.URN,
+					b.URN)
+			}
+		}
+	})
+}
+
 // Generators --------------------------------------------------------------------------------------
 
 // Generates ordered values of type `[]ResourceState` that:

pkg/resource/graph/dependency_graph_test.go

@@ -1,4 +1,4 @@
-// Copyright 2016-2018, Pulumi Corporation.  All rights reserved.
+// Copyright 2016-2021, Pulumi Corporation.  All rights reserved.
 
 package graph
 
@@ -229,3 +229,29 @@ func TestDependenciesOfRemoteComponentsNoCycle(t *testing.T) {
 	assert.True(t, rDependencies[parent])
 	assert.False(t, rDependencies[child])
 }
+
+func TestTransitiveDependenciesOf(t *testing.T) {
+	aws := NewProviderResource("aws", "default", "0")
+	parent := NewResource("parent", aws)
+	greatUncle := NewResource("greatUncle", aws)
+	uncle := NewResource("r", aws)
+	uncle.Parent = greatUncle.URN
+	child := NewResource("child", aws, uncle.URN)
+	child.Parent = parent.URN
+	baby := NewResource("baby", aws)
+	baby.Parent = child.URN
+
+	dg := NewDependencyGraph([]*resource.State{
+		aws,
+		parent,
+		greatUncle,
+		uncle,
+		child,
+		baby,
+	})
+	// <(relation)- as an alias for depends on via relation
+	// baby <(Parent)- child <(Dependency)- uncle <(Parent)- greatUncle <(Provider)- aws
+	set := dg.TransitiveDependenciesOf(baby)
+	assert.True(t, set[aws], "everything should depend on the provider")
+	assert.True(t, set[greatUncle], "child depends on greatUncle")
+}

pkg/resource/graph/resource_set.go

@@ -1,4 +1,4 @@
-// Copyright 2016-2018, Pulumi Corporation.
+// Copyright 2016-2021, Pulumi Corporation.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@ package graph
 
 import (
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
+	"sort"
 )
 
 // ResourceSet represents a set of Resources.
@@ -32,3 +33,62 @@ func (s ResourceSet) Intersect(other ResourceSet) ResourceSet {
 
 	return newSet
 }
+
+// Returns the contents of the set as an array of resources. To ensure
+// determinism, they are sorted by urn.
+func (s ResourceSet) ToArray() []*resource.State {
+	arr := make([]*resource.State, len(s))
+	i := 0
+	for r := range s {
+		arr[i] = r
+		i++
+	}
+	sort.Slice(arr, func(i, j int) bool {
+		return arr[i].URN < arr[j].URN
+	})
+	return arr
+}
+
+// Produces a set from an array.
+func NewResourceSetFromArray(arr []*resource.State) ResourceSet {
+	s := ResourceSet{}
+	for _, r := range arr {
+		s[r] = true
+	}
+	return s
+}
+
+// Produces a shallow copy of `s`.
+func CopyResourceSet(s ResourceSet) ResourceSet {
+	result := ResourceSet{}
+	for k, v := range s {
+		result[k] = v
+	}
+	return result
+}
+
+// Computes s - other. Input sets are unchanged. If `other[k] = false`, then `k`
+// will not be removed from `s`.
+func (s ResourceSet) SetMinus(other ResourceSet) ResourceSet {
+	result := CopyResourceSet(s)
+	for k, v := range other {
+		if v {
+			delete(result, k)
+		}
+	}
+	return result
+}
+
+// Produces a new set with elements from both sets. The original sets are unchanged.
+func (s ResourceSet) Union(other ResourceSet) ResourceSet {
+	result := CopyResourceSet(s)
+	return result.UnionWith(other)
+}
+
+// Alters `s` to include elements of `other`.
+func (s ResourceSet) UnionWith(other ResourceSet) ResourceSet {
+	for k, v := range other {
+		s[k] = v || s[k]
+	}
+	return s
+}

pkg/resource/graph/resource_set_test.go

@@ -1,4 +1,4 @@
-// Copyright 2016-2018, Pulumi Corporation.
+// Copyright 2016-2021, Pulumi Corporation.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -37,3 +37,40 @@ func TestIntersect(t *testing.T) {
 	assert.True(t, setC[b])
 	assert.False(t, setC[c])
 }
+
+func TestUnion(t *testing.T) {
+	a := NewResource("a", nil)
+	b := NewResource("b", nil)
+	c := NewResource("c", nil)
+
+	setA := make(ResourceSet)
+	setA[a] = true
+	setA[c] = true
+
+	setB := make(ResourceSet)
+	setB[b] = true
+
+	setC := setA.Union(setB)
+	assert.True(t, setC[a])
+	assert.True(t, setC[b])
+	assert.True(t, setC[c])
+}
+
+func TestSetMinus(t *testing.T) {
+	a := NewResource("a", nil)
+	b := NewResource("b", nil)
+	c := NewResource("c", nil)
+
+	setA := make(ResourceSet)
+	setA[a] = true
+	setA[b] = true
+
+	setB := make(ResourceSet)
+	setB[b] = true
+	setB[c] = true
+
+	setC := setA.SetMinus(setB)
+	assert.True(t, setC[a])
+	assert.False(t, setC[b])
+	assert.False(t, setC[c])
+}

tests/integration/exclude_protected/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: exclude-protected
+runtime: nodejs
+description: A minimal AWS TypeScript Pulumi program

tests/integration/exclude_protected/index.ts

@@ -0,0 +1,35 @@
+import * as pulumi from "@pulumi/pulumi";
+
+
+class Resource extends pulumi.ComponentResource {
+    constructor(name: string, _?: {}, opts?: pulumi.ComponentResourceOptions) {
+        super("my:module:Resource", name, {}, opts);
+    }
+}
+
+
+
+const bucket1 = new Resource("my-bucket", {}, { protect: true });
+// Because `protect` is explicitly set to false, we will delete this.
+new Resource("my-bucket-child", {}, { protect: false, parent: bucket1 });
+new Resource("my-bucket-child-protected", {}, { protect: true, parent: bucket1 });
+
+const bucket2 = new Resource("my-2bucket", {}, { protect: false });
+new Resource("my-2bucket-child", {}, { protect: false, parent: bucket2 });
+new Resource("my-2bucket-protected-child", {}, { protect: true, parent: bucket2 });
+
+const p = new Resource("provided-bucket", {}, { protect: true })
+// Inherits protected status from `p`. This is protected in the state, and is thus safe.
+new Resource("provided-bucket-child", {}, { parent: p })
+new Resource("provided-bucket-child-unprotected", {}, { parent: p, protect: false })
+
+// If possible, we should do a test with providers, that looks something like
+// this. Doing a provider test with component resources is problematic because
+// `ComponentResources` don't have CRUD operations.
+//
+// import * as aws from "@pulumi/aws";
+// new aws.s3.Bucket("provider-unprotected", {}, { provider: prov })
+// const p = new aws.s3.Bucket("provided-bucket", {}, { provider: prov, protect: true })
+// // Inherits protected status from `p`. This is protected in the state, and is thus safe.
+// new aws.s3.Bucket("provided-bucket-child", {}, { parent: p })
+// new aws.s3.Bucket("provided-bucket-child-unprotected", {}, { parent: p, protect: false })

tests/integration/exclude_protected/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "test-protect",
+  "devDependencies": {
+    "@types/node": "^10.0.0"
+  },
+  "peerDependencies": {
+    "@pulumi/pulumi": "^3.0.0"
+  }
+}

tests/integration/integration_test.go

@@ -943,3 +943,29 @@ func TestJSONOutputWithStreamingPreview(t *testing.T) {
 		},
 	})
 }
+
+func TestExcludeProtected(t *testing.T) {
+	e := ptesting.NewEnvironment(t)
+	defer func() {
+		if !t.Failed() {
+			e.DeleteEnvironment()
+		}
+	}()
+
+	e.ImportDirectory("exclude_protected")
+
+	e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL())
+
+	e.RunCommand("pulumi", "stack", "init", "dev")
+
+	e.RunCommand("yarn", "link", "@pulumi/pulumi")
+	e.RunCommand("yarn", "install")
+
+	e.RunCommand("pulumi", "up", "--skip-preview", "--yes")
+
+	stdout, _ := e.RunCommand("pulumi", "destroy", "--skip-preview", "--yes", "--exclude-protected")
+	assert.Contains(t, stdout, "All unprotected resources were destroyed. There are still 7 protected resources")
+	// We run the command again, but this time there are not unprotected resources to destroy.
+	stdout, _ = e.RunCommand("pulumi", "destroy", "--skip-preview", "--yes", "--exclude-protected")
+	assert.Contains(t, stdout, "There were no unprotected resources to destroy. There are still 7")
+}