maxmustermann/pulumi
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge pull request #434 from pulumi/PendingDeletes

Browse source 
Track resources that are pending deletion in checkpoints.
This commit is contained in:
Pat Gavlin 2017-10-19 10:57:00 -07:00 committed by GitHub
parent 599ca8ea43 23864b9459
commit 9895e8006f
10 changed files with 69 additions and 191 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Makefile
View file

@ -39,6 +39,10 @@ install:
go install -ldflags "-X main.version=${VERSION}" ${PROJECT}
go install -ldflags "-X main.version=${VERSION}" ${PROJECT}/cmd/lumidl
.PHONY: format
format:
find . -iname "*.go" -not -path "./vendor/*" | xargs gofmt -s -w
.PHONY: lint
lint:
@$(ECHO) "\033[0;32mLINT:\033[0m"

6
examples/examples_test.go
View file

@ -29,9 +29,9 @@ func TestExamples(t *testing.T) {
Dir: path.Join(cwd, "dynamic-provider/simple"),
Dependencies: []string{"pulumi"},
Config: map[string]string{
"simple:config:w": "1",
"simple:config:x": "1",
"simple:config:y": "1",
"simple:config:w": "1",
"simple:config:x": "1",
"simple:config:y": "1",
},
},
}

5
pkg/resource/deploy/plan.go
View file

@ -43,6 +43,11 @@ func NewPlan(ctx *plugin.Context, target *Target, prev *Snapshot, source Source,
olds := make(map[resource.URN]*resource.State)
if prev != nil {
for _, oldres := range prev.Resources {
// Ignore resources that are pending deletion; these should not be recorded in the LUT.
if oldres.Delete {
continue
}
urn := oldres.URN
contract.Assert(olds[urn] == nil)
olds[urn] = oldres

6
pkg/resource/deploy/plan_apply.go
View file

@ -190,7 +190,7 @@ func (iter *PlanIterator) nextResourceSteps(goal SourceGoal) ([]Step, error) {
// Produce a new state object that we'll build up as operations are performed. It begins with empty outputs.
// Ultimately, this is what will get serialized into the checkpoint file.
new := resource.NewState(res.Type, urn, res.Custom, "", res.Properties, nil, nil, res.Children)
new := resource.NewState(res.Type, urn, res.Custom, false, "", res.Properties, nil, nil, res.Children)
// If there is an old resource, apply its default properties before going any further.
old, hasold := iter.p.Olds()[urn]
@ -380,8 +380,8 @@ func (iter *PlanIterator) calculateDeletes() []*resource.State {
for i := len(prev.Resources) - 1; i >= 0; i-- {
res := prev.Resources[i]
urn := res.URN
contract.Assert(!iter.creates[urn])
if (!iter.sames[urn] && !iter.updates[urn]) || iter.replaces[urn] {
contract.Assert(!iter.creates[urn] || res.Delete)
if res.Delete || (!iter.sames[urn] && !iter.updates[urn]) || iter.replaces[urn] {
dels = append(dels, res)
}
}

6
pkg/resource/deploy/plan_test.go
View file

@ -158,7 +158,7 @@ func TestBasicCRUDPlan(t *testing.T) {
urnD := resource.NewURN(ns, pkgname, typD, namD)
// Create the old resources snapshot.
oldResB := resource.NewState(typB, urnB, true, resource.ID("b-b-b"),
oldResB := resource.NewState(typB, urnB, true, false, resource.ID("b-b-b"),
resource.PropertyMap{
"bf1": resource.NewStringProperty("b-value"),
"bf2": resource.NewNumberProperty(42),
@ -167,7 +167,7 @@ func TestBasicCRUDPlan(t *testing.T) {
nil,
nil,
)
oldResC := resource.NewState(typC, urnC, true, resource.ID("c-c-c"),
oldResC := resource.NewState(typC, urnC, true, false, resource.ID("c-c-c"),
resource.PropertyMap{
"cf1": resource.NewStringProperty("c-value"),
"cf2": resource.NewNumberProperty(83),
@ -179,7 +179,7 @@ func TestBasicCRUDPlan(t *testing.T) {
},
nil,
)
oldResD := resource.NewState(typD, urnD, true, resource.ID("d-d-d"),
oldResD := resource.NewState(typD, urnD, true, false, resource.ID("d-d-d"),
resource.PropertyMap{
"df1": resource.NewStringProperty("d-value"),
"df2": resource.NewNumberProperty(167),

17
pkg/resource/deploy/step.go
View file

@ -44,9 +44,11 @@ func NewSameStep(iter *PlanIterator, goal SourceGoal, old *resource.State, new *
contract.Assert(old != nil)
contract.Assert(old.URN != "")
contract.Assert(old.ID != "" || !old.Custom)
contract.Assert(!old.Delete)
contract.Assert(new != nil)
contract.Assert(new.URN != "")
contract.Assert(new.ID == "")
contract.Assert(!new.Delete)
return &SameStep{
iter: iter,
goal: goal,
@ -95,6 +97,7 @@ func NewCreateStep(iter *PlanIterator, goal SourceGoal, new *resource.State) Ste
contract.Assert(new != nil)
contract.Assert(new.URN != "")
contract.Assert(new.ID == "")
contract.Assert(!new.Delete)
return &CreateStep{
iter: iter,
goal: goal,
@ -108,9 +111,11 @@ func NewCreateReplacementStep(iter *PlanIterator, goal SourceGoal,
contract.Assert(old != nil)
contract.Assert(old.URN != "")
contract.Assert(old.ID != "" || !old.Custom)
contract.Assert(!old.Delete)
contract.Assert(new != nil)
contract.Assert(new.URN != "")
contract.Assert(new.ID == "")
contract.Assert(!new.Delete)
contract.Assert(old.Type == new.Type)
return &CreateStep{
iter: iter,
@ -155,6 +160,11 @@ func (s *CreateStep) Apply() (resource.Status, error) {
s.new.Outputs = outs
}
// Mark the old resource as pending deletion if necessary.
if s.replacing {
s.old.Delete = true
}
// And finish the overall operation.
s.goal.Done(s.new, false, nil)
s.iter.AppendStateSnapshot(s.new)
@ -180,6 +190,7 @@ func NewDeleteStep(iter *PlanIterator, old *resource.State, replacing bool) Step
contract.Assert(old != nil)
contract.Assert(old.URN != "")
contract.Assert(old.ID != "" || !old.Custom)
contract.Assert(!replacing || old.Delete)
return &DeleteStep{
iter: iter,
old: old,
@ -237,9 +248,11 @@ func NewUpdateStep(iter *PlanIterator, goal SourceGoal, old *resource.State,
contract.Assert(old != nil)
contract.Assert(old.URN != "")
contract.Assert(old.ID != "" || !old.Custom)
contract.Assert(!old.Delete)
contract.Assert(new != nil)
contract.Assert(new.URN != "")
contract.Assert(new.ID == "")
contract.Assert(!new.Delete)
contract.Assert(old.Type == new.Type)
return &UpdateStep{
iter: iter,
@ -304,9 +317,11 @@ func NewReplaceStep(iter *PlanIterator, old *resource.State, new *resource.State
contract.Assert(old != nil)
contract.Assert(old.URN != "")
contract.Assert(old.ID != "" || !old.Custom)
contract.Assert(!old.Delete)
contract.Assert(new != nil)
contract.Assert(new.URN != "")
contract.Assert(new.ID == "")
contract.Assert(!new.Delete)
return &ReplaceStep{
iter: iter,
old: old,
@ -326,6 +341,8 @@ func (s *ReplaceStep) Keys() []resource.PropertyKey { return s.keys }
func (s *ReplaceStep) Logical() bool { return true }
func (s *ReplaceStep) Apply() (resource.Status, error) {
// We should have marked the old resource for deletion in the CreateReplacement step.
contract.Assert(s.old.Delete)
return resource.StatusOK, nil
}

4
pkg/resource/resource_state.go
View file

@ -14,6 +14,7 @@ type State struct {
Type tokens.Type // the resource's type.
URN URN // the resource's object urn, a human-friendly, unique name for the resource.
Custom bool // true if the resource is custom, managed by a plugin.
Delete bool // true if this resource is pending deletion due to a replacement.
ID ID // the resource's unique ID, assigned by the resource provider (or blank if none/uncreated).
Inputs PropertyMap // the resource's input properties (as specified by the program).
Defaults PropertyMap // the resource's default property values (if any, given by the provider).
@ -22,7 +23,7 @@ type State struct {
}
// NewState creates a new resource value from existing resource state information.
func NewState(t tokens.Type, urn URN, custom bool, id ID,
func NewState(t tokens.Type, urn URN, custom bool, del bool, id ID,
inputs PropertyMap, defaults PropertyMap, outputs PropertyMap, children []URN) *State {
contract.Assert(t != "")
contract.Assert(custom || id == "")
@ -31,6 +32,7 @@ func NewState(t tokens.Type, urn URN, custom bool, id ID,
Type: t,
URN: urn,
Custom: custom,
Delete: del,
ID: id,
Inputs: inputs,
Defaults: defaults,

20
pkg/resource/stack/checkpoint.go
View file

@ -44,24 +44,8 @@ func DeserializeCheckpoint(chkpoint *Checkpoint) (*deploy.Target, *deploy.Snapsh
if latest := chkpoint.Latest; latest != nil {
// For every serialized resource vertex, create a ResourceDeployment out of it.
var resources []*resource.State
if latest.Resources != nil {
for _, kvp := range latest.Resources.Iter() {
// Deserialize the resource properties, if they exist.
res := kvp.Value
inputs := DeserializeProperties(res.Inputs)
defaults := DeserializeProperties(res.Defaults)
outputs := DeserializeProperties(res.Outputs)
var children []resource.URN
for _, child := range res.Children {
children = append(children, resource.URN(child))
}
// And now just produce a resource object using the information available.
resources = append(resources,
resource.NewState(res.Type, kvp.Key, res.Custom, res.ID,
inputs, defaults, outputs, children))
}
for _, res := range latest.Resources {
resources = append(resources, DeserializeResource(res))
}
snap = deploy.NewSnapshot(name, chkpoint.Latest.Time, resources, latest.Info)

191
pkg/resource/stack/deployment.go
View file

@ -3,8 +3,6 @@
package stack
import (
"bytes"
"encoding/json"
"reflect"
"sort"
"time"
@ -21,12 +19,14 @@ import (
type Deployment struct {
Time time.Time `json:"time"` // the time of the deploy.
Info interface{} `json:"info,omitempty"` // optional information about the source.
Resources *Resources `json:"resources,omitempty"` // a map of resource.URNs to resource vertices.
Resources []Resource `json:"resources,omitempty"` // an array of resources.
}
// Resource is a serializable vertex within a LumiGL graph, specifically for resource snapshots.
type Resource struct {
URN resource.URN `json:"urn"` // the URN for this resource.
Custom bool `json:"custom"` // true if a custom resource managed by a plugin.
Delete bool `json:"delete,omitempty"` // true if this resource should be deleted during the next update.
ID resource.ID `json:"id,omitempty"` // the provider ID for this resource, if any.
Type tokens.Type `json:"type"` // this resource's full type token.
Inputs map[string]interface{} `json:"inputs,omitempty"` // the input properties from the program.
@ -38,27 +38,22 @@ type Resource struct {
// SerializeDeployment serializes an entire snapshot as a deploy record.
func SerializeDeployment(snap *deploy.Snapshot) *Deployment {
// Serialize all vertices and only include a vertex section if non-empty.
var resm *Resources
if snapres := snap.Resources; len(snapres) > 0 {
resm = NewResources()
for _, res := range snapres {
urn := res.URN
contract.Assertf(string(urn) != "", "Unexpected empty resource resource.URN")
contract.Assertf(!resm.Has(urn), "Unexpected duplicate resource resource.URN '%v'", urn)
resm.Add(urn, SerializeResource(res))
}
var resources []Resource
for _, res := range snap.Resources {
resources = append(resources, SerializeResource(res))
}
return &Deployment{
Time: snap.Time,
Info: snap.Info,
Resources: resm,
Resources: resources,
}
}
// SerializeResource turns a resource into a LumiGL data structure suitable for serialization.
func SerializeResource(res *resource.State) *Resource {
// SerializeResource turns a resource into a structure suitable for serialization.
func SerializeResource(res *resource.State) Resource {
contract.Assert(res != nil)
contract.Assertf(string(res.URN) != "", "Unexpected empty resource resource.URN")
// Serialize all input and output properties recursively, and add them if non-empty.
var inputs map[string]interface{}
@ -81,8 +76,10 @@ func SerializeResource(res *resource.State) *Resource {
}
sort.Strings(children)
return &Resource{
return Resource{
URN: res.URN,
Custom: res.Custom,
Delete: res.Delete,
ID: res.ID,
Type: res.Type,
Children: children,
@ -139,6 +136,21 @@ func SerializePropertyValue(prop resource.PropertyValue) interface{} {
return prop.V
}
// DeserializeResource turns a serialized resource back into its usual form.
func DeserializeResource(res Resource) *resource.State {
// Deserialize the resource properties, if they exist.
inputs := DeserializeProperties(res.Inputs)
defaults := DeserializeProperties(res.Defaults)
outputs := DeserializeProperties(res.Outputs)
var children []resource.URN
for _, child := range res.Children {
children = append(children, resource.URN(child))
}
return resource.NewState(res.Type, res.URN, res.Custom, res.Delete, res.ID, inputs, defaults, outputs, children)
}
// DeserializeProperties deserializes an entire map of deploy properties into a resource property map.
func DeserializeProperties(props map[string]interface{}) resource.PropertyMap {
result := make(resource.PropertyMap)
@ -182,150 +194,3 @@ func DeserializePropertyValue(v interface{}) resource.PropertyValue {
return resource.NewNullProperty()
}
// Resources is a map of URN to resource, that also preserves a stable order of its keys. This ensures
// enumerations are ordered deterministically, versus Go's built-in map type whose enumeration is randomized.
// Additionally, because of this stable ordering, marshaling to and from JSON also preserves the order of keys.
type Resources struct {
m map[resource.URN]*Resource
keys []resource.URN
}
func NewResources() *Resources {
return &Resources{m: make(map[resource.URN]*Resource)}
}
func (m *Resources) Keys() []resource.URN { return m.keys }
func (m *Resources) Len() int { return len(m.keys) }
func (m *Resources) Add(k resource.URN, v *Resource) {
_, has := m.m[k]
contract.Assertf(!has, "Unexpected duplicate key '%v' added to map")
m.m[k] = v
m.keys = append(m.keys, k)
}
func (m *Resources) Delete(k resource.URN) {
_, has := m.m[k]
contract.Assertf(has, "Unexpected delete of non-existent key key '%v'")
delete(m.m, k)
for i, ek := range m.keys {
if ek == k {
newk := m.keys[:i]
m.keys = append(newk, m.keys[i+1:]...)
break
}
contract.Assertf(i != len(m.keys)-1, "Expected to find deleted key '%v' in map's keys")
}
}
func (m *Resources) Get(k resource.URN) (*Resource, bool) {
v, has := m.m[k]
return v, has
}
func (m *Resources) Has(k resource.URN) bool {
_, has := m.m[k]
return has
}
func (m *Resources) Must(k resource.URN) *Resource {
v, has := m.m[k]
contract.Assertf(has, "Expected key '%v' to exist in this map", k)
return v
}
func (m *Resources) Set(k resource.URN, v *Resource) {
_, has := m.m[k]
contract.Assertf(has, "Expected key '%v' to exist in this map for setting an element", k)
m.m[k] = v
}
func (m *Resources) SetOrAdd(k resource.URN, v *Resource) {
if _, has := m.m[k]; has {
m.Set(k, v)
} else {
m.Add(k, v)
}
}
type ResourceKV struct {
Key resource.URN
Value *Resource
}
// Iter can be used to conveniently range over a map's contents stably.
func (m *Resources) Iter() []ResourceKV {
var kvps []ResourceKV
for _, k := range m.Keys() {
kvps = append(kvps, ResourceKV{k, m.Must(k)})
}
return kvps
}
func (m *Resources) MarshalJSON() ([]byte, error) {
var b bytes.Buffer
b.WriteString("{")
for i, k := range m.Keys() {
if i != 0 {
b.WriteString(",")
}
kb, err := json.Marshal(k)
if err != nil {
return nil, err
}
b.Write(kb)
b.WriteString(":")
vb, err := json.Marshal(m.Must(k))
if err != nil {
return nil, err
}
b.Write(vb)
}
b.WriteString("}")
return b.Bytes(), nil
}
func (m *Resources) UnmarshalJSON(b []byte) error {
contract.Assert(m.m == nil)
m.m = make(map[resource.URN]*Resource)
// Do a pass and read keys and values in the right order.
rdr := bytes.NewReader(b)
dec := json.NewDecoder(rdr)
// First, eat the open object curly '{':
contract.Assert(dec.More())
opencurly, err := dec.Token()
if err != nil {
return err
}
contract.Assert(opencurly.(json.Delim) == '{')
// Parse out every resource key (resource.URN) and element (*Deployment):
for dec.More() {
// See if we've reached the closing '}'; if yes, chew on it and break.
token, err := dec.Token()
if err != nil {
return err
}
if closecurly, isclose := token.(json.Delim); isclose {
contract.Assert(closecurly == '}')
break
}
k := resource.URN(token.(string))
contract.Assert(dec.More())
var v *Resource
if err := dec.Decode(&v); err != nil {
return err
}
contract.Assert(!m.Has(k))
m.Add(k, v)
}
return nil
}

1
pkg/resource/stack/deployment_test.go
View file

@ -22,6 +22,7 @@ func TestDeploymentSerialization(t *testing.T) {
tokens.QName("resource-x"),
),
true,
false,
resource.ID("test-resource-x"),
resource.NewPropertyMapFromMap(map[string]interface{}{
"in-nil": nil,