diff --git a/pkg/engine/lifecycle_test.go b/pkg/engine/lifecycle_test.go index 856f8d110..15f1f9f7c 100644 --- a/pkg/engine/lifecycle_test.go +++ b/pkg/engine/lifecycle_test.go @@ -1029,3 +1029,96 @@ func TestExternalRefresh(t *testing.T) { assert.Equal(t, string(snap.Resources[1].URN.Name()), "resA") assert.True(t, snap.Resources[1].External) } + +func TestRefreshInitFailure(t *testing.T) { + // + // Refresh will persist any initialization errors that are returned by `Read`. This provider + // will error out or not based on the value of `refreshShouldFail`. + // + refreshShouldFail := false + + // + // Set up test environment to use `readFailProvider` as the underlying resource provider. + // + loaders := []*deploytest.ProviderLoader{ + deploytest.NewProviderLoader("pkgA", semver.MustParse("1.0.0"), func() (plugin.Provider, error) { + return &deploytest.Provider{ + ReadF: func( + urn resource.URN, id resource.ID, props resource.PropertyMap, + ) (resource.PropertyMap, resource.Status, error) { + if refreshShouldFail { + err := &plugin.InitError{ + Reasons: []string{"Refresh reports continued to fail to initialize"}, + } + return resource.PropertyMap{}, resource.StatusPartialFailure, err + } + return resource.PropertyMap{}, resource.StatusOK, nil + }, + }, nil + }), + } + + program := deploytest.NewLanguageRuntime(func(_ plugin.RunInfo, monitor *deploytest.ResourceMonitor) error { + _, _, _, err := monitor.RegisterResource("pkgA:m:typA", "resA", true, "", false, nil, "", + resource.PropertyMap{}) + assert.NoError(t, err) + return nil + }) + host := deploytest.NewPluginHost(nil, program, loaders...) + + p := &TestPlan{ + Options: UpdateOptions{host: host}, + } + + provURN := p.NewProviderURN("pkgA", "default", "") + resURN := p.NewURN("pkgA:m:typA", "resA", "") + + // + // Create an old snapshot with a single initialization failure. + // + old := &deploy.Snapshot{ + Resources: []*resource.State{{ + Type: resURN.Type(), + URN: resURN, + Custom: true, + ID: "0", + Inputs: resource.PropertyMap{}, + Outputs: resource.PropertyMap{}, + InitErrors: []string{"Resource failed to initialize"}, + }}, + } + + // + // Refresh DOES NOT fail, causing the initialization error to disappear. + // + p.Steps = []TestStep{{Op: Refresh}} + snap := p.Run(t, old) + + for _, resource := range snap.Resources { + switch urn := resource.URN; urn { + case provURN: + // break + case resURN: + assert.Equal(t, []string{}, resource.InitErrors) + default: + t.Fatalf("unexpected resource %v", urn) + } + } + + // + // Refresh DOES fail, causing the new initialization error to appear. + // + refreshShouldFail = true + p.Steps = []TestStep{{Op: Refresh}} + snap = p.Run(t, old) + for _, resource := range snap.Resources { + switch urn := resource.URN; urn { + case provURN: + // break + case resURN: + assert.Equal(t, []string{"Refresh reports continued to fail to initialize"}, resource.InitErrors) + default: + t.Fatalf("unexpected resource %v", urn) + } + } +} diff --git a/pkg/resource/deploy/deploytest/provider.go b/pkg/resource/deploy/deploytest/provider.go index 06e2828d7..0d520ff25 100644 --- a/pkg/resource/deploy/deploytest/provider.go +++ b/pkg/resource/deploy/deploytest/provider.go @@ -45,7 +45,8 @@ type Provider struct { olds, news resource.PropertyMap) (resource.PropertyMap, resource.Status, error) DeleteF func(urn resource.URN, id resource.ID, olds resource.PropertyMap) (resource.Status, error) - ReadF func(urn resource.URN, id resource.ID, props resource.PropertyMap) (resource.PropertyMap, error) + ReadF func(urn resource.URN, id resource.ID, + props resource.PropertyMap) (resource.PropertyMap, resource.Status, error) InvokeF func(tok tokens.ModuleMember, inputs resource.PropertyMap) (resource.PropertyMap, []plugin.CheckFailure, error) } @@ -129,9 +130,9 @@ func (prov *Provider) Delete(urn resource.URN, } func (prov *Provider) Read(urn resource.URN, id resource.ID, - props resource.PropertyMap) (resource.PropertyMap, error) { + props resource.PropertyMap) (resource.PropertyMap, resource.Status, error) { if prov.ReadF == nil { - return resource.PropertyMap{}, nil + return resource.PropertyMap{}, resource.StatusUnknown, nil } return prov.ReadF(urn, id, props) } diff --git a/pkg/resource/deploy/providers/registry.go b/pkg/resource/deploy/providers/registry.go index 22e9b9856..a77ded003 100644 --- a/pkg/resource/deploy/providers/registry.go +++ b/pkg/resource/deploy/providers/registry.go @@ -339,8 +339,8 @@ func (r *Registry) Delete(urn resource.URN, id resource.ID, props resource.Prope } func (r *Registry) Read(urn resource.URN, id resource.ID, - props resource.PropertyMap) (resource.PropertyMap, error) { - return nil, errors.New("provider resources may not be read") + props resource.PropertyMap) (resource.PropertyMap, resource.Status, error) { + return nil, resource.StatusUnknown, errors.New("provider resources may not be read") } func (r *Registry) Invoke(tok tokens.ModuleMember, diff --git a/pkg/resource/deploy/providers/registry_test.go b/pkg/resource/deploy/providers/registry_test.go index 2632defd5..939da4e23 100644 --- a/pkg/resource/deploy/providers/registry_test.go +++ b/pkg/resource/deploy/providers/registry_test.go @@ -111,8 +111,8 @@ func (prov *testProvider) Create(urn resource.URN, props resource.PropertyMap) ( return "", nil, resource.StatusOK, errors.New("unsupported") } func (prov *testProvider) Read(urn resource.URN, id resource.ID, - props resource.PropertyMap) (resource.PropertyMap, error) { - return nil, errors.New("unsupported") + props resource.PropertyMap) (resource.PropertyMap, resource.Status, error) { + return nil, resource.StatusUnknown, errors.New("unsupported") } func (prov *testProvider) Diff(urn resource.URN, id resource.ID, olds resource.PropertyMap, news resource.PropertyMap, _ bool) (plugin.DiffResult, error) { diff --git a/pkg/resource/deploy/source_eval.go b/pkg/resource/deploy/source_eval.go index ecb20a981..5ab0f54ca 100644 --- a/pkg/resource/deploy/source_eval.go +++ b/pkg/resource/deploy/source_eval.go @@ -257,7 +257,7 @@ func (d *defaultProviders) newRegisterDefaultProviderEvent( // Create the result channel and the event. done := make(chan *RegisterResult) event := ®isterResourceEvent{ - goal: resource.NewGoal(providers.MakeProviderType(pkg), "default", true, inputs, "", false, nil, ""), + goal: resource.NewGoal(providers.MakeProviderType(pkg), "default", true, inputs, "", false, nil, "", nil), done: done, } return event, done, nil @@ -601,7 +601,7 @@ func (rm *resmon) RegisterResource(ctx context.Context, // Send the goal state to the engine. step := ®isterResourceEvent{ - goal: resource.NewGoal(t, name, custom, props, parent, protect, dependencies, provider), + goal: resource.NewGoal(t, name, custom, props, parent, protect, dependencies, provider, nil), done: make(chan *RegisterResult), } diff --git a/pkg/resource/deploy/source_eval_test.go b/pkg/resource/deploy/source_eval_test.go index e985e0627..e8f74e68c 100644 --- a/pkg/resource/deploy/source_eval_test.go +++ b/pkg/resource/deploy/source_eval_test.go @@ -142,16 +142,16 @@ func TestRegisterNoDefaultProviders(t *testing.T) { // Register a component resource. &testRegEvent{ goal: resource.NewGoal(componentURN.Type(), componentURN.Name(), false, resource.PropertyMap{}, "", false, - nil, ""), + nil, "", []string{}), }, // Register a couple resources using provider A. &testRegEvent{ goal: resource.NewGoal("pkgA:index:typA", "res1", true, resource.PropertyMap{}, componentURN, false, nil, - providerARef.String()), + providerARef.String(), []string{}), }, &testRegEvent{ goal: resource.NewGoal("pkgA:index:typA", "res2", true, resource.PropertyMap{}, componentURN, false, nil, - providerARef.String()), + providerARef.String(), []string{}), }, // Register two more providers. newProviderEvent("pkgA", "providerB", nil, ""), @@ -159,11 +159,11 @@ func TestRegisterNoDefaultProviders(t *testing.T) { // Register a few resources that use the new providers. &testRegEvent{ goal: resource.NewGoal("pkgB:index:typB", "res3", true, resource.PropertyMap{}, "", false, nil, - providerBRef.String()), + providerBRef.String(), []string{}), }, &testRegEvent{ goal: resource.NewGoal("pkgB:index:typC", "res4", true, resource.PropertyMap{}, "", false, nil, - providerCRef.String()), + providerCRef.String(), []string{}), }, } @@ -225,21 +225,25 @@ func TestRegisterDefaultProviders(t *testing.T) { // Register a component resource. &testRegEvent{ goal: resource.NewGoal(componentURN.Type(), componentURN.Name(), false, resource.PropertyMap{}, "", false, - nil, ""), + nil, "", []string{}), }, // Register a couple resources from package A. &testRegEvent{ - goal: resource.NewGoal("pkgA:m:typA", "res1", true, resource.PropertyMap{}, componentURN, false, nil, ""), + goal: resource.NewGoal("pkgA:m:typA", "res1", true, resource.PropertyMap{}, + componentURN, false, nil, "", []string{}), }, &testRegEvent{ - goal: resource.NewGoal("pkgA:m:typA", "res2", true, resource.PropertyMap{}, componentURN, false, nil, ""), + goal: resource.NewGoal("pkgA:m:typA", "res2", true, resource.PropertyMap{}, + componentURN, false, nil, "", []string{}), }, // Register a few resources from other packages. &testRegEvent{ - goal: resource.NewGoal("pkgB:m:typB", "res3", true, resource.PropertyMap{}, "", false, nil, ""), + goal: resource.NewGoal("pkgB:m:typB", "res3", true, resource.PropertyMap{}, "", false, + nil, "", []string{}), }, &testRegEvent{ - goal: resource.NewGoal("pkgB:m:typC", "res4", true, resource.PropertyMap{}, "", false, nil, ""), + goal: resource.NewGoal("pkgB:m:typC", "res4", true, resource.PropertyMap{}, "", false, + nil, "", []string{}), }, } diff --git a/pkg/resource/deploy/source_refresh.go b/pkg/resource/deploy/source_refresh.go index 378865c3a..a4318c8bf 100644 --- a/pkg/resource/deploy/source_refresh.go +++ b/pkg/resource/deploy/source_refresh.go @@ -18,7 +18,6 @@ import ( "context" "github.com/pkg/errors" - "github.com/pulumi/pulumi/pkg/resource" "github.com/pulumi/pulumi/pkg/resource/deploy/providers" "github.com/pulumi/pulumi/pkg/resource/plugin" @@ -152,20 +151,29 @@ func (iter *refreshSourceIterator) newRefreshGoal(s *resource.State) (*resource. if !ok { return nil, errors.Errorf("unknown provider '%v' for resource '%v'", s.Provider, s.URN) } - refreshed, err := provider.Read(s.URN, s.ID, s.Outputs) + + initErrorReasons := []string{} + refreshed, resourceStatus, err := provider.Read(s.URN, s.ID, s.Outputs) if err != nil { - return nil, errors.Wrapf(err, "refreshing %s's state", s.URN) + if resourceStatus != resource.StatusPartialFailure { + return nil, errors.Wrapf(err, "refreshing %s's state", s.URN) + } + + // Else it's a `StatusPartialError`. + if initErr, isInitErr := err.(*plugin.InitError); isInitErr { + initErrorReasons = initErr.Reasons + } } else if refreshed == nil { return nil, nil // the resource was deleted. } s = resource.NewState( s.Type, s.URN, s.Custom, s.Delete, s.ID, s.Inputs, refreshed, - s.Parent, s.Protect, s.External, s.Dependencies, s.InitErrors, s.Provider) + s.Parent, s.Protect, s.External, s.Dependencies, initErrorReasons, s.Provider) } // Now just return the actual state as the goal state. - return resource.NewGoal(s.Type, s.URN.Name(), s.Custom, s.Outputs, s.Parent, s.Protect, s.Dependencies, - s.Provider), nil + return resource.NewGoal(s.Type, s.URN.Name(), s.Custom, s.Outputs, s.Parent, s.Protect, + s.Dependencies, s.Provider, s.InitErrors), nil } type refreshSourceEvent struct { diff --git a/pkg/resource/deploy/source_refresh_test.go b/pkg/resource/deploy/source_refresh_test.go index ec5019968..1a7380fe7 100644 --- a/pkg/resource/deploy/source_refresh_test.go +++ b/pkg/resource/deploy/source_refresh_test.go @@ -81,9 +81,9 @@ func TestRefresh(t *testing.T) { reads := int32(0) noopProvider := &deploytest.Provider{ - ReadF: func(resource.URN, resource.ID, resource.PropertyMap) (resource.PropertyMap, error) { + ReadF: func(resource.URN, resource.ID, resource.PropertyMap) (resource.PropertyMap, resource.Status, error) { atomic.AddInt32(&reads, 1) - return resource.PropertyMap{}, nil + return resource.PropertyMap{}, resource.StatusUnknown, nil }, } diff --git a/pkg/resource/deploy/step.go b/pkg/resource/deploy/step.go index dc86e50d1..64318162f 100644 --- a/pkg/resource/deploy/step.go +++ b/pkg/resource/deploy/step.go @@ -496,6 +496,8 @@ func (s *ReadStep) Apply(preview bool) (resource.Status, StepCompleteFunc, error urn := s.new.URN id := s.new.ID + var resourceError error + resourceStatus := resource.StatusOK // Unlike most steps, Read steps run during previews. The only time // we can't run is if the ID we are given is unknown. if id == "" || id == plugin.UnknownStringValue { @@ -506,9 +508,18 @@ func (s *ReadStep) Apply(preview bool) (resource.Status, StepCompleteFunc, error return resource.StatusOK, nil, err } - result, err := prov.Read(urn, id, s.new.Inputs) + result, rst, err := prov.Read(urn, id, s.new.Inputs) if err != nil { - return resource.StatusUnknown, nil, err + if rst != resource.StatusPartialFailure { + return rst, nil, err + } + + resourceError = err + resourceStatus = rst + + if initErr, isInitErr := err.(*plugin.InitError); isInitErr { + s.new.InitErrors = initErr.Reasons + } } s.new.Outputs = result @@ -521,7 +532,10 @@ func (s *ReadStep) Apply(preview bool) (resource.Status, StepCompleteFunc, error } complete := func() { s.event.Done(&ReadResult{State: s.new}) } - return resource.StatusOK, complete, nil + if resourceError == nil { + return resourceStatus, complete, nil + } + return resourceStatus, complete, resourceError } // StepOp represents the kind of operation performed by a step. It evaluates to its string label. diff --git a/pkg/resource/deploy/step_generator.go b/pkg/resource/deploy/step_generator.go index 277ff0b54..9234c895a 100644 --- a/pkg/resource/deploy/step_generator.go +++ b/pkg/resource/deploy/step_generator.go @@ -502,7 +502,7 @@ func (sg *stepGenerator) getResourcePropertyStates(urn resource.URN, goal *resou } return props, inputs, outputs, resource.NewState(goal.Type, urn, goal.Custom, false, "", - inputs, outputs, goal.Parent, goal.Protect, false, goal.Dependencies, nil, goal.Provider) + inputs, outputs, goal.Parent, goal.Protect, false, goal.Dependencies, goal.InitErrors, goal.Provider) } // issueCheckErrors prints any check errors to the diagnostics sink. diff --git a/pkg/resource/plugin/provider.go b/pkg/resource/plugin/provider.go index 4667bfebf..52c5b2608 100644 --- a/pkg/resource/plugin/provider.go +++ b/pkg/resource/plugin/provider.go @@ -57,7 +57,8 @@ type Provider interface { // Read the current live state associated with a resource. Enough state must be include in the inputs to uniquely // identify the resource; this is typically just the resource ID, but may also include some properties. If the // resource is missing (for instance, because it has been deleted), the resulting property map will be nil. - Read(urn resource.URN, id resource.ID, props resource.PropertyMap) (resource.PropertyMap, error) + Read(urn resource.URN, id resource.ID, + props resource.PropertyMap) (resource.PropertyMap, resource.Status, error) // Update updates an existing resource with new values. Update(urn resource.URN, id resource.ID, olds resource.PropertyMap, news resource.PropertyMap) (resource.PropertyMap, resource.Status, error) diff --git a/pkg/resource/plugin/provider_plugin.go b/pkg/resource/plugin/provider_plugin.go index d0c5fd6e8..2c524327e 100644 --- a/pkg/resource/plugin/provider_plugin.go +++ b/pkg/resource/plugin/provider_plugin.go @@ -365,7 +365,7 @@ func (p *provider) Create(urn resource.URN, props resource.PropertyMap) (resourc resourceStatus, id, liveObject, resourceError = parseError(err) logging.V(7).Infof("%s failed: %v", label, resourceError) - if resourceStatus == resource.StatusUnknown { + if resourceStatus != resource.StatusPartialFailure { return "", nil, resourceStatus, resourceError } // Else it's a `StatusPartialFailure`. @@ -394,7 +394,9 @@ func (p *provider) Create(urn resource.URN, props resource.PropertyMap) (resourc // read the current live state associated with a resource. enough state must be include in the inputs to uniquely // identify the resource; this is typically just the resource id, but may also include some properties. -func (p *provider) Read(urn resource.URN, id resource.ID, props resource.PropertyMap) (resource.PropertyMap, error) { +func (p *provider) Read( + urn resource.URN, id resource.ID, props resource.PropertyMap, +) (resource.PropertyMap, resource.Status, error) { contract.Assert(urn != "") contract.Assert(id != "") @@ -404,49 +406,60 @@ func (p *provider) Read(urn resource.URN, id resource.ID, props resource.Propert // Get the RPC client and ensure it's configured. client, err := p.getClient() if err != nil { - return nil, err + return nil, resource.StatusUnknown, err } // If the provider is not fully configured, return an empty bag. if !p.cfgknown { - return resource.PropertyMap{}, nil + return resource.PropertyMap{}, resource.StatusUnknown, nil } // Marshal the input state so we can perform the RPC. marshaled, err := MarshalProperties(props, MarshalOptions{Label: label, ElideAssetContents: true}) if err != nil { - return nil, err + return nil, resource.StatusUnknown, err } // Now issue the read request over RPC, blocking until it finished. + var readID resource.ID + var liveObject *_struct.Struct + var resourceError error + var resourceStatus = resource.StatusOK resp, err := client.Read(p.ctx.Request(), &pulumirpc.ReadRequest{ Id: string(id), Urn: string(urn), Properties: marshaled, }) if err != nil { + resourceStatus, readID, liveObject, resourceError = parseError(err) logging.V(7).Infof("%s failed: %v", label, err) - return nil, err + + if resourceStatus != resource.StatusPartialFailure { + return nil, resourceStatus, resourceError + } + // Else it's a `StatusPartialFailure`. + } else { + id = resource.ID(resp.GetId()) + liveObject = resp.GetProperties() } // If the resource was missing, simply return a nil property map. - readID := resp.GetId() - if readID == "" { - return nil, nil - } else if readID != string(id) { - return nil, errors.Errorf( + if string(readID) == "" { + return nil, resourceStatus, nil + } else if readID != id { + return nil, resourceStatus, errors.Errorf( "reading resource %s yielded an unexpected ID; expected %s, got %s", urn, id, readID) } // Finally, unmarshal the resulting state properties and return them. - results, err := UnmarshalProperties(resp.GetProperties(), MarshalOptions{ + results, err := UnmarshalProperties(liveObject, MarshalOptions{ Label: fmt.Sprintf("%s.outputs", label), RejectUnknowns: true}) if err != nil { - return nil, err + return nil, resourceStatus, err } logging.V(7).Infof("%s success; #outs=%d", label, len(results)) - return results, nil + return results, resourceStatus, resourceError } // Update updates an existing resource with new values. @@ -492,7 +505,7 @@ func (p *provider) Update(urn resource.URN, id resource.ID, resourceStatus, _, liveObject, resourceError = parseError(err) logging.V(7).Infof("%s failed: %v", label, resourceError) - if resourceStatus == resource.StatusUnknown { + if resourceStatus != resource.StatusPartialFailure { return nil, resourceStatus, resourceError } // Else it's a `StatusPartialFailure`. diff --git a/pkg/resource/resource_goal.go b/pkg/resource/resource_goal.go index 441aa587d..a8bf66d14 100644 --- a/pkg/resource/resource_goal.go +++ b/pkg/resource/resource_goal.go @@ -29,11 +29,12 @@ type Goal struct { Protect bool // true to protect this resource from deletion. Dependencies []URN // dependencies of this resource object. Provider string // the provider to use for this resource. + InitErrors []string // errors encountered as we attempted to initialize the resource. } // NewGoal allocates a new resource goal state. func NewGoal(t tokens.Type, name tokens.QName, custom bool, props PropertyMap, - parent URN, protect bool, dependencies []URN, provider string) *Goal { + parent URN, protect bool, dependencies []URN, provider string, initErrors []string) *Goal { return &Goal{ Type: t, Name: name, @@ -43,5 +44,6 @@ func NewGoal(t tokens.Type, name tokens.QName, custom bool, props PropertyMap, Protect: protect, Dependencies: dependencies, Provider: provider, + InitErrors: initErrors, } }