Implement partial Read

Some time ago, we introduced the concept of the initialization error to
Pulumi (i.e., an error where the resource was successfully created but
failed to fully initialize). This was originally implemented in `Create`
and `Update`  methods of the resource provider interface; when we
detected an initialization failure, we'd pack the live version of the
object into the error, and return that to the engine.

Omitted from this initial implementation was a similar semantics for
`Read`. There are many implications of this, but one of them is that a
`pulumi refresh` will erase any initialization errors that had
previously been observed, even if the initialization errors still exist
in the resource.

This commit will introduce the initialization error semantics to `Read`,
fixing this issue.

pkg/engine/lifecycle_test.go

@@ -1029,3 +1029,96 @@ func TestExternalRefresh(t *testing.T) {
 	assert.Equal(t, string(snap.Resources[1].URN.Name()), "resA")
 	assert.True(t, snap.Resources[1].External)
 }
+
+func TestRefreshInitFailure(t *testing.T) {
+	//
+	// Refresh will persist any initialization errors that are returned by `Read`. This provider
+	// will error out or not based on the value of `refreshShouldFail`.
+	//
+	refreshShouldFail := false
+
+	//
+	// Set up test environment to use `readFailProvider` as the underlying resource provider.
+	//
+	loaders := []*deploytest.ProviderLoader{
+		deploytest.NewProviderLoader("pkgA", semver.MustParse("1.0.0"), func() (plugin.Provider, error) {
+			return &deploytest.Provider{
+				ReadF: func(
+					urn resource.URN, id resource.ID, props resource.PropertyMap,
+				) (resource.PropertyMap, resource.Status, error) {
+					if refreshShouldFail {
+						err := &plugin.InitError{
+							Reasons: []string{"Refresh reports continued to fail to initialize"},
+						}
+						return resource.PropertyMap{}, resource.StatusPartialFailure, err
+					}
+					return resource.PropertyMap{}, resource.StatusOK, nil
+				},
+			}, nil
+		}),
+	}
+
+	program := deploytest.NewLanguageRuntime(func(_ plugin.RunInfo, monitor *deploytest.ResourceMonitor) error {
+		_, _, _, err := monitor.RegisterResource("pkgA:m:typA", "resA", true, "", false, nil, "",
+			resource.PropertyMap{})
+		assert.NoError(t, err)
+		return nil
+	})
+	host := deploytest.NewPluginHost(nil, program, loaders...)
+
+	p := &TestPlan{
+		Options: UpdateOptions{host: host},
+	}
+
+	provURN := p.NewProviderURN("pkgA", "default", "")
+	resURN := p.NewURN("pkgA:m:typA", "resA", "")
+
+	//
+	// Create an old snapshot with a single initialization failure.
+	//
+	old := &deploy.Snapshot{
+		Resources: []*resource.State{{
+			Type:       resURN.Type(),
+			URN:        resURN,
+			Custom:     true,
+			ID:         "0",
+			Inputs:     resource.PropertyMap{},
+			Outputs:    resource.PropertyMap{},
+			InitErrors: []string{"Resource failed to initialize"},
+		}},
+	}
+
+	//
+	// Refresh DOES NOT fail, causing the initialization error to disappear.
+	//
+	p.Steps = []TestStep{{Op: Refresh}}
+	snap := p.Run(t, old)
+
+	for _, resource := range snap.Resources {
+		switch urn := resource.URN; urn {
+		case provURN:
+			// break
+		case resURN:
+			assert.Equal(t, []string{}, resource.InitErrors)
+		default:
+			t.Fatalf("unexpected resource %v", urn)
+		}
+	}
+
+	//
+	// Refresh DOES fail, causing the new initialization error to appear.
+	//
+	refreshShouldFail = true
+	p.Steps = []TestStep{{Op: Refresh}}
+	snap = p.Run(t, old)
+	for _, resource := range snap.Resources {
+		switch urn := resource.URN; urn {
+		case provURN:
+			// break
+		case resURN:
+			assert.Equal(t, []string{"Refresh reports continued to fail to initialize"}, resource.InitErrors)
+		default:
+			t.Fatalf("unexpected resource %v", urn)
+		}
+	}
+}

pkg/resource/deploy/deploytest/provider.go

@@ -45,7 +45,8 @@ type Provider struct {
 		olds, news resource.PropertyMap) (resource.PropertyMap, resource.Status, error)
 	DeleteF func(urn resource.URN, id resource.ID, olds resource.PropertyMap) (resource.Status, error)
 
-	ReadF   func(urn resource.URN, id resource.ID, props resource.PropertyMap) (resource.PropertyMap, error)
+	ReadF func(urn resource.URN, id resource.ID,
+		props resource.PropertyMap) (resource.PropertyMap, resource.Status, error)
 	InvokeF func(tok tokens.ModuleMember,
 		inputs resource.PropertyMap) (resource.PropertyMap, []plugin.CheckFailure, error)
 }
@@ -129,9 +130,9 @@ func (prov *Provider) Delete(urn resource.URN,
 }
 
 func (prov *Provider) Read(urn resource.URN, id resource.ID,
-	props resource.PropertyMap) (resource.PropertyMap, error) {
+	props resource.PropertyMap) (resource.PropertyMap, resource.Status, error) {
 	if prov.ReadF == nil {
-		return resource.PropertyMap{}, nil
+		return resource.PropertyMap{}, resource.StatusUnknown, nil
 	}
 	return prov.ReadF(urn, id, props)
 }

pkg/resource/deploy/providers/registry.go

@@ -339,8 +339,8 @@ func (r *Registry) Delete(urn resource.URN, id resource.ID, props resource.Prope
 }
 
 func (r *Registry) Read(urn resource.URN, id resource.ID,
-	props resource.PropertyMap) (resource.PropertyMap, error) {
+	props resource.PropertyMap) (resource.PropertyMap, resource.Status, error) {
-	return nil, errors.New("provider resources may not be read")
+	return nil, resource.StatusUnknown, errors.New("provider resources may not be read")
 }
 
 func (r *Registry) Invoke(tok tokens.ModuleMember,

pkg/resource/deploy/providers/registry_test.go

@@ -111,8 +111,8 @@ func (prov *testProvider) Create(urn resource.URN, props resource.PropertyMap) (
 	return "", nil, resource.StatusOK, errors.New("unsupported")
 }
 func (prov *testProvider) Read(urn resource.URN, id resource.ID,
-	props resource.PropertyMap) (resource.PropertyMap, error) {
+	props resource.PropertyMap) (resource.PropertyMap, resource.Status, error) {
-	return nil, errors.New("unsupported")
+	return nil, resource.StatusUnknown, errors.New("unsupported")
 }
 func (prov *testProvider) Diff(urn resource.URN, id resource.ID,
 	olds resource.PropertyMap, news resource.PropertyMap, _ bool) (plugin.DiffResult, error) {

pkg/resource/deploy/source_eval.go

@@ -257,7 +257,7 @@ func (d *defaultProviders) newRegisterDefaultProviderEvent(
 	// Create the result channel and the event.
 	done := make(chan *RegisterResult)
 	event := &registerResourceEvent{
-		goal: resource.NewGoal(providers.MakeProviderType(pkg), "default", true, inputs, "", false, nil, ""),
+		goal: resource.NewGoal(providers.MakeProviderType(pkg), "default", true, inputs, "", false, nil, "", nil),
 		done: done,
 	}
 	return event, done, nil
@@ -601,7 +601,7 @@ func (rm *resmon) RegisterResource(ctx context.Context,
 
 	// Send the goal state to the engine.
 	step := &registerResourceEvent{
-		goal: resource.NewGoal(t, name, custom, props, parent, protect, dependencies, provider),
+		goal: resource.NewGoal(t, name, custom, props, parent, protect, dependencies, provider, nil),
 		done: make(chan *RegisterResult),
 	}
 

pkg/resource/deploy/source_eval_test.go

@@ -142,16 +142,16 @@ func TestRegisterNoDefaultProviders(t *testing.T) {
 		// Register a component resource.
 		&testRegEvent{
 			goal: resource.NewGoal(componentURN.Type(), componentURN.Name(), false, resource.PropertyMap{}, "", false,
-				nil, ""),
+				nil, "", []string{}),
 		},
 		// Register a couple resources using provider A.
 		&testRegEvent{
 			goal: resource.NewGoal("pkgA:index:typA", "res1", true, resource.PropertyMap{}, componentURN, false, nil,
-				providerARef.String()),
+				providerARef.String(), []string{}),
 		},
 		&testRegEvent{
 			goal: resource.NewGoal("pkgA:index:typA", "res2", true, resource.PropertyMap{}, componentURN, false, nil,
-				providerARef.String()),
+				providerARef.String(), []string{}),
 		},
 		// Register two more providers.
 		newProviderEvent("pkgA", "providerB", nil, ""),
@@ -159,11 +159,11 @@ func TestRegisterNoDefaultProviders(t *testing.T) {
 		// Register a few resources that use the new providers.
 		&testRegEvent{
 			goal: resource.NewGoal("pkgB:index:typB", "res3", true, resource.PropertyMap{}, "", false, nil,
-				providerBRef.String()),
+				providerBRef.String(), []string{}),
 		},
 		&testRegEvent{
 			goal: resource.NewGoal("pkgB:index:typC", "res4", true, resource.PropertyMap{}, "", false, nil,
-				providerCRef.String()),
+				providerCRef.String(), []string{}),
 		},
 	}
 
@@ -225,21 +225,25 @@ func TestRegisterDefaultProviders(t *testing.T) {
 		// Register a component resource.
 		&testRegEvent{
 			goal: resource.NewGoal(componentURN.Type(), componentURN.Name(), false, resource.PropertyMap{}, "", false,
-				nil, ""),
+				nil, "", []string{}),
 		},
 		// Register a couple resources from package A.
 		&testRegEvent{
-			goal: resource.NewGoal("pkgA:m:typA", "res1", true, resource.PropertyMap{}, componentURN, false, nil, ""),
+			goal: resource.NewGoal("pkgA:m:typA", "res1", true, resource.PropertyMap{},
+				componentURN, false, nil, "", []string{}),
 		},
 		&testRegEvent{
-			goal: resource.NewGoal("pkgA:m:typA", "res2", true, resource.PropertyMap{}, componentURN, false, nil, ""),
+			goal: resource.NewGoal("pkgA:m:typA", "res2", true, resource.PropertyMap{},
+				componentURN, false, nil, "", []string{}),
 		},
 		// Register a few resources from other packages.
 		&testRegEvent{
-			goal: resource.NewGoal("pkgB:m:typB", "res3", true, resource.PropertyMap{}, "", false, nil, ""),
+			goal: resource.NewGoal("pkgB:m:typB", "res3", true, resource.PropertyMap{}, "", false,
+				nil, "", []string{}),
 		},
 		&testRegEvent{
-			goal: resource.NewGoal("pkgB:m:typC", "res4", true, resource.PropertyMap{}, "", false, nil, ""),
+			goal: resource.NewGoal("pkgB:m:typC", "res4", true, resource.PropertyMap{}, "", false,
+				nil, "", []string{}),
 		},
 	}
 

pkg/resource/deploy/source_refresh.go

@@ -18,7 +18,6 @@ import (
 	"context"
 
 	"github.com/pkg/errors"
-
 	"github.com/pulumi/pulumi/pkg/resource"
 	"github.com/pulumi/pulumi/pkg/resource/deploy/providers"
 	"github.com/pulumi/pulumi/pkg/resource/plugin"
@@ -152,20 +151,29 @@ func (iter *refreshSourceIterator) newRefreshGoal(s *resource.State) (*resource.
 		if !ok {
 			return nil, errors.Errorf("unknown provider '%v' for resource '%v'", s.Provider, s.URN)
 		}
-		refreshed, err := provider.Read(s.URN, s.ID, s.Outputs)
+
+		initErrorReasons := []string{}
+		refreshed, resourceStatus, err := provider.Read(s.URN, s.ID, s.Outputs)
 		if err != nil {
-			return nil, errors.Wrapf(err, "refreshing %s's state", s.URN)
+			if resourceStatus != resource.StatusPartialFailure {
+				return nil, errors.Wrapf(err, "refreshing %s's state", s.URN)
+			}
+
+			// Else it's a `StatusPartialError`.
+			if initErr, isInitErr := err.(*plugin.InitError); isInitErr {
+				initErrorReasons = initErr.Reasons
+			}
 		} else if refreshed == nil {
 			return nil, nil // the resource was deleted.
 		}
 		s = resource.NewState(
 			s.Type, s.URN, s.Custom, s.Delete, s.ID, s.Inputs, refreshed,
-			s.Parent, s.Protect, s.External, s.Dependencies, s.InitErrors, s.Provider)
+			s.Parent, s.Protect, s.External, s.Dependencies, initErrorReasons, s.Provider)
 	}
 
 	// Now just return the actual state as the goal state.
-	return resource.NewGoal(s.Type, s.URN.Name(), s.Custom, s.Outputs, s.Parent, s.Protect, s.Dependencies,
+	return resource.NewGoal(s.Type, s.URN.Name(), s.Custom, s.Outputs, s.Parent, s.Protect,
-		s.Provider), nil
+		s.Dependencies, s.Provider, s.InitErrors), nil
 }
 
 type refreshSourceEvent struct {

pkg/resource/deploy/source_refresh_test.go

@@ -81,9 +81,9 @@ func TestRefresh(t *testing.T) {
 
 	reads := int32(0)
 	noopProvider := &deploytest.Provider{
-		ReadF: func(resource.URN, resource.ID, resource.PropertyMap) (resource.PropertyMap, error) {
+		ReadF: func(resource.URN, resource.ID, resource.PropertyMap) (resource.PropertyMap, resource.Status, error) {
 			atomic.AddInt32(&reads, 1)
-			return resource.PropertyMap{}, nil
+			return resource.PropertyMap{}, resource.StatusUnknown, nil
 		},
 	}
 

pkg/resource/deploy/step.go

@@ -496,6 +496,8 @@ func (s *ReadStep) Apply(preview bool) (resource.Status, StepCompleteFunc, error
 	urn := s.new.URN
 	id := s.new.ID
 
+	var resourceError error
+	resourceStatus := resource.StatusOK
 	// Unlike most steps, Read steps run during previews. The only time
 	// we can't run is if the ID we are given is unknown.
 	if id == "" || id == plugin.UnknownStringValue {
@@ -506,9 +508,18 @@ func (s *ReadStep) Apply(preview bool) (resource.Status, StepCompleteFunc, error
 			return resource.StatusOK, nil, err
 		}
 
-		result, err := prov.Read(urn, id, s.new.Inputs)
+		result, rst, err := prov.Read(urn, id, s.new.Inputs)
 		if err != nil {
-			return resource.StatusUnknown, nil, err
+			if rst != resource.StatusPartialFailure {
+				return rst, nil, err
+			}
+
+			resourceError = err
+			resourceStatus = rst
+
+			if initErr, isInitErr := err.(*plugin.InitError); isInitErr {
+				s.new.InitErrors = initErr.Reasons
+			}
 		}
 
 		s.new.Outputs = result
@@ -521,7 +532,10 @@ func (s *ReadStep) Apply(preview bool) (resource.Status, StepCompleteFunc, error
 	}
 
 	complete := func() { s.event.Done(&ReadResult{State: s.new}) }
-	return resource.StatusOK, complete, nil
+	if resourceError == nil {
+		return resourceStatus, complete, nil
+	}
+	return resourceStatus, complete, resourceError
 }
 
 // StepOp represents the kind of operation performed by a step.  It evaluates to its string label.

pkg/resource/deploy/step_generator.go

@@ -502,7 +502,7 @@ func (sg *stepGenerator) getResourcePropertyStates(urn resource.URN, goal *resou
 	}
 	return props, inputs, outputs,
 		resource.NewState(goal.Type, urn, goal.Custom, false, "",
-			inputs, outputs, goal.Parent, goal.Protect, false, goal.Dependencies, nil, goal.Provider)
+			inputs, outputs, goal.Parent, goal.Protect, false, goal.Dependencies, goal.InitErrors, goal.Provider)
 }
 
 // issueCheckErrors prints any check errors to the diagnostics sink.

pkg/resource/plugin/provider.go

@@ -57,7 +57,8 @@ type Provider interface {
 	// Read the current live state associated with a resource.  Enough state must be include in the inputs to uniquely
 	// identify the resource; this is typically just the resource ID, but may also include some properties.  If the
 	// resource is missing (for instance, because it has been deleted), the resulting property map will be nil.
-	Read(urn resource.URN, id resource.ID, props resource.PropertyMap) (resource.PropertyMap, error)
+	Read(urn resource.URN, id resource.ID,
+		props resource.PropertyMap) (resource.PropertyMap, resource.Status, error)
 	// Update updates an existing resource with new values.
 	Update(urn resource.URN, id resource.ID,
 		olds resource.PropertyMap, news resource.PropertyMap) (resource.PropertyMap, resource.Status, error)

pkg/resource/plugin/provider_plugin.go

@@ -365,7 +365,7 @@ func (p *provider) Create(urn resource.URN, props resource.PropertyMap) (resourc
 		resourceStatus, id, liveObject, resourceError = parseError(err)
 		logging.V(7).Infof("%s failed: %v", label, resourceError)
 
-		if resourceStatus == resource.StatusUnknown {
+		if resourceStatus != resource.StatusPartialFailure {
 			return "", nil, resourceStatus, resourceError
 		}
 		// Else it's a `StatusPartialFailure`.
@@ -394,7 +394,9 @@ func (p *provider) Create(urn resource.URN, props resource.PropertyMap) (resourc
 
 // read the current live state associated with a resource.  enough state must be include in the inputs to uniquely
 // identify the resource; this is typically just the resource id, but may also include some properties.
-func (p *provider) Read(urn resource.URN, id resource.ID, props resource.PropertyMap) (resource.PropertyMap, error) {
+func (p *provider) Read(
+	urn resource.URN, id resource.ID, props resource.PropertyMap,
+) (resource.PropertyMap, resource.Status, error) {
 	contract.Assert(urn != "")
 	contract.Assert(id != "")
 
@@ -404,49 +406,60 @@ func (p *provider) Read(urn resource.URN, id resource.ID, props resource.Propert
 	// Get the RPC client and ensure it's configured.
 	client, err := p.getClient()
 	if err != nil {
-		return nil, err
+		return nil, resource.StatusUnknown, err
 	}
 
 	// If the provider is not fully configured, return an empty bag.
 	if !p.cfgknown {
-		return resource.PropertyMap{}, nil
+		return resource.PropertyMap{}, resource.StatusUnknown, nil
 	}
 
 	// Marshal the input state so we can perform the RPC.
 	marshaled, err := MarshalProperties(props, MarshalOptions{Label: label, ElideAssetContents: true})
 	if err != nil {
-		return nil, err
+		return nil, resource.StatusUnknown, err
 	}
 
 	// Now issue the read request over RPC, blocking until it finished.
+	var readID resource.ID
+	var liveObject *_struct.Struct
+	var resourceError error
+	var resourceStatus = resource.StatusOK
 	resp, err := client.Read(p.ctx.Request(), &pulumirpc.ReadRequest{
 		Id:         string(id),
 		Urn:        string(urn),
 		Properties: marshaled,
 	})
 	if err != nil {
+		resourceStatus, readID, liveObject, resourceError = parseError(err)
 		logging.V(7).Infof("%s failed: %v", label, err)
-		return nil, err
+
+		if resourceStatus != resource.StatusPartialFailure {
+			return nil, resourceStatus, resourceError
+		}
+		// Else it's a `StatusPartialFailure`.
+	} else {
+		id = resource.ID(resp.GetId())
+		liveObject = resp.GetProperties()
 	}
 
 	// If the resource was missing, simply return a nil property map.
-	readID := resp.GetId()
+	if string(readID) == "" {
-	if readID == "" {
+		return nil, resourceStatus, nil
-		return nil, nil
+	} else if readID != id {
-	} else if readID != string(id) {
+		return nil, resourceStatus, errors.Errorf(
-		return nil, errors.Errorf(
 			"reading resource %s yielded an unexpected ID; expected %s, got %s", urn, id, readID)
 	}
 
 	// Finally, unmarshal the resulting state properties and return them.
-	results, err := UnmarshalProperties(resp.GetProperties(), MarshalOptions{
+	results, err := UnmarshalProperties(liveObject, MarshalOptions{
 		Label: fmt.Sprintf("%s.outputs", label), RejectUnknowns: true})
 	if err != nil {
-		return nil, err
+		return nil, resourceStatus, err
 	}
 
 	logging.V(7).Infof("%s success; #outs=%d", label, len(results))
-	return results, nil
+	return results, resourceStatus, resourceError
 }
 
 // Update updates an existing resource with new values.
@@ -492,7 +505,7 @@ func (p *provider) Update(urn resource.URN, id resource.ID,
 		resourceStatus, _, liveObject, resourceError = parseError(err)
 		logging.V(7).Infof("%s failed: %v", label, resourceError)
 
-		if resourceStatus == resource.StatusUnknown {
+		if resourceStatus != resource.StatusPartialFailure {
 			return nil, resourceStatus, resourceError
 		}
 		// Else it's a `StatusPartialFailure`.

pkg/resource/resource_goal.go

@@ -29,11 +29,12 @@ type Goal struct {
 	Protect      bool         // true to protect this resource from deletion.
 	Dependencies []URN        // dependencies of this resource object.
 	Provider     string       // the provider to use for this resource.
+	InitErrors   []string     // errors encountered as we attempted to initialize the resource.
 }
 
 // NewGoal allocates a new resource goal state.
 func NewGoal(t tokens.Type, name tokens.QName, custom bool, props PropertyMap,
-	parent URN, protect bool, dependencies []URN, provider string) *Goal {
+	parent URN, protect bool, dependencies []URN, provider string, initErrors []string) *Goal {
 	return &Goal{
 		Type:         t,
 		Name:         name,
@@ -43,5 +44,6 @@ func NewGoal(t tokens.Type, name tokens.QName, custom bool, props PropertyMap,
 		Protect:      protect,
 		Dependencies: dependencies,
 		Provider:     provider,
+		InitErrors:   initErrors,
 	}
 }