Warn instead of error when refresh'd resource is unhealthy

Fixes #2633.

Currently when a user runs `refresh` and a resource is in a state of
error, the `refresh` will fail and the resource state will not be
persisted. This can make it vastly harder to incrementally fix
infrastructure. The issue mentioned above explains more of the
historical context, as well as some specific failure modes.

This commit resolves this issue by causing refresh to *not* report an
error in this case, and instead to simply log a warning that the
`refresh` has recognized that the resource is in an unhealthy state
during state sync.

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ### Improvements
 
+- `refresh` will now warn instead of returning an error when it notices a resource is in an
+  unhealthy state. This is in service of https://github.com/pulumi/pulumi/issues/2633.
+
 ## 0.17.5 (Released April 8, 2019)
 
 ### Improvements

pkg/engine/lifecycle_test.go

@@ -1372,10 +1372,11 @@ func TestRefreshInitFailure(t *testing.T) {
 	}
 
 	//
-	// Refresh DOES fail, causing the new initialization error to appear.
+	// Refresh again, see the resource is in a partial state of failure, but the refresh operation
+	// DOES NOT fail. The initialization error is still persisted.
 	//
 	refreshShouldFail = true
-	p.Steps = []TestStep{{Op: Refresh, SkipPreview: true, ExpectFailure: true}}
+	p.Steps = []TestStep{{Op: Refresh, SkipPreview: true}}
 	snap = p.Run(t, old)
 	for _, resource := range snap.Resources {
 		switch urn := resource.URN; urn {

pkg/resource/deploy/step.go

@@ -15,7 +15,11 @@
 package deploy
 
 import (
+	"fmt"
+	"strings"
+
 	"github.com/pkg/errors"
+	"github.com/pulumi/pulumi/pkg/diag"
 
 	"github.com/pulumi/pulumi/pkg/diag/colors"
 	"github.com/pulumi/pulumi/pkg/resource"
@@ -667,6 +671,15 @@ func (s *RefreshStep) Apply(preview bool) (resource.Status, StepCompleteFunc, er
 		}
 		if initErr, isInitErr := err.(*plugin.InitError); isInitErr {
 			initErrors = initErr.Reasons
+
+			// Partial failure SHOULD NOT cause refresh to fail. Instead:
+			//
+			// 1. Warn instead that during refresh we noticed the resource has become unhealthy.
+			// 2. Make sure the initialization errors are persisted in the state, so that the next
+			//    `pulumi up` will surface them to the user.
+			err = nil
+			msg := fmt.Sprintf("Refreshed resource is in an unhealthy state:\n* %s", strings.Join(initErrors, "\n* "))
+			s.Plan().Diag().Warningf(diag.RawMessage(s.URN(), msg))
 		}
 	}
 	outputs := refreshed.Outputs