Retry some HTTP operations

We've seen failures in CI where DNS lookups fail which cause our
operations against the service to fail, as well as other sorts of
timeouts.

Add a set of helper methods in a new httputil package that helps us do
retries on these operations, and then update our client library to use
them when we are doing GET requests. We also provide a way for non GET
requests to be retried, and use this when updating a lease (since it
is safe to retry multiple requests in this case).

pkg/backend/cloud/client/api.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/pulumi/pulumi/pkg/apitype"
 	"github.com/pulumi/pulumi/pkg/util/contract"
+	"github.com/pulumi/pulumi/pkg/util/httputil"
 	"github.com/pulumi/pulumi/pkg/version"
 )
 
@@ -69,6 +70,11 @@ type accessToken interface {
 	String() string
 }
 
+type httpCallOptions struct {
+	// RetryAllMethods allows non-GET calls to be retried if the server fails to return a response.
+	RetryAllMethods bool
+}
+
 // apiAccessToken is an implementation of accessToken for Pulumi API tokens (i.e. tokens of kind
 // accessTokenKindAPIToken)
 type apiAccessToken string
@@ -94,7 +100,8 @@ func (t updateAccessToken) String() string {
 }
 
 // pulumiAPICall makes an HTTP request to the Pulumi API.
-func pulumiAPICall(cloudAPI, method, path string, body []byte, tok accessToken) (string, *http.Response, error) {
+func pulumiAPICall(cloudAPI, method, path string, body []byte, tok accessToken,
+	opts httpCallOptions) (string, *http.Response, error) {
 	// Normalize URL components
 	cloudAPI = strings.TrimSuffix(cloudAPI, "/")
 	path = strings.TrimPrefix(path, "/")
@@ -121,7 +128,13 @@ func pulumiAPICall(cloudAPI, method, path string, body []byte, tok accessToken)
 		glog.V(9).Infof("Pulumi API call details (%s): headers=%v; body=%v", url, req.Header, string(body))
 	}
 
-	resp, err := http.DefaultClient.Do(req)
+	var resp *http.Response
+	if req.Method == "GET" || opts.RetryAllMethods {
+		resp, err = httputil.DoWithRetry(req, http.DefaultClient)
+	} else {
+		resp, err = http.DefaultClient.Do(req)
+	}
+
 	if err != nil {
 		return "", nil, errors.Wrapf(err, "performing HTTP request")
 	}
@@ -159,7 +172,8 @@ func pulumiAPICall(cloudAPI, method, path string, body []byte, tok accessToken)
 // the request body (use nil for GETs), and if successful, marshalling the responseObj
 // as JSON and storing it in respObj (use nil for NoContent). The error return type might
 // be an instance of apitype.ErrorResponse, in which case will have the response code.
-func pulumiRESTCall(cloudAPI, method, path string, queryObj, reqObj, respObj interface{}, tok accessToken) error {
+func pulumiRESTCall(cloudAPI, method, path string, queryObj, reqObj, respObj interface{}, tok accessToken,
+	opts httpCallOptions) error {
 	// Compute query string from query object
 	querystring := ""
 	if queryObj != nil {
@@ -184,7 +198,7 @@ func pulumiRESTCall(cloudAPI, method, path string, queryObj, reqObj, respObj int
 	}
 
 	// Make API call
-	url, resp, err := pulumiAPICall(cloudAPI, method, path+querystring, reqBody, tok)
+	url, resp, err := pulumiAPICall(cloudAPI, method, path+querystring, reqBody, tok, opts)
 	if err != nil {
 		return err
 	}

pkg/backend/cloud/client/client.go

@@ -40,13 +40,20 @@ func NewClient(apiURL, apiToken string) *Client {
 
 // apiCall makes a raw HTTP request to the Pulumi API using the given method, path, and request body.
 func (pc *Client) apiCall(method, path string, body []byte) (string, *http.Response, error) {
-	return pulumiAPICall(pc.apiURL, method, path, body, pc.apiToken)
+	return pulumiAPICall(pc.apiURL, method, path, body, pc.apiToken, httpCallOptions{})
 }
 
 // restCall makes a REST-style request to the Pulumi API using the given method, path, query object, and request
 // object. If a response object is provided, the server's response is deserialized into that object.
 func (pc *Client) restCall(method, path string, queryObj, reqObj, respObj interface{}) error {
-	return pulumiRESTCall(pc.apiURL, method, path, queryObj, reqObj, respObj, pc.apiToken)
+	return pulumiRESTCall(pc.apiURL, method, path, queryObj, reqObj, respObj, pc.apiToken, httpCallOptions{})
+}
+
+// restCall makes a REST-style request to the Pulumi API using the given method, path, query object, and request
+// object. If a response object is provided, the server's response is deserialized into that object.
+func (pc *Client) restCallWithOptions(method, path string, queryObj, reqObj,
+	respObj interface{}, opts httpCallOptions) error {
+	return pulumiRESTCall(pc.apiURL, method, path, queryObj, reqObj, respObj, pc.apiToken, opts)
 }
 
 // updateRESTCall makes a REST-style request to the Pulumi API using the given method, path, query object, and request
@@ -55,7 +62,7 @@ func (pc *Client) restCall(method, path string, queryObj, reqObj, respObj interf
 func (pc *Client) updateRESTCall(method, path string, queryObj, reqObj, respObj interface{},
 	token updateAccessToken) error {
 
-	return pulumiRESTCall(pc.apiURL, method, path, queryObj, reqObj, respObj, token)
+	return pulumiRESTCall(pc.apiURL, method, path, queryObj, reqObj, respObj, token, httpCallOptions{})
 }
 
 // getProjectPath returns the API path to for the given project with the given components joined with path separators
@@ -375,7 +382,8 @@ func (pc *Client) RenewUpdateLease(update UpdateIdentifier, token string, durati
 		Duration: int(duration / time.Second),
 	}
 	var resp apitype.RenewUpdateLeaseResponse
-	if err := pc.restCall("POST", getUpdatePath(update, "renew_lease"), nil, req, &resp); err != nil {
+	if err := pc.restCallWithOptions("POST", getUpdatePath(update, "renew_lease"), nil,
+		req, &resp, httpCallOptions{RetryAllMethods: true}); err != nil {
 		return "", err
 	}
 	return resp.Token, nil

pkg/resource/asset.go

@@ -24,6 +24,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/pulumi/pulumi/pkg/util/contract"
+	"github.com/pulumi/pulumi/pkg/util/httputil"
 	"github.com/pulumi/pulumi/pkg/workspace"
 )
 
@@ -321,7 +322,7 @@ func (a *Asset) readURI() (*Blob, error) {
 	contract.Assertf(isurl, "Expected a URI-based asset")
 	switch s := url.Scheme; s {
 	case "http", "https":
-		resp, err := http.Get(url.String())
+		resp, err := httputil.GetWithRetry(url.String(), http.DefaultClient)
 		if err != nil {
 			return nil, err
 		}
@@ -827,7 +828,7 @@ func (a *Archive) readURI() (ArchiveReader, error) {
 func (a *Archive) openURLStream(url *url.URL) (io.ReadCloser, error) {
 	switch s := url.Scheme; s {
 	case "http", "https":
-		resp, err := http.Get(url.String())
+		resp, err := httputil.GetWithRetry(url.String(), http.DefaultClient)
 		if err != nil {
 			return nil, err
 		}

pkg/util/httputil/http.go

@@ -0,0 +1,49 @@
+package httputil
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/pulumi/pulumi/pkg/util/retry"
+)
+
+// maxRetryCount is the number of times to try an http request before giving up an returning the last error
+const maxRetryCount = 5
+
+// DoWithRetry calls client.Do, and in the case of an error, retries the operation again after a slight delay.
+func DoWithRetry(req *http.Request, client *http.Client) (*http.Response, error) {
+	inRange := func(test, lower, upper int) bool {
+		return lower <= test && test <= upper
+	}
+
+	_, res, err := retry.Until(context.Background(), retry.Acceptor{
+		Accept: func(try int, nextRetryTime time.Duration) (bool, interface{}, error) {
+			res, resErr := client.Do(req)
+			if resErr == nil && !inRange(res.StatusCode, 500, 599) {
+				return true, res, nil
+			}
+			if try >= (maxRetryCount - 1) {
+				return false, res, resErr
+			}
+			return false, nil, nil
+		},
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return res.(*http.Response), nil
+}
+
+// GetWithRetry issues a GET request with the given client, and in the case of an error, retries the operation again
+// after a slight delay.
+func GetWithRetry(url string, client *http.Client) (*http.Response, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return DoWithRetry(req, client)
+}