50843a98c1
We've seen failures in CI where DNS lookups fail which cause our operations against the service to fail, as well as other sorts of timeouts. Add a set of helper methods in a new httputil package that helps us do retries on these operations, and then update our client library to use them when we are doing GET requests. We also provide a way for non GET requests to be retried, and use this when updating a lease (since it is safe to retry multiple requests in this case).
49 lines
1.3 KiB
Go
49 lines
1.3 KiB
Go
package httputil
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"time"
|
|
|
|
"github.com/pulumi/pulumi/pkg/util/retry"
|
|
)
|
|
|
|
// maxRetryCount is the number of times to try an http request before giving up an returning the last error
|
|
const maxRetryCount = 5
|
|
|
|
// DoWithRetry calls client.Do, and in the case of an error, retries the operation again after a slight delay.
|
|
func DoWithRetry(req *http.Request, client *http.Client) (*http.Response, error) {
|
|
inRange := func(test, lower, upper int) bool {
|
|
return lower <= test && test <= upper
|
|
}
|
|
|
|
_, res, err := retry.Until(context.Background(), retry.Acceptor{
|
|
Accept: func(try int, nextRetryTime time.Duration) (bool, interface{}, error) {
|
|
res, resErr := client.Do(req)
|
|
if resErr == nil && !inRange(res.StatusCode, 500, 599) {
|
|
return true, res, nil
|
|
}
|
|
if try >= (maxRetryCount - 1) {
|
|
return false, res, resErr
|
|
}
|
|
return false, nil, nil
|
|
},
|
|
})
|
|
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return res.(*http.Response), nil
|
|
}
|
|
|
|
// GetWithRetry issues a GET request with the given client, and in the case of an error, retries the operation again
|
|
// after a slight delay.
|
|
func GetWithRetry(url string, client *http.Client) (*http.Response, error) {
|
|
req, err := http.NewRequest("GET", url, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return DoWithRetry(req, client)
|
|
}
|