maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

rest client: Expect context timeouts for locks (#10782)

Browse source 
Add option for rest clients to not mark a remote offline for context timeouts.

This can be used if context timeouts are expected on the call.
This commit is contained in:
Klaus Post 2020-10-29 09:52:11 -07:00 committed by GitHub
parent 6b14c4ab1e
commit e63a44b734
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 27 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmd/crypto/kes.go
View file

@ -413,7 +413,7 @@ func (c *kesClient) postRetry(path string, body io.ReadSeeker, limit int64) (io.
}
// If the error is not temp. / retryable => fail the request immediately.
if !xnet.IsNetworkOrHostDown(err) &&
if !xnet.IsNetworkOrHostDown(err, false) &&
!errors.Is(err, io.EOF) &&
!errors.Is(err, io.ErrUnexpectedEOF) &&
!errors.Is(err, context.DeadlineExceeded) {

4
cmd/gateway-common.go
View file

@ -270,7 +270,7 @@ func IsBackendOnline(ctx context.Context, clnt *http.Client, urlStr string) bool
resp, err := clnt.Do(req)
if err != nil {
clnt.CloseIdleConnections()
return !xnet.IsNetworkOrHostDown(err)
return !xnet.IsNetworkOrHostDown(err, false)
}
xhttp.DrainBody(resp.Body)
return true
@ -291,7 +291,7 @@ func ErrorRespToObjectError(err error, params ...string) error {
object = params[1]
}
if xnet.IsNetworkOrHostDown(err) {
if xnet.IsNetworkOrHostDown(err, false) {
return BackendDown{}
}

1
cmd/lock-rest-client.go
View file

@ -162,6 +162,7 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient {
trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultTimeout)
restClient := rest.NewClient(serverURL, trFn, newAuthToken)
restClient.ExpectTimeouts = true
restClient.HealthCheckFn = func() bool {
ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
// Instantiate a new rest client for healthcheck

8
cmd/rest/client.go
View file

@ -77,6 +77,10 @@ type Client struct {
// Should only be modified before any calls are made.
MaxErrResponseSize int64
// ExpectTimeouts indicates if context timeouts are expected.
// This will not mark the client offline in these cases.
ExpectTimeouts bool
httpClient *http.Client
url *url.URL
newAuthToken func(audience string) string
@ -114,7 +118,7 @@ func (c *Client) Call(ctx context.Context, method string, values url.Values, bod
}
resp, err := c.httpClient.Do(req)
if err != nil {
if xnet.IsNetworkOrHostDown(err) {
if xnet.IsNetworkOrHostDown(err, c.ExpectTimeouts) {
logger.LogIf(ctx, err, "marking disk offline")
c.MarkOffline()
}
@ -144,7 +148,7 @@ func (c *Client) Call(ctx context.Context, method string, values url.Values, bod
// Limit the ReadAll(), just in case, because of a bug, the server responds with large data.
b, err := ioutil.ReadAll(io.LimitReader(resp.Body, c.MaxErrResponseSize))
if err != nil {
if xnet.IsNetworkOrHostDown(err) {
if xnet.IsNetworkOrHostDown(err, c.ExpectTimeouts) {
logger.LogIf(ctx, err, "marking disk offline")
c.MarkOffline()
}

2
cmd/storage-rest-client.go
View file

@ -42,7 +42,7 @@ func isNetworkError(err error) bool {
return false
}
if nerr, ok := err.(*rest.NetworkError); ok {
return xnet.IsNetworkOrHostDown(nerr.Err)
return xnet.IsNetworkOrHostDown(nerr.Err, false)
}
return false
}

4
cmd/update.go
View file

@ -309,7 +309,7 @@ func downloadReleaseURL(u *url.URL, timeout time.Duration, mode string) (content
client := &http.Client{Transport: getUpdateTransport(timeout)}
resp, err := client.Do(req)
if err != nil {
if xnet.IsNetworkOrHostDown(err) {
if xnet.IsNetworkOrHostDown(err, false) {
return content, AdminError{
Code: AdminUpdateURLNotReachable,
Message: err.Error(),
@ -501,7 +501,7 @@ func getUpdateReaderFromURL(u *url.URL, transport http.RoundTripper, mode string
resp, err := clnt.Do(req)
if err != nil {
if xnet.IsNetworkOrHostDown(err) {
if xnet.IsNetworkOrHostDown(err, false) {
return nil, AdminError{
Code: AdminUpdateURLNotReachable,
Message: err.Error(),

8
pkg/event/target/elasticsearch.go
View file

@ -124,7 +124,7 @@ func (target *ElasticsearchTarget) IsActive() (bool, error) {
}
_, code, err := target.client.Ping(target.args.URL.String()).HttpHeadOnly(true).Do(ctx)
if err != nil {
if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) {
if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) {
return false, errNotConnected
}
return false, err
@ -138,7 +138,7 @@ func (target *ElasticsearchTarget) Save(eventData event.Event) error {
return target.store.Put(eventData)
}
err := target.send(eventData)
if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) {
if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) {
return errNotConnected
}
return err
@ -214,7 +214,7 @@ func (target *ElasticsearchTarget) Send(eventKey string) error {
}
if err := target.send(eventData); err != nil {
if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) {
if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) {
return errNotConnected
}
return err
@ -267,7 +267,7 @@ func newClient(args ElasticsearchArgs) (*elastic.Client, error) {
client, err := elastic.NewClient(options...)
if err != nil {
// https://github.com/olivere/elastic/wiki/Connection-Errors
if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) {
if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) {
return nil, errNotConnected
}
return nil, err

8
pkg/event/target/webhook.go
View file

@ -111,7 +111,7 @@ func (target *WebhookTarget) IsActive() (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodHead, target.args.Endpoint.String(), nil)
if err != nil {
if xnet.IsNetworkOrHostDown(err) {
if xnet.IsNetworkOrHostDown(err, false) {
return false, errNotConnected
}
return false, err
@ -119,7 +119,7 @@ func (target *WebhookTarget) IsActive() (bool, error) {
resp, err := target.httpClient.Do(req)
if err != nil {
if xnet.IsNetworkOrHostDown(err) || errors.Is(err, context.DeadlineExceeded) {
if xnet.IsNetworkOrHostDown(err, false) || errors.Is(err, context.DeadlineExceeded) {
return false, errNotConnected
}
return false, err
@ -137,7 +137,7 @@ func (target *WebhookTarget) Save(eventData event.Event) error {
}
err := target.send(eventData)
if err != nil {
if xnet.IsNetworkOrHostDown(err) {
if xnet.IsNetworkOrHostDown(err, false) {
return errNotConnected
}
}
@ -197,7 +197,7 @@ func (target *WebhookTarget) Send(eventKey string) error {
}
if err := target.send(eventData); err != nil {
if xnet.IsNetworkOrHostDown(err) {
if xnet.IsNetworkOrHostDown(err, false) {
return errNotConnected
}
return err

8
pkg/net/url.go
View file

@ -141,14 +141,18 @@ func ParseURL(s string) (u *URL, err error) {
}
// IsNetworkOrHostDown - if there was a network error or if the host is down.
func IsNetworkOrHostDown(err error) bool {
// expectTimeouts indicates that context timeouts are expected and does not
// indicate a downed host. Other timeouts still returns down.
func IsNetworkOrHostDown(err error, expectTimeouts bool) bool {
if err == nil {
return false
}
if errors.Is(err, context.Canceled) {
return false
}
if expectTimeouts && errors.Is(err, context.DeadlineExceeded) {
return false
}
// We need to figure if the error either a timeout
// or a non-temporary error.
e, ok := err.(net.Error)