maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Handle connection failures on webhook/url pings (#8204)

Browse source 
Properly handle connection failures while replaying events

Fixes #8194
This commit is contained in:
Praveen raj Mani 2019-09-13 05:14:51 +05:30 committed by Harshavardhana
parent ff6aabd9c0
commit 8700945cdf
7 changed files with 54 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
cmd/gateway-common.go
View file

@ -24,6 +24,7 @@ import (
xhttp "github.com/minio/minio/cmd/http"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/hash"
xnet "github.com/minio/minio/pkg/net"
minio "github.com/minio/minio-go/v6"
)
@ -299,7 +300,7 @@ func ErrorRespToObjectError(err error, params ...string) error {
object = params[1]
}
if isNetworkOrHostDown(err) {
if xnet.IsNetworkOrHostDown(err) {
return BackendDown{}
}

2
cmd/storage-rest-client.go
View file

@ -45,7 +45,7 @@ func isNetworkError(err error) bool {
return true
}
if nerr, ok := err.(*rest.NetworkError); ok {
return isNetworkOrHostDown(nerr.Err)
return xnet.IsNetworkOrHostDown(nerr.Err)
}
return false
}

5
cmd/update.go
View file

@ -35,6 +35,7 @@ import (
"github.com/inconshreveable/go-update"
xhttp "github.com/minio/minio/cmd/http"
"github.com/minio/minio/cmd/logger"
xnet "github.com/minio/minio/pkg/net"
_ "github.com/minio/sha256-simd" // Needed for sha256 hash verifier.
)
@ -283,7 +284,7 @@ func downloadReleaseURL(releaseChecksumURL string, timeout time.Duration, mode s
client := &http.Client{Transport: getUpdateTransport(timeout)}
resp, err := client.Do(req)
if err != nil {
if isNetworkOrHostDown(err) {
if xnet.IsNetworkOrHostDown(err) {
return content, AdminError{
Code: AdminUpdateURLNotReachable,
Message: err.Error(),
@ -500,7 +501,7 @@ func doUpdate(updateURL, sha256Hex, mode string) (err error) {
resp, err := clnt.Do(req)
if err != nil {
if isNetworkOrHostDown(err) {
if xnet.IsNetworkOrHostDown(err) {
return AdminError{
Code: AdminUpdateURLNotReachable,
Message: err.Error(),

38
cmd/utils.go
View file

@ -29,7 +29,6 @@ import (
"io/ioutil"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"reflect"
@ -426,43 +425,6 @@ func newContext(r *http.Request, w http.ResponseWriter, api string) context.Cont
return logger.SetReqInfo(r.Context(), reqInfo)
}
// isNetworkOrHostDown - if there was a network error or if the host is down.
func isNetworkOrHostDown(err error) bool {
if err == nil {
return false
}
// We need to figure if the error either a timeout
// or a non-temporary error.
e, ok := err.(net.Error)
if ok {
urlErr, ok := e.(*url.Error)
if ok {
switch urlErr.Err.(type) {
case *net.DNSError, *net.OpError, net.UnknownNetworkError:
return true
}
}
if e.Timeout() {
return true
}
}
ok = false
// Fallback to other mechanisms.
if strings.Contains(err.Error(), "Connection closed by foreign host") {
ok = true
} else if strings.Contains(err.Error(), "TLS handshake timeout") {
// If error is - tlsHandshakeTimeoutError.
ok = true
} else if strings.Contains(err.Error(), "i/o timeout") {
// If error is - tcp timeoutError.
ok = true
} else if strings.Contains(err.Error(), "connection timed out") {
// If err is a net.Dial timeout.
ok = true
}
return ok
}
// Used for registering with rest handlers (have a look at registerStorageRESTHandlers for usage example)
// If it is passed ["aaaa", "bbbb"], it returns ["aaaa", "{aaaa:.*}", "bbbb", "{bbbb:.*}"]
func restQueries(keys ...string) []string {

14
pkg/event/target/elasticsearch.go
View file

@ -83,11 +83,8 @@ func (target *ElasticsearchTarget) Save(eventData event.Event) error {
return target.store.Put(eventData)
}
if dErr := target.args.URL.DialHTTP(); dErr != nil {
if urlErr, ok := dErr.(*url.Error); ok {
// To treat "connection refused" errors as errNotConnected.
if IsConnRefusedErr(urlErr.Err) {
return errNotConnected
}
if xnet.IsNetworkOrHostDown(dErr) {
return errNotConnected
}
return dErr
}
@ -157,11 +154,8 @@ func (target *ElasticsearchTarget) Send(eventKey string) error {
}
if dErr := target.args.URL.DialHTTP(); dErr != nil {
if urlErr, ok := dErr.(*url.Error); ok {
// To treat "connection refused" errors as errNotConnected.
if IsConnRefusedErr(urlErr.Err) {
return errNotConnected
}
if xnet.IsNetworkOrHostDown(dErr) {
return errNotConnected
}
return dErr
}

15
pkg/event/target/webhook.go
View file

@ -87,11 +87,8 @@ func (target *WebhookTarget) Save(eventData event.Event) error {
return pErr
}
if dErr := u.DialHTTP(); dErr != nil {
if urlErr, ok := dErr.(*url.Error); ok {
// To treat "connection refused" errors as errNotConnected.
if IsConnRefusedErr(urlErr.Err) {
return errNotConnected
}
if xnet.IsNetworkOrHostDown(dErr) {
return errNotConnected
}
return dErr
}
@ -136,17 +133,13 @@ func (target *WebhookTarget) send(eventData event.Event) error {
// Send - reads an event from store and sends it to webhook.
func (target *WebhookTarget) Send(eventKey string) error {
u, pErr := xnet.ParseURL(target.args.Endpoint.String())
if pErr != nil {
return pErr
}
if dErr := u.DialHTTP(); dErr != nil {
if urlErr, ok := dErr.(*url.Error); ok {
// To treat "connection refused" errors as errNotConnected.
if IsConnRefusedErr(urlErr.Err) {
return errNotConnected
}
if xnet.IsNetworkOrHostDown(dErr) {
return errNotConnected
}
return dErr
}

40
pkg/net/url.go
View file

@ -134,3 +134,43 @@ func ParseURL(s string) (u *URL, err error) {
u = &v
return u, nil
}
// IsNetworkOrHostDown - if there was a network error or if the host is down.
func IsNetworkOrHostDown(err error) bool {
if err == nil {
return false
}
// We need to figure if the error either a timeout
// or a non-temporary error.
e, ok := err.(net.Error)
if ok {
urlErr, ok := e.(*url.Error)
if ok {
switch urlErr.Err.(type) {
case *net.DNSError, *net.OpError, net.UnknownNetworkError:
return true
}
}
if e.Timeout() {
return true
}
}
ok = false
// Fallback to other mechanisms.
if strings.Contains(err.Error(), "Connection closed by foreign host") {
ok = true
} else if strings.Contains(err.Error(), "TLS handshake timeout") {
// If error is - tlsHandshakeTimeoutError.
ok = true
} else if strings.Contains(err.Error(), "i/o timeout") {
// If error is - tcp timeoutError.
ok = true
} else if strings.Contains(err.Error(), "connection timed out") {
// If err is a net.Dial timeout.
ok = true
} else if strings.Contains(strings.ToLower(err.Error()), "503 service unavailable") {
// Denial errors
ok = true
}
return ok
}