maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add proper contexts with timeouts for etcd operations (#7097)

Browse source 
This fixes an issue of perceived hang when incorrect
unreachable URLs are specified in MINIO_ETCD_ENDPOINTS
variable.

Fixes #7096
This commit is contained in:
Harshavardhana 2019-01-18 23:06:45 +05:30 committed by kannappanr
parent 730ac5381c
commit 74c2048ea9
5 changed files with 56 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cmd/common-main.go
View file

@ -242,6 +242,7 @@ func handleCommonEnvVars() {
return &cert, terr
}
}
globalEtcdClient, err = etcd.New(etcd.Config{
Endpoints: etcdEndpoints,
DialTimeout: defaultDialTimeout,

34
cmd/config-common.go
View file

@ -20,6 +20,7 @@ import (
"bytes"
"context"
"errors"
"fmt"
etcd "github.com/coreos/etcd/clientv3"
"github.com/minio/minio/cmd/logger"
@ -60,8 +61,15 @@ func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) er
}
func saveConfigEtcd(ctx context.Context, client *etcd.Client, configFile string, data []byte) error {
_, err := client.Put(ctx, configFile, string(data))
return err
timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
defer cancel()
_, err := client.Put(timeoutCtx, configFile, string(data))
if err == context.DeadlineExceeded {
return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
} else if err != nil {
return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
}
return nil
}
func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data []byte) error {
@ -75,9 +83,14 @@ func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data
}
func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) ([]byte, error) {
resp, err := client.Get(ctx, configFile)
timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
defer cancel()
resp, err := client.Get(timeoutCtx, configFile)
if err != nil {
return nil, err
if err == context.DeadlineExceeded {
return nil, fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
}
return nil, fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
}
if resp.Count == 0 {
return nil, errConfigNotFound
@ -93,7 +106,9 @@ func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string)
// watchConfig - watches for changes on `configFile` on etcd and loads them.
func watchConfig(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
if globalEtcdClient != nil {
for watchResp := range globalEtcdClient.Watch(context.Background(), configFile) {
ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
defer cancel()
for watchResp := range globalEtcdClient.Watch(ctx, configFile) {
for _, event := range watchResp.Events {
if event.IsModify() || event.IsCreate() {
loadCfgFn(objAPI)
@ -104,9 +119,14 @@ func watchConfig(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLay
}
func checkConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
resp, err := globalEtcdClient.Get(ctx, configFile)
timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
defer cancel()
resp, err := client.Get(timeoutCtx, configFile)
if err != nil {
return err
if err == context.DeadlineExceeded {
return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
}
return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
}
if resp.Count == 0 {
return errConfigNotFound

28
cmd/config.go
View file

@ -24,7 +24,6 @@ import (
"path"
"runtime"
"strings"
"time"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/quick"
@ -52,10 +51,7 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverCon
configFile := path.Join(minioConfigPrefix, minioConfigFile)
if globalEtcdClient != nil {
timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
_, err = globalEtcdClient.Put(timeoutCtx, configFile, string(data))
defer cancel()
return err
return saveConfigEtcd(ctx, globalEtcdClient, configFile, data)
}
// Create a backup of the current config
@ -163,18 +159,16 @@ func initConfig(objAPI ObjectLayer) error {
}
if globalEtcdClient != nil {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
resp, err := globalEtcdClient.Get(ctx, getConfigFile())
cancel()
if err == nil && resp.Count > 0 {
if err = migrateConfig(); err != nil {
return err
}
// Migrates etcd ${HOME}/.minio/config.json to '/config/config.json'
if err := migrateConfigToMinioSys(objAPI); err != nil {
return err
}
if err := checkConfigEtcd(context.Background(), globalEtcdClient, getConfigFile()); err != nil {
return err
}
// Migrates all configs at old location.
if err := migrateConfig(); err != nil {
return err
}
// Migrates etcd ${HOME}/.minio/config.json to '/config/config.json'
if err := migrateConfigToMinioSys(objAPI); err != nil {
return err
}
} else {
if isFile(getConfigFile()) {

6
cmd/iam.go
View file

@ -464,13 +464,13 @@ func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
return args.IsOwner
}
var defaultContextTimeout = 5 * time.Minute
var defaultContextTimeout = 30 * time.Second
// Similar to reloadUsers but updates users, policies maps from etcd server,
func reloadEtcdUsers(prefix string, usersMap map[string]auth.Credentials, policyMap map[string]string) error {
ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
r, err := globalEtcdClient.Get(ctx, prefix, etcd.WithPrefix(), etcd.WithKeysOnly())
defer cancel()
r, err := globalEtcdClient.Get(ctx, prefix, etcd.WithPrefix(), etcd.WithKeysOnly())
if err != nil {
return err
}
@ -536,8 +536,8 @@ func reloadEtcdUsers(prefix string, usersMap map[string]auth.Credentials, policy
func reloadEtcdPolicies(prefix string, cannedPolicyMap map[string]iampolicy.Policy) error {
ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
r, err := globalEtcdClient.Get(ctx, prefix, etcd.WithPrefix(), etcd.WithKeysOnly())
defer cancel()
r, err := globalEtcdClient.Get(ctx, prefix, etcd.WithPrefix(), etcd.WithKeysOnly())
if err != nil {
return err
}

20
pkg/quick/encoding.go
View file

@ -138,18 +138,26 @@ func saveFileConfigEtcd(filename string, clnt *etcd.Client, v interface{}) error
dataBytes = []byte(strings.Replace(string(dataBytes), "\n", "\r\n", -1))
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
_, err = clnt.Put(ctx, filename, string(dataBytes))
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
return err
_, err = clnt.Put(ctx, filename, string(dataBytes))
if err == context.DeadlineExceeded {
return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", clnt.Endpoints())
} else if err != nil {
return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, clnt.Endpoints())
}
return nil
}
func loadFileConfigEtcd(filename string, clnt *etcd.Client, v interface{}) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
resp, err := clnt.Get(ctx, filename)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
resp, err := clnt.Get(ctx, filename)
if err != nil {
return err
if err == context.DeadlineExceeded {
return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", clnt.Endpoints())
}
return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, clnt.Endpoints())
}
if resp.Count == 0 {
return dns.ErrNoEntriesFound