diff --git a/cmd/config-common.go b/cmd/config-common.go index 5b1fc8e62..ba53b3180 100644 --- a/cmd/config-common.go +++ b/cmd/config-common.go @@ -50,6 +50,20 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b return buffer.Bytes(), nil } +func deleteConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error { + _, err := client.Delete(ctx, configFile) + return err +} + +func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error { + return objAPI.DeleteObject(ctx, minioMetaBucket, configFile) +} + +func saveConfigEtcd(ctx context.Context, client *etcd.Client, configFile string, data []byte) error { + _, err := client.Put(ctx, configFile, string(data)) + return err +} + func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data []byte) error { hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data))) if err != nil { diff --git a/cmd/config-current.go b/cmd/config-current.go index 2eb0dbc3c..8d449cdfc 100644 --- a/cmd/config-current.go +++ b/cmd/config-current.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "os" "reflect" "sync" @@ -30,6 +31,7 @@ import ( "github.com/minio/minio/pkg/event/target" "github.com/minio/minio/pkg/iam/policy" "github.com/minio/minio/pkg/iam/validator" + xnet "github.com/minio/minio/pkg/net" ) // Steps to move from version N to version N+1 @@ -270,6 +272,20 @@ func (s *serverConfig) loadFromEnvs() { if globalIsEnvCompression { s.SetCompressionConfig(globalCompressExtensions, globalCompressMimeTypes) } + + if jwksURL, ok := os.LookupEnv("MINIO_IAM_JWKS_URL"); ok { + if u, err := xnet.ParseURL(jwksURL); err == nil { + s.OpenID.JWKS.URL = u + s.OpenID.JWKS.PopulatePublicKey() + } + } + + if opaURL, ok := os.LookupEnv("MINIO_IAM_OPA_URL"); ok { + if u, err := xnet.ParseURL(opaURL); err == nil { + s.Policy.OPA.URL = u + s.Policy.OPA.AuthToken = os.Getenv("MINIO_IAM_OPA_AUTHTOKEN") + } + } } // TestNotificationTargets tries to establish connections to all notification diff --git a/cmd/gateway-main.go b/cmd/gateway-main.go index 34cdc71d2..8ae650abf 100644 --- a/cmd/gateway-main.go +++ b/cmd/gateway-main.go @@ -244,9 +244,10 @@ func StartGateway(ctx *cli.Context, gw Gateway) { // Create new IAM system. globalIAMSys = NewIAMSys() - - // Initialize IAM sys. - go globalIAMSys.Init(newObject) + if globalEtcdClient != nil { + // Initialize IAM sys. + go globalIAMSys.Init(newObject) + } // Create new policy system. globalPolicySys = NewPolicySys() diff --git a/cmd/iam.go b/cmd/iam.go index c1c50197c..36f0d7151 100644 --- a/cmd/iam.go +++ b/cmd/iam.go @@ -20,9 +20,12 @@ import ( "context" "encoding/json" "path" + "strings" "sync" "time" + etcd "github.com/coreos/etcd/clientv3" + "github.com/minio/minio-go/pkg/set" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" "github.com/minio/minio/pkg/iam/policy" @@ -64,25 +67,46 @@ func (sys *IAMSys) Init(objAPI ObjectLayer) error { return errInvalidArgument } - if err := sys.refresh(objAPI); err != nil { - return err - } - - // Refresh IAMSys in background. - go func() { - ticker := time.NewTicker(globalRefreshIAMInterval) - defer ticker.Stop() - for { - select { - case <-globalServiceDoneCh: - return - case <-ticker.C: - logger.LogIf(context.Background(), sys.refresh(objAPI)) + defer func() { + // Refresh IAMSys in background. + go func() { + ticker := time.NewTicker(globalRefreshIAMInterval) + defer ticker.Stop() + for { + select { + case <-globalServiceDoneCh: + return + case <-ticker.C: + sys.refresh(objAPI) + } } - } + }() }() - return nil + doneCh := make(chan struct{}) + defer close(doneCh) + + // Initializing IAM needs a retry mechanism for + // the following reasons: + // - Read quorum is lost just after the initialization + // of the object layer. + retryTimerCh := newRetryTimerSimple(doneCh) + for { + select { + case _ = <-retryTimerCh: + // Load IAMSys once during boot. + if err := sys.refresh(objAPI); err != nil { + if err == errDiskNotFound || + strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) || + strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) { + logger.Info("Waiting for IAM subsystem to be initialized..") + continue + } + return err + } + return nil + } + } } // SetPolicy - sets policy to given user name. If policy is empty, @@ -99,13 +123,19 @@ func (sys *IAMSys) SetPolicy(accessKey string, p iampolicy.Policy) error { return err } + if globalEtcdClient != nil { + if err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data); err != nil { + return err + } + } else { + if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil { + return err + } + } + sys.Lock() defer sys.Unlock() - if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil { - return err - } - if p.IsEmpty() { delete(sys.iamPolicyMap, accessKey) } else { @@ -128,13 +158,19 @@ func (sys *IAMSys) SaveTempPolicy(accessKey string, p iampolicy.Policy) error { return err } + if globalEtcdClient != nil { + if err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data); err != nil { + return err + } + } else { + if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil { + return err + } + } + sys.Lock() defer sys.Unlock() - if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil { - return err - } - if p.IsEmpty() { delete(sys.iamPolicyMap, accessKey) } else { @@ -152,13 +188,17 @@ func (sys *IAMSys) DeletePolicy(accessKey string) error { return errServerNotInitialized } + var err error configFile := pathJoin(iamConfigUsersPrefix, accessKey, iamPolicyFile) + if globalEtcdClient != nil { + err = deleteConfigEtcd(context.Background(), globalEtcdClient, configFile) + } else { + err = deleteConfig(context.Background(), objectAPI, configFile) + } sys.Lock() defer sys.Unlock() - err := objectAPI.DeleteObject(context.Background(), minioMetaBucket, configFile) - delete(sys.iamPolicyMap, accessKey) return err @@ -171,12 +211,17 @@ func (sys *IAMSys) DeleteUser(accessKey string) error { return errServerNotInitialized } + var err error + configFile := pathJoin(iamConfigUsersPrefix, accessKey, iamPolicyFile) + if globalEtcdClient != nil { + err = deleteConfigEtcd(context.Background(), globalEtcdClient, configFile) + } else { + err = deleteConfig(context.Background(), objectAPI, configFile) + } + sys.Lock() defer sys.Unlock() - configFile := pathJoin(iamConfigUsersPrefix, accessKey, iamIdentityFile) - err := objectAPI.DeleteObject(context.Background(), minioMetaBucket, configFile) - delete(sys.iamUsersMap, accessKey) return err } @@ -188,19 +233,25 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials) error { return errServerNotInitialized } - sys.Lock() - defer sys.Unlock() - configFile := pathJoin(iamConfigSTSPrefix, accessKey, iamIdentityFile) data, err := json.Marshal(cred) if err != nil { return err } - if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil { - return err + if globalEtcdClient != nil { + if err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data); err != nil { + return err + } + } else { + if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil { + return err + } } + sys.Lock() + defer sys.Unlock() + sys.iamUsersMap[accessKey] = cred return nil } @@ -218,13 +269,19 @@ func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error { return err } + if globalEtcdClient != nil { + if err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data); err != nil { + return err + } + } else { + if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil { + return err + } + } + sys.Lock() defer sys.Unlock() - if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil { - return err - } - sys.iamUsersMap[accessKey] = auth.Credentials{ AccessKey: accessKey, SecretKey: uinfo.SecretKey, @@ -266,6 +323,76 @@ func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool { return args.IsOwner } +var defaultContextTimeout = 5 * time.Minute + +// Similar to reloadUsers but updates users, policies maps from etcd server, +func reloadEtcdUsers(prefix string, usersMap map[string]auth.Credentials, policyMap map[string]iampolicy.Policy) error { + ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout) + r, err := globalEtcdClient.Get(ctx, prefix, etcd.WithPrefix(), etcd.WithKeysOnly()) + defer cancel() + if err != nil { + return err + } + // No users are created yet. + if r.Count == 0 { + return nil + } + + users := set.NewStringSet() + for _, kv := range r.Kvs { + // Extract user by stripping off the `prefix` value as suffix, + // then strip off the remaining basename to obtain the prefix + // value, usually in the following form. + // + // key := "config/iam/users/newuser/identity.json" + // prefix := "config/iam/users/" + // v := trim(trim(key, prefix), base(key)) == "newuser" + // + user := strings.TrimSuffix(strings.TrimSuffix(string(kv.Key), prefix), path.Base(string(kv.Key))) + if !users.Contains(user) { + users.Add(user) + } + } + + // Reload config and policies for all users. + for _, user := range users.ToSlice() { + idFile := pathJoin(prefix, user, iamIdentityFile) + pFile := pathJoin(prefix, user, iamPolicyFile) + cdata, cerr := readConfigEtcd(ctx, globalEtcdClient, idFile) + pdata, perr := readConfigEtcd(ctx, globalEtcdClient, pFile) + if cerr != nil && cerr != errConfigNotFound { + return cerr + } + if perr != nil && perr != errConfigNotFound { + return perr + } + if cerr == errConfigNotFound && perr == errConfigNotFound { + continue + } + if cerr == nil { + var cred auth.Credentials + if err = json.Unmarshal(cdata, &cred); err != nil { + return err + } + cred.AccessKey = user + if cred.IsExpired() { + deleteConfigEtcd(ctx, globalEtcdClient, idFile) + deleteConfigEtcd(ctx, globalEtcdClient, pFile) + continue + } + usersMap[cred.AccessKey] = cred + } + if perr == nil { + var p iampolicy.Policy + if err = json.Unmarshal(pdata, &p); err != nil { + return err + } + policyMap[path.Base(prefix)] = p + } + } + return nil +} + // reloadUsers reads an updates users, policies from object layer into user and policy maps. func reloadUsers(objectAPI ObjectLayer, prefix string, usersMap map[string]auth.Credentials, policyMap map[string]iampolicy.Policy) error { marker := "" @@ -326,12 +453,20 @@ func (sys *IAMSys) refresh(objAPI ObjectLayer) error { iamUsersMap := make(map[string]auth.Credentials) iamPolicyMap := make(map[string]iampolicy.Policy) - if err := reloadUsers(objAPI, iamConfigUsersPrefix, iamUsersMap, iamPolicyMap); err != nil { - return err - } - - if err := reloadUsers(objAPI, iamConfigSTSPrefix, iamUsersMap, iamPolicyMap); err != nil { - return err + if globalEtcdClient != nil { + if err := reloadEtcdUsers(iamConfigUsersPrefix, iamUsersMap, iamPolicyMap); err != nil { + return err + } + if err := reloadEtcdUsers(iamConfigSTSPrefix, iamUsersMap, iamPolicyMap); err != nil { + return err + } + } else { + if err := reloadUsers(objAPI, iamConfigUsersPrefix, iamUsersMap, iamPolicyMap); err != nil { + return err + } + if err := reloadUsers(objAPI, iamConfigSTSPrefix, iamUsersMap, iamPolicyMap); err != nil { + return err + } } sys.Lock() diff --git a/cmd/object-api-errors.go b/cmd/object-api-errors.go index 32a3d92f6..035737c91 100644 --- a/cmd/object-api-errors.go +++ b/cmd/object-api-errors.go @@ -383,15 +383,3 @@ func isErrObjectNotFound(err error) bool { _, ok := err.(ObjectNotFound) return ok } - -// isInsufficientReadQuorum - Check if error type is InsufficientReadQuorum. -func isInsufficientReadQuorum(err error) bool { - _, ok := err.(InsufficientReadQuorum) - return ok -} - -// isInsufficientWriteQuorum - Check if error type is InsufficientWriteQuorum. -func isInsufficientWriteQuorum(err error) bool { - _, ok := err.(InsufficientWriteQuorum) - return ok -} diff --git a/docs/sts/README.md b/docs/sts/README.md index 342c59b4b..55a953c5e 100644 --- a/docs/sts/README.md +++ b/docs/sts/README.md @@ -19,6 +19,7 @@ In this document we will explain in detail on how to configure all the prerequis ### 1. Prerequisites - [Configuring wso2](./wso2.md) - [Configuring opa](./opa.md) +- [Configuring etcd (optional needed only in gateway or federation mode)](./etcd.md) ### 2. Setup Minio with WSO2, OPA Make sure we have followed the previous step and configured each software independently, once done we can now proceed to use Minio STS API and Minio server to use these credentials to perform object API operations. @@ -31,7 +32,21 @@ export MINIO_IAM_OPA_URL=http://localhost:8181/v1/data/httpapi/authz minio server /mnt/data ``` -### 3. Test using full-example.go +### 3. Setup Minio Gateway with WSO2, OPA, ETCD +Make sure we have followed the previous step and configured each software independently, once done we can now proceed to use Minio STS API and Minio gateway to use these credentials to perform object API operations. + +> NOTE: Minio gateway requires etcd to be configured to use STS API. + +``` +export MINIO_ACCESS_KEY=aws_access_key +export MINIO_SECRET_KEY=aws_secret_key +export MINIO_IAM_JWKS_URL=https://localhost:9443/oauth2/jwks +export MINIO_IAM_OPA_URL=http://localhost:8181/v1/data/httpapi/authz +export MINIO_ETCD_ENDPOINTS=localhost:2379 +minio gateway s3 +``` + +### 4. Test using full-example.go On another terminal run `full-example.go` a sample client application which obtains JWT access tokens from an identity provider, in our case its WSO2. Uses the returned access token response to get new temporary credentials from the Minio server using the STS API call `AssumeRoleWithClientGrants`. ``` diff --git a/docs/sts/etcd.md b/docs/sts/etcd.md new file mode 100644 index 000000000..e52f72230 --- /dev/null +++ b/docs/sts/etcd.md @@ -0,0 +1,57 @@ +# etcd V3 Quickstart Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) +etcd is a distributed key value store that provides a reliable way to store data across a cluster of machines. + +## Get started +### 1. Prerequisites +- Docker 18.03 or above, refer here for [installation](https://docs.docker.com/install/). + +### 2. Start etcd +etcd uses [gcr.io/etcd-development/etcd](gcr.io/etcd-development/etcd) as a primary container registry. + +``` +rm -rf /tmp/etcd-data.tmp && mkdir -p /tmp/etcd-data.tmp && \ + docker rmi gcr.io/etcd-development/etcd:v3.3.9 || true && \ + docker run \ + -p 2379:2379 \ + -p 2380:2380 \ + --mount type=bind,source=/tmp/etcd-data.tmp,destination=/etcd-data \ + --name etcd-gcr-v3.3.9 \ + gcr.io/etcd-development/etcd:v3.3.9 \ + /usr/local/bin/etcd \ + --name s1 \ + --data-dir /etcd-data \ + --listen-client-urls http://0.0.0.0:2379 \ + --advertise-client-urls http://0.0.0.0:2379 \ + --listen-peer-urls http://0.0.0.0:2380 \ + --initial-advertise-peer-urls http://0.0.0.0:2380 \ + --initial-cluster s1=http://0.0.0.0:2380 \ + --initial-cluster-token tkn \ + --initial-cluster-state new +``` + +### 3. Setup Minio with etcd +Minio server expects environment variable for etcd as `MINIO_ETCD_ENDPOINTS`, this environment variable takes many comma separated entries. +``` +export MINIO_ETCD_ENDPOINTS=localhost:2379 +minio server /data +``` + +### 5. Test with Minio STS API +Assuming that you have configured Minio server to support STS API by following the doc [Minio STS Quickstart Guide](https://docs.minio.io/docs/minio-sts-quickstart-guide) and once you have obtained the JWT from WSO2 as mentioned in [WSO2 Quickstart Guide](https://docs.minio.io/docs/wso2-quickstart-guide). +``` +go run full-example.go -cid PoEgXP6uVO45IsENRngDXj5Au5Ya -csec eKsw6z8CtOJVBtrOWvhRWL4TUCga + +##### Credentials +{ + "accessKey": "IRBLVDGN5QGMDCMO1X8V", + "secretKey": "KzS3UZKE7xqNdtRbKyfcWgxBS6P1G4kwZn4DXKuY", + "expiration": "2018-08-21T15:49:38-07:00", + "sessionToken": "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJJUkJMVkRHTjVRR01EQ01PMVg4ViIsImF1ZCI6IlBvRWdYUDZ1Vk80NUlzRU5SbmdEWGo1QXU1WWEiLCJhenAiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiZXhwIjoxNTM0ODkxNzc4LCJpYXQiOjE1MzQ4ODgxNzgsImlzcyI6Imh0dHBzOi8vbG9jYWxob3N0Ojk0NDMvb2F1dGgyL3Rva2VuIiwianRpIjoiMTg0NDMyOWMtZDY1YS00OGEzLTgyMjgtOWRmNzNmZTgzZDU2In0.4rKsZ8VkZnIS_ALzfTJ9UbEKPFlQVvIyuHw6AWTJcDFDVgQA2ooQHmH9wUDnhXBi1M7o8yWJ47DXP-TLPhwCgQ" +} +``` + +These credentials can now be used to perform Minio API operations, these credentials automatically expire in 1hr. To understand more about credential expiry duration and client grants STS API read further [here](https://docs.minio.io/docs/api-assume-role-with-client-grants). + +## Explore Further +- [Minio STS Quickstart Guide](https://docs.minio.io/docs/minio-sts-quickstart-guide) +- [The Minio documentation website](https://docs.minio.io) diff --git a/docs/sts/wso2.md b/docs/sts/wso2.md index 0e2e055bb..011678ff7 100644 --- a/docs/sts/wso2.md +++ b/docs/sts/wso2.md @@ -83,6 +83,8 @@ go run full-example.go -cid PoEgXP6uVO45IsENRngDXj5Au5Ya -csec eKsw6z8CtOJVBtrOW } ``` +These credentials can now be used to perform Minio API operations, these credentials automatically expire in 1hr. To understand more about credential expiry duration and client grants STS API read further [here](https://docs.minio.io/docs/api-assume-role-with-client-grants). + ## Explore Further - [Minio STS Quickstart Guide](https://docs.minio.io/docs/minio-sts-quickstart-guide) - [The Minio documentation website](https://docs.minio.io)