maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add etcd support to support STS on gateway mode (#6531)

Browse source
This commit is contained in:
Harshavardhana 2018-10-12 11:32:18 -07:00 committed by Dee Koder
parent f09e7ca764
commit 143e7fe300
8 changed files with 288 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
cmd/config-common.go
View file

@ -50,6 +50,20 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b
return buffer.Bytes(), nil
}
func deleteConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
_, err := client.Delete(ctx, configFile)
return err
}
func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
return objAPI.DeleteObject(ctx, minioMetaBucket, configFile)
}
func saveConfigEtcd(ctx context.Context, client *etcd.Client, configFile string, data []byte) error {
_, err := client.Put(ctx, configFile, string(data))
return err
}
func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data []byte) error {
hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
if err != nil {

16
cmd/config-current.go
View file

@ -20,6 +20,7 @@ import (
"context"
"errors"
"fmt"
"os"
"reflect"
"sync"
@ -30,6 +31,7 @@ import (
"github.com/minio/minio/pkg/event/target"
"github.com/minio/minio/pkg/iam/policy"
"github.com/minio/minio/pkg/iam/validator"
xnet "github.com/minio/minio/pkg/net"
)
// Steps to move from version N to version N+1
@ -270,6 +272,20 @@ func (s *serverConfig) loadFromEnvs() {
if globalIsEnvCompression {
s.SetCompressionConfig(globalCompressExtensions, globalCompressMimeTypes)
}
if jwksURL, ok := os.LookupEnv("MINIO_IAM_JWKS_URL"); ok {
if u, err := xnet.ParseURL(jwksURL); err == nil {
s.OpenID.JWKS.URL = u
s.OpenID.JWKS.PopulatePublicKey()
}
}
if opaURL, ok := os.LookupEnv("MINIO_IAM_OPA_URL"); ok {
if u, err := xnet.ParseURL(opaURL); err == nil {
s.Policy.OPA.URL = u
s.Policy.OPA.AuthToken = os.Getenv("MINIO_IAM_OPA_AUTHTOKEN")
}
}
}
// TestNotificationTargets tries to establish connections to all notification

7
cmd/gateway-main.go
View file

@ -244,9 +244,10 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
// Create new IAM system.
globalIAMSys = NewIAMSys()
// Initialize IAM sys.
go globalIAMSys.Init(newObject)
if globalEtcdClient != nil {
// Initialize IAM sys.
go globalIAMSys.Init(newObject)
}
// Create new policy system.
globalPolicySys = NewPolicySys()

223
cmd/iam.go
View file

@ -20,9 +20,12 @@ import (
"context"
"encoding/json"
"path"
"strings"
"sync"
"time"
etcd "github.com/coreos/etcd/clientv3"
"github.com/minio/minio-go/pkg/set"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/iam/policy"
@ -64,25 +67,46 @@ func (sys *IAMSys) Init(objAPI ObjectLayer) error {
return errInvalidArgument
}
if err := sys.refresh(objAPI); err != nil {
return err
}
// Refresh IAMSys in background.
go func() {
ticker := time.NewTicker(globalRefreshIAMInterval)
defer ticker.Stop()
for {
select {
case <-globalServiceDoneCh:
return
case <-ticker.C:
logger.LogIf(context.Background(), sys.refresh(objAPI))
defer func() {
// Refresh IAMSys in background.
go func() {
ticker := time.NewTicker(globalRefreshIAMInterval)
defer ticker.Stop()
for {
select {
case <-globalServiceDoneCh:
return
case <-ticker.C:
sys.refresh(objAPI)
}
}
}
}()
}()
return nil
doneCh := make(chan struct{})
defer close(doneCh)
// Initializing IAM needs a retry mechanism for
// the following reasons:
// - Read quorum is lost just after the initialization
// of the object layer.
retryTimerCh := newRetryTimerSimple(doneCh)
for {
select {
case _ = <-retryTimerCh:
// Load IAMSys once during boot.
if err := sys.refresh(objAPI); err != nil {
if err == errDiskNotFound ||
strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
logger.Info("Waiting for IAM subsystem to be initialized..")
continue
}
return err
}
return nil
}
}
}
// SetPolicy - sets policy to given user name. If policy is empty,
@ -99,13 +123,19 @@ func (sys *IAMSys) SetPolicy(accessKey string, p iampolicy.Policy) error {
return err
}
if globalEtcdClient != nil {
if err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data); err != nil {
return err
}
} else {
if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil {
return err
}
}
sys.Lock()
defer sys.Unlock()
if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil {
return err
}
if p.IsEmpty() {
delete(sys.iamPolicyMap, accessKey)
} else {
@ -128,13 +158,19 @@ func (sys *IAMSys) SaveTempPolicy(accessKey string, p iampolicy.Policy) error {
return err
}
if globalEtcdClient != nil {
if err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data); err != nil {
return err
}
} else {
if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil {
return err
}
}
sys.Lock()
defer sys.Unlock()
if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil {
return err
}
if p.IsEmpty() {
delete(sys.iamPolicyMap, accessKey)
} else {
@ -152,13 +188,17 @@ func (sys *IAMSys) DeletePolicy(accessKey string) error {
return errServerNotInitialized
}
var err error
configFile := pathJoin(iamConfigUsersPrefix, accessKey, iamPolicyFile)
if globalEtcdClient != nil {
err = deleteConfigEtcd(context.Background(), globalEtcdClient, configFile)
} else {
err = deleteConfig(context.Background(), objectAPI, configFile)
}
sys.Lock()
defer sys.Unlock()
err := objectAPI.DeleteObject(context.Background(), minioMetaBucket, configFile)
delete(sys.iamPolicyMap, accessKey)
return err
@ -171,12 +211,17 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
return errServerNotInitialized
}
var err error
configFile := pathJoin(iamConfigUsersPrefix, accessKey, iamPolicyFile)
if globalEtcdClient != nil {
err = deleteConfigEtcd(context.Background(), globalEtcdClient, configFile)
} else {
err = deleteConfig(context.Background(), objectAPI, configFile)
}
sys.Lock()
defer sys.Unlock()
configFile := pathJoin(iamConfigUsersPrefix, accessKey, iamIdentityFile)
err := objectAPI.DeleteObject(context.Background(), minioMetaBucket, configFile)
delete(sys.iamUsersMap, accessKey)
return err
}
@ -188,19 +233,25 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials) error {
return errServerNotInitialized
}
sys.Lock()
defer sys.Unlock()
configFile := pathJoin(iamConfigSTSPrefix, accessKey, iamIdentityFile)
data, err := json.Marshal(cred)
if err != nil {
return err
}
if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil {
return err
if globalEtcdClient != nil {
if err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data); err != nil {
return err
}
} else {
if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil {
return err
}
}
sys.Lock()
defer sys.Unlock()
sys.iamUsersMap[accessKey] = cred
return nil
}
@ -218,13 +269,19 @@ func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
return err
}
if globalEtcdClient != nil {
if err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data); err != nil {
return err
}
} else {
if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil {
return err
}
}
sys.Lock()
defer sys.Unlock()
if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil {
return err
}
sys.iamUsersMap[accessKey] = auth.Credentials{
AccessKey: accessKey,
SecretKey: uinfo.SecretKey,
@ -266,6 +323,76 @@ func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
return args.IsOwner
}
var defaultContextTimeout = 5 * time.Minute
// Similar to reloadUsers but updates users, policies maps from etcd server,
func reloadEtcdUsers(prefix string, usersMap map[string]auth.Credentials, policyMap map[string]iampolicy.Policy) error {
ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
r, err := globalEtcdClient.Get(ctx, prefix, etcd.WithPrefix(), etcd.WithKeysOnly())
defer cancel()
if err != nil {
return err
}
// No users are created yet.
if r.Count == 0 {
return nil
}
users := set.NewStringSet()
for _, kv := range r.Kvs {
// Extract user by stripping off the `prefix` value as suffix,
// then strip off the remaining basename to obtain the prefix
// value, usually in the following form.
//
// key := "config/iam/users/newuser/identity.json"
// prefix := "config/iam/users/"
// v := trim(trim(key, prefix), base(key)) == "newuser"
//
user := strings.TrimSuffix(strings.TrimSuffix(string(kv.Key), prefix), path.Base(string(kv.Key)))
if !users.Contains(user) {
users.Add(user)
}
}
// Reload config and policies for all users.
for _, user := range users.ToSlice() {
idFile := pathJoin(prefix, user, iamIdentityFile)
pFile := pathJoin(prefix, user, iamPolicyFile)
cdata, cerr := readConfigEtcd(ctx, globalEtcdClient, idFile)
pdata, perr := readConfigEtcd(ctx, globalEtcdClient, pFile)
if cerr != nil && cerr != errConfigNotFound {
return cerr
}
if perr != nil && perr != errConfigNotFound {
return perr
}
if cerr == errConfigNotFound && perr == errConfigNotFound {
continue
}
if cerr == nil {
var cred auth.Credentials
if err = json.Unmarshal(cdata, &cred); err != nil {
return err
}
cred.AccessKey = user
if cred.IsExpired() {
deleteConfigEtcd(ctx, globalEtcdClient, idFile)
deleteConfigEtcd(ctx, globalEtcdClient, pFile)
continue
}
usersMap[cred.AccessKey] = cred
}
if perr == nil {
var p iampolicy.Policy
if err = json.Unmarshal(pdata, &p); err != nil {
return err
}
policyMap[path.Base(prefix)] = p
}
}
return nil
}
// reloadUsers reads an updates users, policies from object layer into user and policy maps.
func reloadUsers(objectAPI ObjectLayer, prefix string, usersMap map[string]auth.Credentials, policyMap map[string]iampolicy.Policy) error {
marker := ""
@ -326,12 +453,20 @@ func (sys *IAMSys) refresh(objAPI ObjectLayer) error {
iamUsersMap := make(map[string]auth.Credentials)
iamPolicyMap := make(map[string]iampolicy.Policy)
if err := reloadUsers(objAPI, iamConfigUsersPrefix, iamUsersMap, iamPolicyMap); err != nil {
return err
}
if err := reloadUsers(objAPI, iamConfigSTSPrefix, iamUsersMap, iamPolicyMap); err != nil {
return err
if globalEtcdClient != nil {
if err := reloadEtcdUsers(iamConfigUsersPrefix, iamUsersMap, iamPolicyMap); err != nil {
return err
}
if err := reloadEtcdUsers(iamConfigSTSPrefix, iamUsersMap, iamPolicyMap); err != nil {
return err
}
} else {
if err := reloadUsers(objAPI, iamConfigUsersPrefix, iamUsersMap, iamPolicyMap); err != nil {
return err
}
if err := reloadUsers(objAPI, iamConfigSTSPrefix, iamUsersMap, iamPolicyMap); err != nil {
return err
}
}
sys.Lock()

12
cmd/object-api-errors.go
View file

@ -383,15 +383,3 @@ func isErrObjectNotFound(err error) bool {
_, ok := err.(ObjectNotFound)
return ok
}
// isInsufficientReadQuorum - Check if error type is InsufficientReadQuorum.
func isInsufficientReadQuorum(err error) bool {
_, ok := err.(InsufficientReadQuorum)
return ok
}
// isInsufficientWriteQuorum - Check if error type is InsufficientWriteQuorum.
func isInsufficientWriteQuorum(err error) bool {
_, ok := err.(InsufficientWriteQuorum)
return ok
}

17
docs/sts/README.md
View file

@ -19,6 +19,7 @@ In this document we will explain in detail on how to configure all the prerequis
### 1. Prerequisites
- [Configuring wso2](./wso2.md)
- [Configuring opa](./opa.md)
- [Configuring etcd (optional needed only in gateway or federation mode)](./etcd.md)
### 2. Setup Minio with WSO2, OPA
Make sure we have followed the previous step and configured each software independently, once done we can now proceed to use Minio STS API and Minio server to use these credentials to perform object API operations.
@ -31,7 +32,21 @@ export MINIO_IAM_OPA_URL=http://localhost:8181/v1/data/httpapi/authz
minio server /mnt/data
```
### 3. Test using full-example.go
### 3. Setup Minio Gateway with WSO2, OPA, ETCD
Make sure we have followed the previous step and configured each software independently, once done we can now proceed to use Minio STS API and Minio gateway to use these credentials to perform object API operations.
> NOTE: Minio gateway requires etcd to be configured to use STS API.
```
export MINIO_ACCESS_KEY=aws_access_key
export MINIO_SECRET_KEY=aws_secret_key
export MINIO_IAM_JWKS_URL=https://localhost:9443/oauth2/jwks
export MINIO_IAM_OPA_URL=http://localhost:8181/v1/data/httpapi/authz
export MINIO_ETCD_ENDPOINTS=localhost:2379
minio gateway s3
```
### 4. Test using full-example.go
On another terminal run `full-example.go` a sample client application which obtains JWT access tokens from an identity provider, in our case its WSO2. Uses the returned access token response to get new temporary credentials from the Minio server using the STS API call `AssumeRoleWithClientGrants`.
```

57
docs/sts/etcd.md Normal file
View file

@ -0,0 +1,57 @@
# etcd V3 Quickstart Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io)
etcd is a distributed key value store that provides a reliable way to store data across a cluster of machines.
## Get started
### 1. Prerequisites
- Docker 18.03 or above, refer here for [installation](https://docs.docker.com/install/).
### 2. Start etcd
etcd uses [gcr.io/etcd-development/etcd](gcr.io/etcd-development/etcd) as a primary container registry.
```
rm -rf /tmp/etcd-data.tmp && mkdir -p /tmp/etcd-data.tmp && \
docker rmi gcr.io/etcd-development/etcd:v3.3.9 || true && \
docker run \
-p 2379:2379 \
-p 2380:2380 \
--mount type=bind,source=/tmp/etcd-data.tmp,destination=/etcd-data \
--name etcd-gcr-v3.3.9 \
gcr.io/etcd-development/etcd:v3.3.9 \
/usr/local/bin/etcd \
--name s1 \
--data-dir /etcd-data \
--listen-client-urls http://0.0.0.0:2379 \
--advertise-client-urls http://0.0.0.0:2379 \
--listen-peer-urls http://0.0.0.0:2380 \
--initial-advertise-peer-urls http://0.0.0.0:2380 \
--initial-cluster s1=http://0.0.0.0:2380 \
--initial-cluster-token tkn \
--initial-cluster-state new
```
### 3. Setup Minio with etcd
Minio server expects environment variable for etcd as `MINIO_ETCD_ENDPOINTS`, this environment variable takes many comma separated entries.
```
export MINIO_ETCD_ENDPOINTS=localhost:2379
minio server /data
```
### 5. Test with Minio STS API
Assuming that you have configured Minio server to support STS API by following the doc [Minio STS Quickstart Guide](https://docs.minio.io/docs/minio-sts-quickstart-guide) and once you have obtained the JWT from WSO2 as mentioned in [WSO2 Quickstart Guide](https://docs.minio.io/docs/wso2-quickstart-guide).
```
go run full-example.go -cid PoEgXP6uVO45IsENRngDXj5Au5Ya -csec eKsw6z8CtOJVBtrOWvhRWL4TUCga
##### Credentials
{
"accessKey": "IRBLVDGN5QGMDCMO1X8V",
"secretKey": "KzS3UZKE7xqNdtRbKyfcWgxBS6P1G4kwZn4DXKuY",
"expiration": "2018-08-21T15:49:38-07:00",
"sessionToken": "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJJUkJMVkRHTjVRR01EQ01PMVg4ViIsImF1ZCI6IlBvRWdYUDZ1Vk80NUlzRU5SbmdEWGo1QXU1WWEiLCJhenAiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiZXhwIjoxNTM0ODkxNzc4LCJpYXQiOjE1MzQ4ODgxNzgsImlzcyI6Imh0dHBzOi8vbG9jYWxob3N0Ojk0NDMvb2F1dGgyL3Rva2VuIiwianRpIjoiMTg0NDMyOWMtZDY1YS00OGEzLTgyMjgtOWRmNzNmZTgzZDU2In0.4rKsZ8VkZnIS_ALzfTJ9UbEKPFlQVvIyuHw6AWTJcDFDVgQA2ooQHmH9wUDnhXBi1M7o8yWJ47DXP-TLPhwCgQ"
}
```
These credentials can now be used to perform Minio API operations, these credentials automatically expire in 1hr. To understand more about credential expiry duration and client grants STS API read further [here](https://docs.minio.io/docs/api-assume-role-with-client-grants).
## Explore Further
- [Minio STS Quickstart Guide](https://docs.minio.io/docs/minio-sts-quickstart-guide)
- [The Minio documentation website](https://docs.minio.io)

2
docs/sts/wso2.md
View file

@ -83,6 +83,8 @@ go run full-example.go -cid PoEgXP6uVO45IsENRngDXj5Au5Ya -csec eKsw6z8CtOJVBtrOW
}
```
These credentials can now be used to perform Minio API operations, these credentials automatically expire in 1hr. To understand more about credential expiry duration and client grants STS API read further [here](https://docs.minio.io/docs/api-assume-role-with-client-grants).
## Explore Further
- [Minio STS Quickstart Guide](https://docs.minio.io/docs/minio-sts-quickstart-guide)
- [The Minio documentation website](https://docs.minio.io)