maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge bc5f09fe14 into 68c5ad83fb

Browse source
This commit is contained in:
Aditya Manthramurthy 2021-11-11 08:52:27 +00:00 committed by GitHub
parent 68c5ad83fb bc5f09fe14
commit d4df6305f9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 314 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

102
.github/workflows/iam-integrations.yaml vendored
View file

@ -1,4 +1,4 @@
name: IAM integration with external systems
name: IAM integration
on:
pull_request:
@ -12,80 +12,13 @@ concurrency:
cancel-in-progress: true
jobs:
ldap-test:
name: LDAP Tests with Go ${{ matrix.go-version }}
iam-matrix-test:
name: "[Go=${{ matrix.go-version }}|ldap=${{ matrix.ldap }}|etcd=${{ matrix.etcd }}|openid=${{ matrix.openid }}]"
runs-on: ubuntu-latest
services:
openldap:
image: quay.io/minio/openldap
ports:
- "389:389"
- "636:636"
env:
LDAP_ORGANIZATION: "MinIO Inc"
LDAP_DOMAIN: "min.io"
LDAP_ADMIN_PASSWORD: "admin"
strategy:
matrix:
go-version: [1.16.x, 1.17.x]
steps:
- uses: actions/checkout@v2
- uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
- name: Test LDAP
env:
LDAP_TEST_SERVER: "localhost:389"
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
make test-iam
etcd-test:
name: Etcd Backend Tests with Go ${{ matrix.go-version }}
runs-on: ubuntu-latest
services:
etcd:
image: "quay.io/coreos/etcd:v3.5.1"
env:
ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379"
ETCD_ADVERTISE_CLIENT_URLS: "http://0.0.0.0:2379"
ports:
- "2379:2379"
options: >-
--health-cmd "etcdctl endpoint health"
--health-interval 10s
--health-timeout 5s
--health-retries 5
strategy:
matrix:
go-version: [1.16.x, 1.17.x]
steps:
- uses: actions/checkout@v2
- uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
- name: Test Etcd IAM backend
env:
ETCD_SERVER: "http://localhost:2379"
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
make test-iam
iam-etcd-test:
name: Etcd Backend + LDAP Tests with Go ${{ matrix.go-version }}
runs-on: ubuntu-latest
services:
openldap:
image: quay.io/minio/openldap
image: donatello/openldap
ports:
- "389:389"
- "636:636"
@ -105,20 +38,41 @@ jobs:
--health-interval 10s
--health-timeout 5s
--health-retries 5
openid:
image: donatello/dex
ports:
- "5556:5556"
env:
DEX_LDAP_SERVER: "openldap:389"
strategy:
# When ldap, etcd or openid vars are empty below, those external servers
# are turned off - i.e. if ldap="", then ldap server is not enabled for
# the tests.
matrix:
go-version: [1.16.x, 1.17.x]
ldap: ["", "localhost:389"]
etcd: ["", "http://localhost:2379"]
openid: ["", "http://127.0.0.1:5556/dex"]
exclude:
# exclude combos where all are empty.
- ldap: ""
etcd: ""
openid: ""
# exclude combos where both ldap and openid IDPs are specified.
- ldap: "localhost:389"
openid: "http://127.0.0.1:5556/dex"
steps:
- uses: actions/checkout@v2
- uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
- name: Test Etcd IAM backend with LDAP IDP
- name: Test LDAP/OpenID with Etcd combo
env:
ETCD_SERVER: "http://localhost:2379"
LDAP_TEST_SERVER: "localhost:389"
LDAP_TEST_SERVER: ${{ matrix.ldap }}
ETCD_SERVER: ${{ matrix.etcd }}
OPENID_TEST_SERVER: ${{ matrix.openid }}
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0

282
cmd/sts-handlers_test.go
View file

@ -20,13 +20,18 @@ package cmd
import (
"context"
"fmt"
"net/http"
"net/url"
"os"
"strings"
"testing"
"time"
"github.com/coreos/go-oidc"
"github.com/minio/madmin-go"
minio "github.com/minio/minio-go/v7"
cr "github.com/minio/minio-go/v7/pkg/credentials"
"golang.org/x/oauth2"
)
func runAllIAMSTSTests(suite *TestSuiteIAM, c *check) {
@ -235,7 +240,7 @@ func (s *TestSuiteIAM) TestLDAPSTS(c *check) {
c.Fatalf("bucket create error: %v", err)
}
// Create policy, user and associate policy
// Create policy
policy := "mypolicy"
policyBytes := []byte(fmt.Sprintf(`{
"Version": "2012-10-17",
@ -267,7 +272,7 @@ func (s *TestSuiteIAM) TestLDAPSTS(c *check) {
_, err = ldapID.Retrieve()
if err == nil {
c.Fatalf("Expected to fail to create a user with no associated policy!")
c.Fatalf("Expected to fail to create STS cred with no associated policy!")
}
userDN := "uid=dillon,ou=people,ou=swengg,dc=min,dc=io"
@ -338,3 +343,276 @@ func (s *TestSuiteIAM) TestLDAPSTS(c *check) {
err = minioClient.RemoveObject(ctx, bucket, "someobject", minio.RemoveObjectOptions{})
c.Assert(err.Error(), "Access Denied.")
}
func (s *TestSuiteIAM) TestOpenIDSTS(c *check) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
bucket := getRandomBucketName()
err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
if err != nil {
c.Fatalf("bucket create error: %v", err)
}
// Generate web identity STS token by interacting with OpenID IDP.
token, err := mockTestUserInteraction(ctx, testProvider, "dillon@example.io", "dillon")
if err != nil {
c.Fatalf("mock user err: %v", err)
}
// fmt.Printf("TOKEN: %s\n", token)
webID := cr.STSWebIdentity{
Client: s.TestSuiteCommon.client,
STSEndpoint: s.endPoint,
GetWebIDTokenExpiry: func() (*cr.WebIdentityToken, error) {
return &cr.WebIdentityToken{
Token: token,
}, nil
},
}
// Create policy - with name as one of the groups in OpenID the user is
// a member of.
policy := "projecta"
policyBytes := []byte(fmt.Sprintf(`{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:PutObject",
"s3:GetObject",
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::%s/*"
]
}
]
}`, bucket))
err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
if err != nil {
c.Fatalf("policy add error: %v", err)
}
value, err := webID.Retrieve()
if err != nil {
c.Fatalf("Expected to generate STS creds, got err: %#v", err)
}
minioClient, err := minio.New(s.endpoint, &minio.Options{
Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken),
Secure: s.secure,
Transport: s.TestSuiteCommon.client.Transport,
})
if err != nil {
c.Fatalf("Error initializing client: %v", err)
}
// Validate that the client from sts creds can access the bucket.
c.mustListObjects(ctx, minioClient, bucket)
// Validate that the client cannot remove any objects
err = minioClient.RemoveObject(ctx, bucket, "someobject", minio.RemoveObjectOptions{})
if err.Error() != "Access Denied." {
c.Fatalf("unexpected non-access-denied err: %v", err)
}
}
type providerParams struct {
clientID, clientSecret, providerURL, redirectURL string
}
var testProvider = providerParams{
clientID: "minio-client-app",
clientSecret: "minio-client-app-secret",
providerURL: "http://127.0.0.1:5556/dex",
redirectURL: "http://127.0.0.1:10000/oauth_callback",
}
// mockTestUserInteraction - tries to login to dex using provided credentials.
// It performs the user's browser interaction to login and retrieves the auth
// code from dex and exchanges it for a JWT.
func mockTestUserInteraction(ctx context.Context, pro providerParams, username, password string) (string, error) {
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
provider, err := oidc.NewProvider(ctx, pro.providerURL)
if err != nil {
return "", fmt.Errorf("unable to create provider: %v", err)
}
// Configure an OpenID Connect aware OAuth2 client.
oauth2Config := oauth2.Config{
ClientID: pro.clientID,
ClientSecret: pro.clientSecret,
RedirectURL: pro.redirectURL,
// Discovery returns the OAuth2 endpoints.
Endpoint: provider.Endpoint(),
// "openid" is a required scope for OpenID Connect flows.
Scopes: []string{oidc.ScopeOpenID, "groups"},
}
state := "xxx"
authCodeURL := oauth2Config.AuthCodeURL(state)
// fmt.Printf("authcodeurl: %s\n", authCodeURL)
var lastReq *http.Request
checkRedirect := func(req *http.Request, via []*http.Request) error {
// fmt.Printf("CheckRedirect:\n")
// fmt.Printf("Upcoming: %s %#v\n", req.URL.String(), req)
// for _, c := range via {
// fmt.Printf("Sofar: %s %#v\n", c.URL.String(), c)
// }
// Save the last request in a redirect chain.
lastReq = req
// We do not follow redirect back to client application.
if req.URL.Path == "/oauth_callback" {
return http.ErrUseLastResponse
}
return nil
}
dexClient := http.Client{
CheckRedirect: checkRedirect,
}
u, err := url.Parse(authCodeURL)
if err != nil {
return "", fmt.Errorf("url parse err: %v", err)
}
// Start the user auth flow. This page would present the login with
// email or LDAP option.
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
if err != nil {
return "", fmt.Errorf("new request err: %v", err)
}
_, err = dexClient.Do(req)
// fmt.Printf("Do: %#v %#v\n", resp, err)
if err != nil {
return "", fmt.Errorf("auth url request err: %v", err)
}
// Modify u to choose the ldap option
u.Path += "/ldap"
// fmt.Println(u)
// Pick the LDAP login option. This would return a form page after
// following some redirects. `lastReq` would be the URL of the form
// page, where we need to POST (submit) the form.
req, err = http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
if err != nil {
return "", fmt.Errorf("new request err (/ldap): %v", err)
}
_, err = dexClient.Do(req)
if err != nil {
return "", fmt.Errorf("request err: %v", err)
}
// Fill the login form with our test creds:
// fmt.Printf("login form url: %s\n", lastReq.URL.String())
formData := url.Values{}
formData.Set("login", username)
formData.Set("password", password)
req, err = http.NewRequestWithContext(ctx, http.MethodPost, lastReq.URL.String(), strings.NewReader(formData.Encode()))
if err != nil {
return "", fmt.Errorf("new request err (/login): %v", err)
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
_, err = dexClient.Do(req)
if err != nil {
return "", fmt.Errorf("post form err: %v", err)
}
// fmt.Printf("resp: %#v %#v\n", resp.StatusCode, resp.Header)
// fmt.Printf("lastReq: %#v\n", lastReq.URL.String())
// On form submission, the last redirect response contains the auth
// code, which we now have in `lastReq`. Exchange it for a JWT id_token.
q := lastReq.URL.Query()
code := q.Get("code")
oauth2Token, err := oauth2Config.Exchange(ctx, code)
if err != nil {
return "", fmt.Errorf("unable to exchange code for id token: %v", err)
}
rawIDToken, ok := oauth2Token.Extra("id_token").(string)
if !ok {
return "", fmt.Errorf("id_token not found!")
}
// fmt.Printf("TOKEN: %s\n", rawIDToken)
return rawIDToken, nil
}
const (
EnvTestOpenIDServer = "OPENID_TEST_SERVER"
)
// SetUpOpenID - expects to setup an OpenID test server using the test OpenID
// container and canned data from https://github.com/minio/minio-ldap-testing
func (s *TestSuiteIAM) SetUpOpenID(c *check, serverAddr string) {
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
defer cancel()
configCmds := []string{
"identity_openid",
fmt.Sprintf("config_url=%s/.well-known/openid-configuration", serverAddr),
"client_id=minio-client-app",
"client_secret=minio-client-app-secret",
"claim_name=groups",
"scopes=openid,groups",
"redirect_uri=http://127.0.0.1:10000/oauth_callback",
}
_, err := s.adm.SetConfigKV(ctx, strings.Join(configCmds, " "))
if err != nil {
c.Fatalf("unable to setup OpenID for tests: %v", err)
}
s.RestartIAMSuite(c)
}
func TestIAMWithOpenIDServerSuite(t *testing.T) {
baseTestCases := []TestSuiteCommon{
// Init and run test on FS backend with signature v4.
{serverType: "FS", signer: signerV4},
// Init and run test on FS backend, with tls enabled.
{serverType: "FS", signer: signerV4, secure: true},
// Init and run test on Erasure backend.
{serverType: "Erasure", signer: signerV4},
// Init and run test on ErasureSet backend.
{serverType: "ErasureSet", signer: signerV4},
}
testCases := []*TestSuiteIAM{}
for _, bt := range baseTestCases {
testCases = append(testCases,
newTestSuiteIAM(bt, false),
newTestSuiteIAM(bt, true),
)
}
for i, testCase := range testCases {
etcdStr := ""
if testCase.withEtcdBackend {
etcdStr = " (with etcd backend)"
}
t.Run(
fmt.Sprintf("Test: %d, ServerType: %s%s", i+1, testCase.serverType, etcdStr),
func(t *testing.T) {
c := &check{t, testCase.serverType}
suite := testCase
openIDServer := os.Getenv(EnvTestOpenIDServer)
if openIDServer == "" {
c.Skip("Skipping OpenID test as no OpenID server is provided.")
}
suite.SetUpSuite(c)
suite.SetUpOpenID(c, openIDServer)
suite.TestOpenIDSTS(c)
suite.TearDownSuite(c)
},
)
}
}

4
go.mod
View file

@ -16,6 +16,7 @@ require (
github.com/cheggaaa/pb v1.0.29
github.com/colinmarc/hdfs/v2 v2.2.0
github.com/coredns/coredns v1.4.0
github.com/coreos/go-oidc v2.1.0+incompatible
github.com/cosnicolaou/pbzip2 v1.0.1
github.com/dchest/siphash v1.2.1
github.com/djherbis/atime v1.0.0
@ -84,7 +85,8 @@ require (
go.uber.org/atomic v1.9.0
go.uber.org/zap v1.19.1
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
golang.org/x/net v0.0.0-20211020060615-d418f374d309 // indirect
golang.org/x/net v0.0.0-20211020060615-d418f374d309
golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f
golang.org/x/sys v0.0.0-20211020174200-9d6173849985
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac
google.golang.org/api v0.58.0

3
go.sum
View file

@ -265,6 +265,7 @@ github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkE
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM=
github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
@ -1276,6 +1277,7 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo=
github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021 h1:0XM1XL/OFFJjXsYXlG30spTkV/E9+gmd5GD1w2HE8xM=
github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.44.1/go.mod h1:3WYi4xqXxGGXWDdQIITnLNmuDzO5n6wYva9spVhR4fg=
github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.46.0/go.mod h1:3WYi4xqXxGGXWDdQIITnLNmuDzO5n6wYva9spVhR4fg=
@ -2214,6 +2216,7 @@ gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLv
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4=
gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=