maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Migrate config.json from config-dir to backend (#6195)

Browse source 
This PR is the first set of changes to move the config
to the backend, the changes use the existing `config.json`
allows it to be migrated such that we can save it in on
backend disks.

In future releases, we will slowly migrate out of the
current architecture.

Fixes #6182
This commit is contained in:
Harshavardhana 2018-08-14 21:41:47 -07:00 committed by Nitish Tiwari
parent 380524ae27
commit 0e02328c98
46 changed files with 818 additions and 845 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -22,3 +22,4 @@ parts/
prime/
stage/
.sia_temp/
config.json

4
buildscripts/verify-build.sh
View file

@ -71,6 +71,8 @@ function start_minio_erasure_sets()
function start_minio_dist_erasure_sets()
{
declare -a minio_pids
export MINIO_ACCESS_KEY=$ACCESS_KEY
export MINIO_SECRET_KEY=$SECRET_KEY
"${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets4" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets5" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets6" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets7" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets8" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets9" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets10" "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets11" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets12" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets13" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets14" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets15" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets16" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets17" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets18" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets19" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-9000.log" 2>&1 &
minio_pids[0]=$!
"${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets4" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets5" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets6" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets7" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets8" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets9" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets10" "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets11" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets12" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets13" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets14" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets15" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets16" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets17" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets18" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets19" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-9001.log" 2>&1 &
@ -99,6 +101,8 @@ function start_minio_dist_erasure_sets()
function start_minio_dist_erasure()
{
declare -a minio_pids
export MINIO_ACCESS_KEY=$ACCESS_KEY
export MINIO_SECRET_KEY=$SECRET_KEY
"${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9000.log" 2>&1 &
minio_pids[0]=$!
"${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9001.log" 2>&1 &

45
cmd/admin-handlers.go
View file

@ -25,6 +25,7 @@ import (
"io"
"net/http"
"net/url"
"path"
"sync"
"time"
@ -462,14 +463,22 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
// Get config.json - in distributed mode, the configuration
// occurring on a quorum of the servers is returned.
configBytes, err := getPeerConfig(globalAdminPeers)
configData, err := getPeerConfig(globalAdminPeers)
if err != nil {
logger.LogIf(context.Background(), err)
writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
return
}
writeSuccessResponseJSON(w, configBytes)
password := globalServerConfig.GetCredential().SecretKey
econfigData, err := madmin.EncryptServerConfigData(password, configData)
if err != nil {
logger.LogIf(context.Background(), err)
writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
return
}
writeSuccessResponseJSON(w, econfigData)
}
// toAdminAPIErrCode - converts errXLWriteQuorum error to admin API
@ -572,7 +581,13 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
return
}
configBytes := configBuf[:n]
password := globalServerConfig.GetCredential().SecretKey
configBytes, err := madmin.DecryptServerConfigData(password, bytes.NewReader(configBuf[:n]))
if err != nil {
logger.LogIf(ctx, err)
writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
return
}
// Validate JSON provided in the request body: check the
// client has not sent JSON objects with duplicate keys.
@ -669,8 +684,7 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
// Decode request body
var req madmin.SetCredsReq
err := json.NewDecoder(r.Body).Decode(&req)
if err != nil {
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
logger.LogIf(context.Background(), err)
writeErrorResponseJSON(w, ErrRequestBodyParse, r.URL)
return
@ -697,13 +711,28 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
// Update local credentials in memory.
globalServerConfig.SetCredential(creds)
if err = globalServerConfig.Save(getConfigFile()); err != nil {
writeErrorResponseJSON(w, ErrInternalError, r.URL)
// Construct path to config.json for the given bucket.
configFile := path.Join(bucketConfigPrefix, minioConfigFile)
transactionConfigFile := configFile + ".transaction"
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
// and configFile, take a transaction lock to avoid race.
objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
if err = objLock.GetLock(globalOperationTimeout); err != nil {
writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
return
}
if err = saveServerConfig(newObjectLayerFn(), globalServerConfig); err != nil {
objLock.Unlock()
writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
return
}
objLock.Unlock()
// Notify all other Minio peers to update credentials
for host, err := range globalNotificationSys.SetCredentials(creds) {
for host, err := range globalNotificationSys.LoadCredentials() {
reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", host.String())
ctx := logger.SetReqInfo(context.Background(), reqInfo)
logger.LogIf(ctx, err)

36
cmd/admin-handlers_test.go
View file

@ -193,17 +193,17 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
// reset global variables to start afresh.
resetTestGlobals()
// Initialize minio server config.
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
return nil, err
}
// Initializing objectLayer for HealFormatHandler.
objLayer, xlDirs, xlErr := initTestXLObjLayer()
if xlErr != nil {
return nil, xlErr
}
// Initialize minio server config.
if err := newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
return nil, err
}
// Initialize boot time
globalBootTime = UTCNow()
@ -230,17 +230,15 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
registerAdminRouter(adminRouter)
return &adminXLTestBed{
configPath: rootPath,
xlDirs: xlDirs,
objLayer: objLayer,
router: adminRouter,
xlDirs: xlDirs,
objLayer: objLayer,
router: adminRouter,
}, nil
}
// TearDown - method that resets the test bed for subsequent unit
// tests to start afresh.
func (atb *adminXLTestBed) TearDown() {
os.RemoveAll(atb.configPath)
removeRoots(atb.xlDirs)
resetTestGlobals()
}
@ -680,8 +678,14 @@ func TestSetConfigHandler(t *testing.T) {
queryVal := url.Values{}
queryVal.Set("config", "")
password := globalServerConfig.GetCredential().SecretKey
econfigJSON, err := madmin.EncryptServerConfigData(password, configJSON)
if err != nil {
t.Fatal(err)
}
req, err := buildAdminRequest(queryVal, http.MethodPut, "/config",
int64(len(configJSON)), bytes.NewReader(configJSON))
int64(len(econfigJSON)), bytes.NewReader(econfigJSON))
if err != nil {
t.Fatalf("Failed to construct set-config object request - %v", err)
}
@ -724,7 +728,7 @@ func TestSetConfigHandler(t *testing.T) {
// Check that a config with duplicate keys in an object return
// error.
{
invalidCfg := append(configJSON[:len(configJSON)-1], []byte(`, "version": "15"}`)...)
invalidCfg := append(econfigJSON[:len(econfigJSON)-1], []byte(`, "version": "15"}`)...)
req, err := buildAdminRequest(queryVal, http.MethodPut, "/config",
int64(len(invalidCfg)), bytes.NewReader(invalidCfg))
if err != nil {
@ -823,11 +827,15 @@ func TestToAdminAPIErr(t *testing.T) {
}
func TestWriteSetConfigResponse(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatalf("unable initialize config file, %s", err)
}
testCases := []struct {
status bool
errs []error

9
cmd/api-router.go
View file

@ -20,7 +20,6 @@ import (
"net/http"
"github.com/gorilla/mux"
"github.com/minio/minio/cmd/logger"
)
// objectAPIHandler implements and provides http handlers for S3 API.
@ -31,14 +30,6 @@ type objectAPIHandlers struct {
// registerAPIRouter - registers S3 compatible APIs.
func registerAPIRouter(router *mux.Router) {
var err error
var cacheConfig = globalServerConfig.GetCacheConfig()
if len(cacheConfig.Drives) > 0 {
// initialize the new disk cache objects.
globalCacheObjectAPI, err = newServerCacheObjects(cacheConfig)
logger.FatalIf(err, "Unable to initialize disk caching")
}
// Initialize API.
api := objectAPIHandlers{
ObjectAPI: newObjectLayerFn,

15
cmd/auth-handler_test.go
View file

@ -344,11 +344,14 @@ func mustNewSignedBadMD5Request(method string, urlStr string, contentLength int6
// Tests is requested authenticated function, tests replies for s3 errors.
func TestIsReqAuthenticated(t *testing.T) {
path, err := newTestConfig(globalMinioDefaultRegion)
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatalf("unable initialize config file, %s", err)
}
defer os.RemoveAll(path)
creds, err := auth.CreateCredentials("myuser", "mypassword")
if err != nil {
@ -384,11 +387,15 @@ func TestIsReqAuthenticated(t *testing.T) {
}
}
func TestCheckAdminRequestAuthType(t *testing.T) {
path, err := newTestConfig(globalMinioDefaultRegion)
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatalf("unable initialize config file, %s", err)
}
defer os.RemoveAll(path)
creds, err := auth.CreateCredentials("myuser", "mypassword")
if err != nil {

60
cmd/benchmark-utils_test.go
View file

@ -22,7 +22,6 @@ import (
"io/ioutil"
"math"
"math/rand"
"os"
"strconv"
"testing"
@ -138,12 +137,6 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
// creates XL/FS backend setup, obtains the object layer and calls the runPutObjectPartBenchmark function.
func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
b.Fatalf("Unable to initialize config. %s", err)
}
defer os.RemoveAll(rootPath)
// create a temp XL/FS backend.
objLayer, disks, err := prepareBenchmarkBackend(instanceType)
if err != nil {
@ -151,18 +144,13 @@ func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
}
// cleaning up the backend by removing all the directories and files created on function return.
defer removeRoots(disks)
// uses *testing.B and the object Layer to run the benchmark.
runPutObjectPartBenchmark(b, objLayer, objSize)
}
// creates XL/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function.
func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
b.Fatalf("Unable to initialize config. %s", err)
}
defer os.RemoveAll(rootPath)
// create a temp XL/FS backend.
objLayer, disks, err := prepareBenchmarkBackend(instanceType)
if err != nil {
@ -170,18 +158,13 @@ func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
}
// cleaning up the backend by removing all the directories and files created on function return.
defer removeRoots(disks)
// uses *testing.B and the object Layer to run the benchmark.
runPutObjectBenchmark(b, objLayer, objSize)
}
// creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for put object.
func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
b.Fatalf("Unable to initialize config. %s", err)
}
defer os.RemoveAll(rootPath)
// create a temp XL/FS backend.
objLayer, disks, err := prepareBenchmarkBackend(instanceType)
if err != nil {
@ -189,6 +172,7 @@ func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int)
}
// cleaning up the backend by removing all the directories and files created on function return.
defer removeRoots(disks)
// uses *testing.B and the object Layer to run the benchmark.
runPutObjectBenchmarkParallel(b, objLayer, objSize)
}
@ -196,16 +180,10 @@ func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int)
// Benchmark utility functions for ObjectLayer.GetObject().
// Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark.
func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
b.Fatalf("Unable to initialize config. %s", err)
}
defer os.RemoveAll(rootPath)
// obtains random bucket name.
bucket := getRandomBucketName()
// create bucket.
err = obj.MakeBucketWithLocation(context.Background(), bucket, "")
err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
if err != nil {
b.Fatal(err)
}
@ -269,12 +247,6 @@ func generateBytesData(size int) []byte {
// creates XL/FS backend setup, obtains the object layer and calls the runGetObjectBenchmark function.
func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
b.Fatalf("Unable to initialize config. %s", err)
}
defer os.RemoveAll(rootPath)
// create a temp XL/FS backend.
objLayer, disks, err := prepareBenchmarkBackend(instanceType)
if err != nil {
@ -282,18 +254,13 @@ func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
}
// cleaning up the backend by removing all the directories and files created.
defer removeRoots(disks)
// uses *testing.B and the object Layer to run the benchmark.
runGetObjectBenchmark(b, objLayer, objSize)
}
// creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for ObjectLayer.GetObject() .
func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
b.Fatalf("Unable to initialize config. %s", err)
}
defer os.RemoveAll(rootPath)
// create a temp XL/FS backend.
objLayer, disks, err := prepareBenchmarkBackend(instanceType)
if err != nil {
@ -301,6 +268,7 @@ func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int)
}
// cleaning up the backend by removing all the directories and files created.
defer removeRoots(disks)
// uses *testing.B and the object Layer to run the benchmark.
runGetObjectBenchmarkParallel(b, objLayer, objSize)
}
@ -308,16 +276,10 @@ func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int)
// Parallel benchmark utility functions for ObjectLayer.PutObject().
// Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark.
func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
b.Fatalf("Unable to initialize config. %s", err)
}
defer os.RemoveAll(rootPath)
// obtains random bucket name.
bucket := getRandomBucketName()
// create bucket.
err = obj.MakeBucketWithLocation(context.Background(), bucket, "")
err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
if err != nil {
b.Fatal(err)
}
@ -359,16 +321,10 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
// Parallel benchmark utility functions for ObjectLayer.GetObject().
// Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark.
func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
b.Fatalf("Unable to initialize config. %s", err)
}
defer os.RemoveAll(rootPath)
// obtains random bucket name.
bucket := getRandomBucketName()
// create bucket.
err = obj.MakeBucketWithLocation(context.Background(), bucket, "")
err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
if err != nil {
b.Fatal(err)
}

39
cmd/common-main.go
View file

@ -17,7 +17,6 @@
package cmd
import (
"context"
"errors"
"net"
"os"
@ -48,39 +47,6 @@ func checkUpdate(mode string) {
}
}
// Initialize and load config from remote etcd or local config directory
func initConfig() {
if globalEtcdClient != nil {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
resp, err := globalEtcdClient.Get(ctx, getConfigFile())
cancel()
// This means there are no entries in etcd with config file
// So create a new config
if err == nil && resp.Count == 0 {
logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time.")
logger.Info("Created minio configuration file successfully at %v", globalEtcdClient.Endpoints())
} else {
// This means there is an entry in etcd, update it if required and proceed
if err == nil && resp.Count > 0 {
logger.FatalIf(migrateConfig(), "Config migration failed.")
logger.FatalIf(loadConfig(), "Unable to load config version: '%s'.", serverConfigVersion)
} else {
logger.FatalIf(err, "Unable to load config version: '%s'.", serverConfigVersion)
}
}
return
}
if isFile(getConfigFile()) {
logger.FatalIf(migrateConfig(), "Config migration failed")
logger.FatalIf(loadConfig(), "Unable to load the configuration file")
} else {
// Config file does not exist, we create it fresh and return upon success.
logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time")
logger.Info("Created minio configuration file successfully at " + getConfigDir())
}
}
// Load logger targets based on user's configuration
func loadLoggers() {
if globalServerConfig.Logger.Console.Enabled {
@ -148,6 +114,11 @@ func handleCommonEnvVars() {
globalActiveCred = cred
}
// In distributed setup users need to set ENVs always.
if !globalIsEnvCreds && globalIsDistXL {
logger.Fatal(uiErrEnvCredentialsMissingServer(nil), "Unable to start distributed server mode")
}
if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
browserFlag, err := ParseBoolFlag(browser)
if err != nil {

233
cmd/config-current.go
View file

@ -29,7 +29,6 @@ import (
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/event"
"github.com/minio/minio/pkg/event/target"
"github.com/minio/minio/pkg/quick"
)
// Steps to move from version N to version N+1
@ -69,6 +68,10 @@ func (s *serverConfig) GetRegion() string {
// SetCredential sets new credential and returns the previous credential.
func (s *serverConfig) SetCredential(creds auth.Credentials) (prevCred auth.Credentials) {
if creds.IsValid() && globalActiveCred.IsValid() {
globalActiveCred = creds
}
// Save previous credential.
prevCred = s.Credential
@ -81,6 +84,9 @@ func (s *serverConfig) SetCredential(creds auth.Credentials) (prevCred auth.Cred
// GetCredentials get current credentials.
func (s *serverConfig) GetCredential() auth.Credentials {
if globalActiveCred.IsValid() {
return globalActiveCred
}
return s.Credential
}
@ -208,19 +214,35 @@ func (s *serverConfig) Validate() error {
return nil
}
// Save config file to corresponding backend
func Save(configFile string, data interface{}) error {
return quick.SaveConfig(data, configFile, globalEtcdClient)
}
func (s *serverConfig) loadFromEnvs() {
// If env is set override the credentials from config file.
if globalIsEnvCreds {
s.SetCredential(globalActiveCred)
}
// Load config from backend
func Load(configFile string, data interface{}) (quick.Config, error) {
return quick.LoadConfig(configFile, globalEtcdClient, data)
}
if globalIsEnvBrowser {
s.SetBrowser(globalIsBrowserEnabled)
}
// GetVersion gets config version from backend
func GetVersion(configFile string) (string, error) {
return quick.GetVersion(configFile, globalEtcdClient)
if globalIsEnvWORM {
s.SetWorm(globalWORMEnabled)
}
if globalIsEnvRegion {
s.SetRegion(globalServerRegion)
}
if globalIsEnvDomainName {
s.Domain = globalDomainName
}
if globalIsStorageClass {
s.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
}
if globalIsDiskCacheEnabled {
s.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
}
}
// Returns the string describing a difference with the given
@ -327,153 +349,82 @@ func newServerConfig() *serverConfig {
return srvCfg
}
// newConfig - initialize a new server config, saves env parameters if
// found, otherwise use default parameters
func newConfig() error {
// Initialize server config.
srvCfg, err := newQuickConfig(newServerConfig())
if err != nil {
return err
}
// If env is set override the credentials from config file.
if globalIsEnvCreds {
srvCfg.SetCredential(globalActiveCred)
}
if globalIsEnvBrowser {
srvCfg.SetBrowser(globalIsBrowserEnabled)
}
if globalIsEnvWORM {
srvCfg.SetWorm(globalWORMEnabled)
}
if globalIsEnvRegion {
srvCfg.SetRegion(globalServerRegion)
}
if globalIsEnvDomainName {
srvCfg.Domain = globalDomainName
}
if globalIsStorageClass {
srvCfg.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
}
if globalIsDiskCacheEnabled {
srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
}
// hold the mutex lock before a new config is assigned.
// Save the new config globally.
// unlock the mutex.
globalServerConfigMu.Lock()
globalServerConfig = srvCfg
globalServerConfigMu.Unlock()
// Save config into file.
return Save(getConfigFile(), globalServerConfig)
}
// newQuickConfig - initialize a new server config, with an allocated
// quick.Config interface.
func newQuickConfig(srvCfg *serverConfig) (*serverConfig, error) {
qcfg, err := quick.NewConfig(srvCfg, globalEtcdClient)
if err != nil {
return nil, err
}
srvCfg.Config = qcfg
return srvCfg, nil
}
// getValidConfig - returns valid server configuration
func getValidConfig() (*serverConfig, error) {
srvCfg := &serverConfig{
Region: globalMinioDefaultRegion,
Browser: true,
}
var err error
srvCfg, err = newQuickConfig(srvCfg)
if err != nil {
return nil, err
}
configFile := getConfigFile()
if err = srvCfg.Load(configFile); err != nil {
return nil, err
}
if err = srvCfg.Validate(); err != nil {
return nil, err
}
return srvCfg, nil
}
// loadConfig - loads a new config from disk, overrides params from env
// if found and valid
func loadConfig() error {
srvCfg, err := getValidConfig()
if err != nil {
return uiErrInvalidConfig(nil).Msg(err.Error())
}
// If env is set override the credentials from config file.
if globalIsEnvCreds {
srvCfg.SetCredential(globalActiveCred)
}
if globalIsEnvBrowser {
srvCfg.SetBrowser(globalIsBrowserEnabled)
}
if globalIsEnvRegion {
srvCfg.SetRegion(globalServerRegion)
}
if globalIsEnvDomainName {
srvCfg.Domain = globalDomainName
}
if globalIsStorageClass {
srvCfg.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
}
if globalIsDiskCacheEnabled {
srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
}
// hold the mutex lock before a new config is assigned.
globalServerConfigMu.Lock()
globalServerConfig = srvCfg
func (s *serverConfig) loadToCachedConfigs() {
if !globalIsEnvCreds {
globalActiveCred = globalServerConfig.GetCredential()
globalActiveCred = s.GetCredential()
}
if !globalIsEnvBrowser {
globalIsBrowserEnabled = globalServerConfig.GetBrowser()
globalIsBrowserEnabled = s.GetBrowser()
}
if !globalIsEnvWORM {
globalWORMEnabled = globalServerConfig.GetWorm()
globalWORMEnabled = s.GetWorm()
}
if !globalIsEnvRegion {
globalServerRegion = globalServerConfig.GetRegion()
globalServerRegion = s.GetRegion()
}
if !globalIsEnvDomainName {
globalDomainName = globalServerConfig.Domain
globalDomainName = s.Domain
}
if !globalIsStorageClass {
globalStandardStorageClass, globalRRStorageClass = globalServerConfig.GetStorageClass()
globalStandardStorageClass, globalRRStorageClass = s.GetStorageClass()
}
if !globalIsDiskCacheEnabled {
cacheConf := globalServerConfig.GetCacheConfig()
cacheConf := s.GetCacheConfig()
globalCacheDrives = cacheConf.Drives
globalCacheExcludes = cacheConf.Exclude
globalCacheExpiry = cacheConf.Expiry
globalCacheMaxUse = cacheConf.MaxUse
}
}
// newConfig - initialize a new server config, saves env parameters if
// found, otherwise use default parameters
func newConfig(objAPI ObjectLayer) error {
// Initialize server config.
srvCfg := newServerConfig()
// Override any values from ENVs.
srvCfg.loadFromEnvs()
// Load values to cached global values.
srvCfg.loadToCachedConfigs()
// hold the mutex lock before a new config is assigned.
globalServerConfigMu.Lock()
globalServerConfig = srvCfg
globalServerConfigMu.Unlock()
// Save config into file.
return saveServerConfig(objAPI, globalServerConfig)
}
// getValidConfig - returns valid server configuration
func getValidConfig(objAPI ObjectLayer) (*serverConfig, error) {
srvCfg, err := readServerConfig(context.Background(), objAPI)
if err != nil {
return nil, err
}
return srvCfg, srvCfg.Validate()
}
// loadConfig - loads a new config from disk, overrides params from env
// if found and valid
func loadConfig(objAPI ObjectLayer) error {
srvCfg, err := getValidConfig(objAPI)
if err != nil {
return uiErrInvalidConfig(nil).Msg(err.Error())
}
// Override any values from ENVs.
srvCfg.loadFromEnvs()
// Load values to cached global values.
srvCfg.loadToCachedConfigs()
// hold the mutex lock before a new config is assigned.
globalServerConfigMu.Lock()
globalServerConfig = srvCfg
globalServerConfigMu.Unlock()
return nil

65
cmd/config-current_test.go
View file

@ -17,9 +17,8 @@
package cmd
import (
"io/ioutil"
"os"
"path/filepath"
"path"
"testing"
"github.com/minio/minio/pkg/auth"
@ -27,12 +26,15 @@ import (
)
func TestServerConfig(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatalf("Init Test config failed")
}
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
if globalServerConfig.GetRegion() != globalMinioDefaultRegion {
t.Errorf("Expecting region `us-east-1` found %s", globalServerConfig.GetRegion())
@ -49,16 +51,12 @@ func TestServerConfig(t *testing.T) {
t.Errorf("Expecting version %s found %s", globalServerConfig.GetVersion(), serverConfigVersion)
}
// Attempt to save.
if err := globalServerConfig.Save(getConfigFile()); err != nil {
if err := saveServerConfig(objLayer, globalServerConfig); err != nil {
t.Fatalf("Unable to save updated config file %s", err)
}
// Do this only once here.
setConfigDir(rootPath)
// Initialize server config.
if err := loadConfig(); err != nil {
if err := loadConfig(objLayer); err != nil {
t.Fatalf("Unable to initialize from updated config file %s", err)
}
}
@ -82,23 +80,25 @@ func TestServerConfigWithEnvs(t *testing.T) {
defer resetGlobalIsEnvs()
// Get test root.
rootPath, err := getTestRoot()
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Error(err)
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatalf("Init Test config failed")
}
globalObjLayerMutex.Lock()
globalObjectAPI = objLayer
globalObjLayerMutex.Unlock()
serverHandleEnvVars()
// Do this only once here.
setConfigDir(rootPath)
// Init config
initConfig()
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
// Check if serverConfig has
if globalServerConfig.GetBrowser() {
t.Errorf("Expecting browser is set to false found %v", globalServerConfig.GetBrowser())
@ -127,16 +127,17 @@ func TestServerConfigWithEnvs(t *testing.T) {
// Tests config validator..
func TestValidateConfig(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatalf("Init Test config failed")
}
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
configPath := filepath.Join(rootPath, minioConfigFile)
configPath := path.Join(minioConfigPrefix, minioConfigFile)
v := serverConfigVersion
testCases := []struct {
@ -226,14 +227,14 @@ func TestValidateConfig(t *testing.T) {
}
for i, testCase := range testCases {
if werr := ioutil.WriteFile(configPath, []byte(testCase.configData), 0700); werr != nil {
t.Fatal(werr)
if err = saveConfig(objLayer, configPath, []byte(testCase.configData)); err != nil {
t.Fatal(err)
}
_, verr := getValidConfig()
if testCase.shouldPass && verr != nil {
t.Errorf("Test %d, should pass but it failed with err = %v", i+1, verr)
_, err = getValidConfig(objLayer)
if testCase.shouldPass && err != nil {
t.Errorf("Test %d, should pass but it failed with err = %v", i+1, err)
}
if !testCase.shouldPass && verr == nil {
if !testCase.shouldPass && err == nil {
t.Errorf("Test %d, should fail but it succeeded.", i+1)
}
}

3
cmd/config-dir.go
View file

@ -28,9 +28,6 @@ const (
// Default minio configuration directory where below configuration files/directories are stored.
defaultMinioConfigDir = ".minio"
// Minio configuration file.
minioConfigFile = "config.json"
// Directory contains below files/directories for HTTPS configuration.
certsDir = "certs"

18
cmd/config-migrate.go
View file

@ -33,6 +33,21 @@ import (
// DO NOT EDIT following message template, please open a github issue to discuss instead.
var configMigrateMSGTemplate = "Configuration file %s migrated from version '%s' to '%s' successfully."
// Save config file to corresponding backend
func Save(configFile string, data interface{}) error {
return quick.SaveConfig(data, configFile, globalEtcdClient)
}
// Load config from backend
func Load(configFile string, data interface{}) (quick.Config, error) {
return quick.LoadConfig(configFile, globalEtcdClient, data)
}
// GetVersion gets config version from backend
func GetVersion(configFile string) (string, error) {
return quick.GetVersion(configFile, globalEtcdClient)
}
// Migrates all config versions from "1" to "18".
func migrateConfig() error {
// Purge all configs with version '1',
@ -45,6 +60,9 @@ func migrateConfig() error {
// Load only config version information.
version, err := GetVersion(getConfigFile())
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}

66
cmd/config-migrate_test.go
View file

@ -25,15 +25,26 @@ import (
// Test if config v1 is purged
func TestServerConfigMigrateV1(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
err = newTestConfig(globalMinioDefaultRegion, objLayer)
if err != nil {
t.Fatalf("Init Test config failed")
}
// remove the root directory after the test ends.
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
setConfigDir(rootPath)
globalObjLayerMutex.Lock()
globalObjectAPI = objLayer
globalObjLayerMutex.Unlock()
// Create a V1 config json file and store it
configJSON := "{ \"version\":\"1\", \"accessKeyId\":\"abcde\", \"secretAccessKey\":\"abcdefgh\"}"
configPath := rootPath + "/fsUsers.json"
@ -45,13 +56,14 @@ func TestServerConfigMigrateV1(t *testing.T) {
if err := migrateConfig(); err != nil {
t.Fatal("Unexpected error: ", err)
}
// Check if config v1 is removed from filesystem
if _, err := os.Stat(configPath); err == nil || !os.IsNotExist(err) {
t.Fatal("Config V1 file is not purged")
}
// Initialize server config and check again if everything is fine
if err := loadConfig(); err != nil {
if err := loadConfig(objLayer); err != nil {
t.Fatalf("Unable to initialize from updated config file %s", err)
}
}
@ -59,20 +71,13 @@ func TestServerConfigMigrateV1(t *testing.T) {
// Test if all migrate code returns nil when config file does not
// exist
func TestServerConfigMigrateInexistentConfig(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
if err != nil {
t.Fatalf("Init Test config failed")
t.Fatal(err)
}
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
setConfigDir(rootPath)
configPath := rootPath + "/" + minioConfigFile
// Remove config file
if err := os.Remove(configPath); err != nil {
t.Fatal("Unexpected error: ", err)
}
if err := migrateV2ToV3(); err != nil {
t.Fatal("migrate v2 to v3 should succeed when no config file is found")
@ -153,14 +158,23 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
// Test if a config migration from v2 to v27 is successfully done
func TestServerConfigMigrateV2toV27(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
if err != nil {
t.Fatalf("Init Test config failed")
t.Fatal(err)
}
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
setConfigDir(rootPath)
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
globalObjLayerMutex.Lock()
globalObjectAPI = objLayer
globalObjLayerMutex.Unlock()
configPath := rootPath + "/" + minioConfigFile
// Create a corrupted config file
@ -186,8 +200,12 @@ func TestServerConfigMigrateV2toV27(t *testing.T) {
t.Fatal("Unexpected error: ", err)
}
if err := migrateConfigToMinioSys(); err != nil {
t.Fatal("Unexpected error: ", err)
}
// Initialize server config and check again if everything is fine
if err := loadConfig(); err != nil {
if err := loadConfig(newObjectLayerFn()); err != nil {
t.Fatalf("Unable to initialize from updated config file %s", err)
}
@ -208,13 +226,11 @@ func TestServerConfigMigrateV2toV27(t *testing.T) {
// Test if all migrate code returns error with corrupted config files
func TestServerConfigMigrateFaultyConfig(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
if err != nil {
t.Fatalf("Init Test config failed")
t.Fatal(err)
}
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
setConfigDir(rootPath)
configPath := rootPath + "/" + minioConfigFile
@ -303,13 +319,11 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
// Test if all migrate code returns error with corrupted config files
func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
if err != nil {
t.Fatalf("Init Test config failed")
t.Fatal(err)
}
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
setConfigDir(rootPath)
configPath := rootPath + "/" + minioConfigFile

261
cmd/config.go Normal file
View file

@ -0,0 +1,261 @@
/*
* Minio Cloud Storage, (C) 2018 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"io/ioutil"
"path"
"runtime"
"time"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/hash"
"github.com/minio/minio/pkg/quick"
)
const (
minioConfigPrefix = "config"
// Minio configuration file.
minioConfigFile = "config.json"
)
func saveServerConfig(objAPI ObjectLayer, config *serverConfig) error {
if err := quick.CheckData(config); err != nil {
return err
}
data, err := json.Marshal(config)
if err != nil {
return err
}
configFile := path.Join(minioConfigPrefix, minioConfigFile)
if globalEtcdClient != nil {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
_, err := globalEtcdClient.Put(ctx, configFile, string(data))
defer cancel()
return err
}
return saveConfig(objAPI, configFile, data)
}
func readConfigEtcd(configFile string) ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
resp, err := globalEtcdClient.Get(ctx, configFile)
defer cancel()
if err != nil {
return nil, err
}
if resp.Count == 0 {
return nil, errConfigNotFound
}
for _, ev := range resp.Kvs {
if string(ev.Key) == configFile {
return ev.Value, nil
}
}
return nil, errConfigNotFound
}
func readServerConfig(ctx context.Context, objAPI ObjectLayer) (*serverConfig, error) {
var configData []byte
var err error
configFile := path.Join(minioConfigPrefix, minioConfigFile)
if globalEtcdClient != nil {
configData, err = readConfigEtcd(configFile)
} else {
var reader io.Reader
reader, err = readConfig(ctx, objAPI, configFile)
if err != nil {
return nil, err
}
configData, err = ioutil.ReadAll(reader)
}
if err != nil {
return nil, err
}
if runtime.GOOS == "windows" {
configData = bytes.Replace(configData, []byte("\r\n"), []byte("\n"), -1)
}
if err = quick.CheckDuplicateKeys(string(configData)); err != nil {
return nil, err
}
var config = &serverConfig{}
if err := json.Unmarshal(configData, config); err != nil {
return nil, err
}
if err := quick.CheckData(config); err != nil {
return nil, err
}
return config, nil
}
func checkServerConfigEtcd(configFile string) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
resp, err := globalEtcdClient.Get(ctx, configFile)
defer cancel()
if err != nil {
return err
}
if resp.Count == 0 {
return errConfigNotFound
}
return nil
}
func checkServerConfig(ctx context.Context, objAPI ObjectLayer) error {
configFile := path.Join(minioConfigPrefix, minioConfigFile)
if globalEtcdClient != nil {
return checkServerConfigEtcd(configFile)
}
if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile); err != nil {
if isErrObjectNotFound(err) {
return errConfigNotFound
}
logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
logger.LogIf(ctx, err)
return err
}
return nil
}
func saveConfig(objAPI ObjectLayer, configFile string, data []byte) error {
hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data))
if err != nil {
return err
}
_, err = objAPI.PutObject(context.Background(), minioMetaBucket, configFile, hashReader, nil)
return err
}
var errConfigNotFound = errors.New("config file not found")
func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) (*bytes.Buffer, error) {
var buffer bytes.Buffer
// Read entire content by setting size to -1
if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, ""); err != nil {
// Ignore if err is ObjectNotFound or IncompleteBody when bucket is not configured with notification
if isErrObjectNotFound(err) || isErrIncompleteBody(err) || isInsufficientReadQuorum(err) {
return nil, errConfigNotFound
}
logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
logger.LogIf(ctx, err)
return nil, err
}
// Return config not found on empty content.
if buffer.Len() == 0 {
return nil, errConfigNotFound
}
return &buffer, nil
}
// ConfigSys - config system.
type ConfigSys struct{}
// Load - load config.json.
func (sys *ConfigSys) Load(objAPI ObjectLayer) error {
return sys.Init(objAPI)
}
// Init - initializes config system from config.json.
func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
if objAPI == nil {
return errInvalidArgument
}
return loadConfig(objAPI)
}
// NewConfigSys - creates new config system object.
func NewConfigSys() *ConfigSys {
return &ConfigSys{}
}
// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/minio.json'
func migrateConfigToMinioSys() error {
// Construct path to config.json for the given bucket.
configFile := path.Join(bucketConfigPrefix, minioConfigFile)
transactionConfigFile := configFile + ".transaction"
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
// and configFile, take a transaction lock to avoid race.
objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
if err := objLock.GetLock(globalOperationTimeout); err != nil {
return err
}
defer objLock.Unlock()
// Verify if backend already has the file.
if err := checkServerConfig(context.Background(), newObjectLayerFn()); err != errConfigNotFound {
return err
} // if errConfigNotFound proceed to migrate..
var config = &serverConfig{}
if _, err := Load(getConfigFile(), config); err != nil {
return err
}
return saveServerConfig(newObjectLayerFn(), config)
}
// Initialize and load config from remote etcd or local config directory
func initConfig() {
if globalEtcdClient != nil {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
resp, err := globalEtcdClient.Get(ctx, getConfigFile())
cancel()
if err == nil && resp.Count > 0 {
logger.FatalIf(migrateConfig(), "Config migration failed")
}
} else {
if isFile(getConfigFile()) {
logger.FatalIf(migrateConfig(), "Config migration failed")
// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
if err := migrateConfigToMinioSys(); err != nil {
logger.Fatal(err, "Unable to migrate 'config.json' to '.minio.sys/config/config.json'")
}
}
}
if err := checkServerConfig(context.Background(), newObjectLayerFn()); err != nil {
if err == errConfigNotFound {
// Config file does not exist, we create it fresh and return upon success.
logger.FatalIf(newConfig(newObjectLayerFn()), "Unable to initialize minio config for the first time")
logger.Info("Created minio configuration file successfully at " + getConfigDir())
} else {
logger.FatalIf(err, "Unable to load the configuration file")
}
}
logger.FatalIf(loadConfig(newObjectLayerFn()), "Unable to load the configuration file")
}

18
cmd/disk-cache_test.go
View file

@ -19,7 +19,6 @@ package cmd
import (
"bytes"
"context"
"os"
"reflect"
"testing"
"time"
@ -28,21 +27,15 @@ import (
)
// Initialize cache FS objects.
func initCacheFSObjects(disk string, cacheMaxUse int, t *testing.T) (*cacheFSObjects, error) {
newTestConfig(globalMinioDefaultRegion)
var err error
obj, err := newCacheFSObjects(disk, globalCacheExpiry, cacheMaxUse)
if err != nil {
t.Fatal(err)
}
return obj, nil
func initCacheFSObjects(disk string, cacheMaxUse int) (*cacheFSObjects, error) {
return newCacheFSObjects(disk, globalCacheExpiry, cacheMaxUse)
}
// inits diskCache struct for nDisks
func initDiskCaches(drives []string, cacheMaxUse int, t *testing.T) (*diskCache, error) {
var cfs []*cacheFSObjects
for _, d := range drives {
obj, err := initCacheFSObjects(d, cacheMaxUse, t)
obj, err := initCacheFSObjects(d, cacheMaxUse)
if err != nil {
return nil, err
}
@ -131,11 +124,6 @@ func TestGetCacheFSMaxUse(t *testing.T) {
// test wildcard patterns for excluding entries from cache
func TestCacheExclusion(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
fsDirs, err := getRandomDisks(1)
if err != nil {
t.Fatal(err)

9
cmd/fs-v1-multipart_test.go
View file

@ -90,23 +90,18 @@ func TestNewMultipartUploadFaultyDisk(t *testing.T) {
// TestPutObjectPartFaultyDisk - test PutObjectPart with faulty disks
func TestPutObjectPartFaultyDisk(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
// Prepare for tests
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
defer os.RemoveAll(disk)
obj := initFSObjects(disk, t)
fs := obj.(*FSObjects)
bucketName := "bucket"
objectName := "object"
data := []byte("12345")
dataLen := int64(len(data))
if err = obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
t.Fatal("Cannot create bucket, err: ", err)
}

13
cmd/fs-v1_test.go
View file

@ -26,12 +26,6 @@ import (
// Tests for if parent directory is object
func TestFSParentDirIsObject(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
obj, disk, err := prepareFS()
if err != nil {
t.Fatal(err)
@ -120,12 +114,6 @@ func TestNewFS(t *testing.T) {
// TestFSShutdown - initialize a new FS object layer then calls
// Shutdown to check returned results
func TestFSShutdown(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
bucketName := "testbucket"
objectName := "object"
// Create and return an fsObject with its path in the disk
@ -133,6 +121,7 @@ func TestFSShutdown(t *testing.T) {
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
obj := initFSObjects(disk, t)
fs := obj.(*FSObjects)
objectContent := "12345"
obj.MakeBucketWithLocation(context.Background(), bucketName, "")
obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil)

18
cmd/gateway-main.go
View file

@ -150,17 +150,25 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
// Validate if we have access, secret set through environment.
if !globalIsEnvCreds {
logger.Fatal(uiErrEnvCredentialsMissing(nil), "Unable to start gateway")
logger.Fatal(uiErrEnvCredentialsMissingGateway(nil), "Unable to start gateway")
}
// Create certs path.
logger.FatalIf(createConfigDir(), "Unable to create configuration directories")
// Initialize gateway config.
initConfig()
// Initialize server config.
srvCfg := newServerConfig()
// Load logger subsystem
loadLoggers()
// Override any values from ENVs.
srvCfg.loadFromEnvs()
// Load values to cached global values.
srvCfg.loadToCachedConfigs()
// hold the mutex lock before a new config is assigned.
globalServerConfigMu.Lock()
globalServerConfig = srvCfg
globalServerConfigMu.Unlock()
// Check and load SSL certificates.
var err error

14
cmd/gateway-startup-msg_test.go
View file

@ -23,11 +23,14 @@ import (
// Test printing Gateway common message.
func TestPrintGatewayCommonMessage(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
apiEndpoints := []string{"http://127.0.0.1:9000"}
printGatewayCommonMsg(apiEndpoints)
@ -35,11 +38,14 @@ func TestPrintGatewayCommonMessage(t *testing.T) {
// Test print gateway startup message.
func TestPrintGatewayStartupMessage(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
apiEndpoints := []string{"http://127.0.0.1:9000"}
printGatewayStartupMessage(apiEndpoints, "azure")

3
cmd/globals.go
View file

@ -126,6 +126,9 @@ var (
// Holds the host that was passed using --address
globalMinioHost = ""
// globalConfigSys server config system.
globalConfigSys *ConfigSys
globalNotificationSys *NotificationSys
globalPolicySys *PolicySys

9
cmd/handler-utils_test.go
View file

@ -30,11 +30,14 @@ import (
// Tests validate bucket LocationConstraint.
func TestIsValidLocationContraint(t *testing.T) {
path, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatalf("unable initialize config file, %s", err)
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
defer os.RemoveAll(path)
// Test with corrupted XML
malformedReq := &http.Request{

37
cmd/jwt_test.go
View file

@ -25,11 +25,15 @@ import (
)
func testAuthenticate(authType string, t *testing.T) {
testPath, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatalf("unable initialize config file, %s", err)
t.Fatal(err)
}
defer os.RemoveAll(testPath)
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
cred, err := auth.GetNewCredentials()
if err != nil {
t.Fatalf("Error getting new credentials: %s", err)
@ -92,11 +96,14 @@ func TestAuthenticateURL(t *testing.T) {
// Tests web request authenticator.
func TestWebRequestAuthenticate(t *testing.T) {
testPath, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatalf("unable initialize config file, %s", err)
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
defer os.RemoveAll(testPath)
creds := globalServerConfig.GetCredential()
token, err := getTokenString(creds.AccessKey, creds.SecretKey)
@ -143,11 +150,14 @@ func TestWebRequestAuthenticate(t *testing.T) {
}
func BenchmarkAuthenticateNode(b *testing.B) {
testPath, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
b.Fatalf("unable initialize config file, %s", err)
b.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
b.Fatal(err)
}
defer os.RemoveAll(testPath)
creds := globalServerConfig.GetCredential()
b.ResetTimer()
@ -158,11 +168,14 @@ func BenchmarkAuthenticateNode(b *testing.B) {
}
func BenchmarkAuthenticateWeb(b *testing.B) {
testPath, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
b.Fatalf("unable initialize config file, %s", err)
b.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
b.Fatal(err)
}
defer os.RemoveAll(testPath)
creds := globalServerConfig.GetCredential()
b.ResetTimer()

16
cmd/lock-rpc-server_test.go
View file

@ -44,8 +44,11 @@ func testLockEquality(lriLeft, lriRight []lockRequesterInfo) bool {
// Helper function to create a lock server for testing
func createLockTestServer(t *testing.T) (string, *lockRPCReceiver, string) {
testPath, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatalf("unable initialize config file, %s", err)
}
@ -61,7 +64,7 @@ func createLockTestServer(t *testing.T) (string, *lockRPCReceiver, string) {
if err != nil {
t.Fatal(err)
}
return testPath, locker, token
return fsDir, locker, token
}
// Test Lock functionality
@ -470,11 +473,14 @@ func TestLockServerInit(t *testing.T) {
return
}
rootPath, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatalf("Init Test config failed")
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatalf("unable initialize config file, %s", err)
}
defer os.RemoveAll(rootPath)
currentIsDistXL := globalIsDistXL
currentLockServer := globalLockServer

45
cmd/notification.go
View file

@ -17,11 +17,9 @@
package cmd
import (
"bytes"
"context"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"net/url"
"path"
@ -29,9 +27,7 @@ import (
"time"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/event"
"github.com/minio/minio/pkg/hash"
xnet "github.com/minio/minio/pkg/net"
"github.com/minio/minio/pkg/policy"
)
@ -85,8 +81,8 @@ func (sys *NotificationSys) DeleteBucket(ctx context.Context, bucketName string)
}()
}
// SetCredentials - calls SetCredentials RPC call on all peers.
func (sys *NotificationSys) SetCredentials(credentials auth.Credentials) map[xnet.Host]error {
// LoadCredentials - calls LoadCredentials RPC call on all peers.
func (sys *NotificationSys) LoadCredentials() map[xnet.Host]error {
errors := make(map[xnet.Host]error)
var wg sync.WaitGroup
for addr, client := range sys.peerRPCClientMap {
@ -95,7 +91,7 @@ func (sys *NotificationSys) SetCredentials(credentials auth.Credentials) map[xne
defer wg.Done()
// Try to set credentials in three attempts.
for i := 0; i < 3; i++ {
err := client.SetCredentials(credentials)
err := client.LoadCredentials()
if err == nil {
break
}
@ -529,41 +525,6 @@ func sendEvent(args eventArgs) {
}()
}
func saveConfig(objAPI ObjectLayer, configFile string, data []byte) error {
hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data))
if err != nil {
return err
}
_, err = objAPI.PutObject(context.Background(), minioMetaBucket, configFile, hashReader, nil)
return err
}
var errConfigNotFound = errors.New("config file not found")
func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) (*bytes.Buffer, error) {
var buffer bytes.Buffer
// Read entire content by setting size to -1
err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "")
if err != nil {
// Ignore if err is ObjectNotFound or IncompleteBody when bucket is not configured with notification
if isErrObjectNotFound(err) || isErrIncompleteBody(err) || isInsufficientReadQuorum(err) {
return nil, errConfigNotFound
}
logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
logger.LogIf(ctx, err)
return nil, err
}
// Return NoSuchNotifications on empty content.
if buffer.Len() == 0 {
return nil, errNoSuchNotifications
}
return &buffer, nil
}
func readNotificationConfig(ctx context.Context, objAPI ObjectLayer, bucketName string) (*event.Config, error) {
// Construct path to notification.xml for the given bucket.
configFile := path.Join(bucketConfigPrefix, bucketName, bucketNotificationConfig)

6
cmd/object-api-listobjects_test.go
View file

@ -592,13 +592,7 @@ func BenchmarkListObjects(b *testing.B) {
b.Fatal(err)
}
defer os.RemoveAll(directory)
// initialize the root directory.
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
b.Fatalf("Unable to initialize config. %s", err)
}
defer os.RemoveAll(rootPath)
// Create the obj.
obj := initFSObjectsB(directory, b)

7
cmd/peer-rpc-client.go
View file

@ -21,7 +21,6 @@ import (
"crypto/tls"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/event"
xnet "github.com/minio/minio/pkg/net"
"github.com/minio/minio/pkg/policy"
@ -116,9 +115,9 @@ func (rpcClient *PeerRPCClient) SendEvent(bucketName string, targetID, remoteTar
return err
}
// SetCredentials - calls set credentials RPC.
func (rpcClient *PeerRPCClient) SetCredentials(credentials auth.Credentials) error {
args := SetCredentialsArgs{Credentials: credentials}
// LoadCredentials - calls load credentials RPC.
func (rpcClient *PeerRPCClient) LoadCredentials() error {
args := AuthArgs{}
reply := VoidReply{}
return rpcClient.Call(peerServiceName+".SetCredentials", &args, &reply)

37
cmd/peer-rpc-server.go
View file

@ -24,7 +24,6 @@ import (
"github.com/gorilla/mux"
"github.com/minio/minio/cmd/logger"
xrpc "github.com/minio/minio/cmd/rpc"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/event"
xnet "github.com/minio/minio/pkg/net"
"github.com/minio/minio/pkg/policy"
@ -158,35 +157,21 @@ func (receiver *peerRPCReceiver) SendEvent(args *SendEventArgs, reply *bool) err
return nil
}
// SetCredentialsArgs - set credentials RPC arguments.
type SetCredentialsArgs struct {
AuthArgs
Credentials auth.Credentials
}
// LoadCredentials - handles load credentials RPC call.
func (receiver *peerRPCReceiver) LoadCredentials(args *AuthArgs, reply *VoidReply) error {
// Construct path to config.json for the given bucket.
configFile := path.Join(bucketConfigPrefix, minioConfigFile)
transactionConfigFile := configFile + ".transaction"
// SetCredentials - handles set credentials RPC call.
func (receiver *peerRPCReceiver) SetCredentials(args *SetCredentialsArgs, reply *VoidReply) error {
if !args.Credentials.IsValid() {
return fmt.Errorf("invalid credentials passed")
}
// Acquire lock before updating global configuration.
globalServerConfigMu.Lock()
defer globalServerConfigMu.Unlock()
// Update credentials in memory
prevCred := globalServerConfig.SetCredential(args.Credentials)
// Save credentials to config file
if err := globalServerConfig.Save(getConfigFile()); err != nil {
// As saving configurstion failed, restore previous credential in memory.
globalServerConfig.SetCredential(prevCred)
logger.LogIf(context.Background(), err)
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
// and configFile, take a transaction lock to avoid race.
objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
if err := objLock.GetRLock(globalOperationTimeout); err != nil {
return err
}
objLock.RUnlock()
return nil
return globalConfigSys.Load(newObjectLayerFn())
}
// NewPeerRPCServer - returns new peer RPC server.

11
cmd/post-policy_test.go
View file

@ -26,7 +26,6 @@ import (
"net/http"
"net/http/httptest"
"net/url"
"os"
"testing"
"time"
@ -118,11 +117,9 @@ func TestPostPolicyBucketHandler(t *testing.T) {
// testPostPolicyBucketHandler - Tests validate post policy handler uploading objects.
func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
if err := newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatalf("Initializing config.json failed")
}
defer os.RemoveAll(root)
// get random bucket name.
bucketName := getRandomBucketName()
@ -139,7 +136,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr
// objectNames[0].
// uploadIds [0].
// Create bucket before initiating NewMultipartUpload.
err = obj.MakeBucketWithLocation(context.Background(), bucketName, "")
err := obj.MakeBucketWithLocation(context.Background(), bucketName, "")
if err != nil {
// Failed to create newbucket, abort.
t.Fatalf("%s : %s", instanceType, err.Error())
@ -420,11 +417,9 @@ func TestPostPolicyBucketHandlerRedirect(t *testing.T) {
// testPostPolicyBucketHandlerRedirect tests POST Object when success_action_redirect is specified
func testPostPolicyBucketHandlerRedirect(obj ObjectLayer, instanceType string, t TestErrHandler) {
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
if err := newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatalf("Initializing config.json failed")
}
defer os.RemoveAll(root)
// get random bucket name.
bucketName := getRandomBucketName()

37
cmd/server-main.go
View file

@ -211,15 +211,14 @@ func serverMain(ctx *cli.Context) {
// Handle all server environment vars.
serverHandleEnvVars()
// In distributed setup users need to set ENVs always.
if !globalIsEnvCreds && globalIsDistXL {
logger.Fatal(uiErrEnvCredentialsMissingServer(nil), "Unable to initialize minio server in distributed mode")
}
// Create certs path.
logger.FatalIf(createConfigDir(), "Unable to initialize configuration files")
// Initialize server config.
initConfig()
// Load logger subsystem
loadLoggers()
// Check and load SSL certificates.
var err error
globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()
@ -246,6 +245,11 @@ func serverMain(ctx *cli.Context) {
checkUpdate(mode)
}
// Enforce ENV credentials for distributed setup such that we can create the first config.
if globalIsDistXL && !globalIsEnvCreds {
logger.Fatal(uiErrInvalidCredentials(nil), "Unable to start the server in distrbuted mode. In distributed mode we require explicit credentials.")
}
// Set system resources to maximum.
logger.LogIf(context.Background(), setMaxResources())
@ -305,9 +309,30 @@ func serverMain(ctx *cli.Context) {
initFederatorBackend(newObject)
}
// Initialize server config.
initConfig()
// Load logger subsystem
loadLoggers()
var cacheConfig = globalServerConfig.GetCacheConfig()
if len(cacheConfig.Drives) > 0 {
// initialize the new disk cache objects.
globalCacheObjectAPI, err = newServerCacheObjects(cacheConfig)
logger.FatalIf(err, "Unable to initialize disk caching")
}
// Re-enable logging
logger.Disable = false
// Create a new config system.
globalConfigSys = NewConfigSys()
// Initialize config system.
if err := globalConfigSys.Init(newObjectLayerFn()); err != nil {
logger.Fatal(err, "Unable to initialize config system")
}
// Create new policy system.
globalPolicySys = NewPolicySys()

21
cmd/server-startup-msg_test.go
View file

@ -113,11 +113,14 @@ func TestStripStandardPorts(t *testing.T) {
// Test printing server common message.
func TestPrintServerCommonMessage(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
apiEndpoints := []string{"http://127.0.0.1:9000"}
printServerCommonMsg(apiEndpoints)
@ -125,11 +128,14 @@ func TestPrintServerCommonMessage(t *testing.T) {
// Tests print cli access message.
func TestPrintCLIAccessMsg(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
apiEndpoints := []string{"http://127.0.0.1:9000"}
printCLIAccessMsg(apiEndpoints[0], "myminio")
@ -137,11 +143,14 @@ func TestPrintCLIAccessMsg(t *testing.T) {
// Test print startup message.
func TestPrintStartupMessage(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
apiEndpoints := []string{"http://127.0.0.1:9000"}
printStartupMessage(apiEndpoints)

7
cmd/server_test.go
View file

@ -28,7 +28,6 @@ import (
"math/rand"
"net/http"
"net/url"
"os"
"reflect"
"strings"
"sync"
@ -46,7 +45,6 @@ type TestSuiteCommon struct {
endPoint string
accessKey string
secretKey string
configPath string
signer signerType
secure bool
transport *http.Transport
@ -145,9 +143,6 @@ func TestServerSuite(t *testing.T) {
// Setting up the test suite.
// Starting the Test server with temporary FS backend.
func (s *TestSuiteCommon) SetUpSuite(c *check) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
c.Assert(err, nil)
if s.secure {
cert, key, err := generateTLSCertKey("127.0.0.1")
c.Assert(err, nil)
@ -171,12 +166,10 @@ func (s *TestSuiteCommon) SetUpSuite(c *check) {
s.endPoint = s.testServer.Server.URL
s.accessKey = s.testServer.AccessKey
s.secretKey = s.testServer.SecretKey
s.configPath = rootPath
}
// Called implicitly by "gopkg.in/check.v1" after all tests are run.
func (s *TestSuiteCommon) TearDownSuite(c *check) {
os.RemoveAll(s.configPath)
s.testServer.Stop()
}

28
cmd/signature-v2_test.go
View file

@ -40,11 +40,14 @@ func TestResourceListSorting(t *testing.T) {
// Tests presigned v2 signature.
func TestDoesPresignedV2SignatureMatch(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal("Unable to initialize test config.")
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
now := UTCNow()
@ -157,11 +160,14 @@ func TestDoesPresignedV2SignatureMatch(t *testing.T) {
// TestValidateV2AuthHeader - Tests validate the logic of V2 Authorization header validator.
func TestValidateV2AuthHeader(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal("Unable to initialize test config.")
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
accessID := globalServerConfig.GetCredential().AccessKey
testCases := []struct {
@ -228,11 +234,15 @@ func TestValidateV2AuthHeader(t *testing.T) {
}
func TestDoesPolicySignatureV2Match(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal("Unable to initialize test config.")
t.Fatal(err)
}
defer os.RemoveAll(root)
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
creds := globalServerConfig.GetCredential()
policy := "policy"
testCases := []struct {

7
cmd/signature-v4_test.go
View file

@ -92,11 +92,14 @@ func TestDoesPolicySignatureMatch(t *testing.T) {
}
func TestDoesPresignedSignatureMatch(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
// sha256 hash of "payload"
payloadSHA256 := "239f59ed55e737c77147cf55ad0c1b030b6d7ee748a7426952f9b852d5a935e5"

207
cmd/test-utils_test.go
View file

@ -221,12 +221,12 @@ func prepareXL16() (ObjectLayer, []string, error) {
// Initialize FS objects.
func initFSObjects(disk string, t *testing.T) (obj ObjectLayer) {
newTestConfig(globalMinioDefaultRegion)
var err error
obj, err = NewFSObjectLayer(disk)
if err != nil {
t.Fatal(err)
}
newTestConfig(globalMinioDefaultRegion, obj)
return obj
}
@ -318,9 +318,9 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer {
if err != nil {
t.Fatal(err)
}
// set the server configuration.
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatalf("%s", err)
}
@ -332,7 +332,6 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer {
for _, disk := range disks {
testServer.Disks = append(testServer.Disks, mustGetNewEndpointList(disk)...)
}
testServer.Root = root
testServer.AccessKey = credentials.AccessKey
testServer.SecretKey = credentials.SecretKey
@ -396,98 +395,6 @@ func StartTestServer(t TestErrHandler, instanceType string) TestServer {
return testServer
}
// Initializes storage RPC endpoints.
// The object Layer will be a temp back used for testing purpose.
func initTestStorageRPCEndPoint(endpoints EndpointList) http.Handler {
// Initialize router.
muxRouter := mux.NewRouter().SkipClean(true)
registerStorageRPCRouters(muxRouter, endpoints)
return muxRouter
}
// StartTestStorageRPCServer - Creates a temp XL backend and initializes storage RPC end points,
// then starts a test server with those storage RPC end points registered.
func StartTestStorageRPCServer(t TestErrHandler, instanceType string, diskN int) TestServer {
// create temporary backend for the test server.
disks, err := getRandomDisks(diskN)
if err != nil {
t.Fatal("Failed to create disks for the backend")
}
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatalf("%s", err)
}
// Create an instance of TestServer.
testRPCServer := TestServer{}
// Get credential.
credentials := globalServerConfig.GetCredential()
endpoints := mustGetNewEndpointList(disks...)
testRPCServer.Root = root
testRPCServer.Disks = endpoints
testRPCServer.AccessKey = credentials.AccessKey
testRPCServer.SecretKey = credentials.SecretKey
// Run TestServer.
testRPCServer.Server = httptest.NewServer(initTestStorageRPCEndPoint(endpoints))
return testRPCServer
}
// Sets up a Peers RPC test server.
func StartTestPeersRPCServer(t TestErrHandler, instanceType string) TestServer {
// create temporary backend for the test server.
nDisks := 16
disks, err := getRandomDisks(nDisks)
if err != nil {
t.Fatal("Failed to create disks for the backend")
}
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatalf("%s", err)
}
// create an instance of TestServer.
testRPCServer := TestServer{}
// Get credential.
credentials := globalServerConfig.GetCredential()
endpoints := mustGetNewEndpointList(disks...)
testRPCServer.Root = root
testRPCServer.Disks = endpoints
testRPCServer.AccessKey = credentials.AccessKey
testRPCServer.SecretKey = credentials.SecretKey
// create temporary backend for the test server.
objLayer, _, err := initObjectLayer(endpoints)
if err != nil {
t.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
}
globalObjLayerMutex.Lock()
globalObjectAPI = objLayer
testRPCServer.Obj = objLayer
globalObjLayerMutex.Unlock()
router := mux.NewRouter().SkipClean(true)
// need storage layer for bucket config storage.
registerStorageRPCRouters(router, endpoints)
// need API layer to send requests, etc.
registerAPIRouter(router)
// module being tested is Peer RPCs router.
registerPeerRPCRouter(router)
// Run TestServer.
testRPCServer.Server = httptest.NewServer(router)
// initialize remainder of serverCmdConfig
testRPCServer.endpoints = endpoints
return testRPCServer
}
// Sets the global config path to empty string.
func resetGlobalConfigPath() {
setConfigDir("")
@ -584,31 +491,17 @@ func resetTestGlobals() {
}
// Configure the server for the test run.
func newTestConfig(bucketLocation string) (rootPath string, err error) {
// Get test root.
rootPath, err = getTestRoot()
if err != nil {
return "", err
}
// Do this only once here.
setConfigDir(rootPath)
func newTestConfig(bucketLocation string, obj ObjectLayer) (err error) {
// Initialize server config.
if err = newConfig(); err != nil {
return "", err
if err = newConfig(obj); err != nil {
return err
}
// Set a default region.
globalServerConfig.SetRegion(bucketLocation)
// Save config.
if err = globalServerConfig.Save(getConfigFile()); err != nil {
return "", err
}
// Return root path.
return rootPath, nil
return saveServerConfig(obj, globalServerConfig)
}
// Deleting the temporary backend and stopping the server.
@ -1985,12 +1878,6 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
// initialize NSLock.
initNSLock(false)
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatalf("Unable to initialize server config. %s", err)
}
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatalf("Initialization of object layer failed for single node setup: %s", err)
@ -1999,7 +1886,15 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
if err != nil {
t.Fatalf("Initialzation of API handler tests failed: <ERROR> %s", err)
}
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatalf("Unable to initialize server config. %s", err)
}
credentials := globalServerConfig.GetCredential()
// Executing the object layer tests for single node setup.
objAPITest(objLayer, FSTestStr, bucketFS, fsAPIRouter, credentials, t)
@ -2014,7 +1909,7 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
// Executing the object layer tests for XL.
objAPITest(objLayer, XLTestStr, bucketXL, xlAPIRouter, credentials, t)
// clean up the temporary test backend.
removeRoots(append(xlDisks, fsDir, rootPath))
removeRoots(append(xlDisks, fsDir))
}
// function to be passed to ExecObjectLayerAPITest, for executing object layr API handler tests.
@ -2033,19 +1928,17 @@ type objTestDiskNotFoundType func(obj ObjectLayer, instanceType string, dirs []s
// ExecObjectLayerTest - executes object layer tests.
// Creates single node and XL ObjectLayer instance and runs test for both the layers.
func ExecObjectLayerTest(t TestErrHandler, objTest objTestType) {
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal("Unexpected error", err)
}
defer os.RemoveAll(rootPath)
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatalf("Initialization of object layer failed for single node setup: %s", err)
}
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatal("Unexpected error", err)
}
// Executing the object layer tests for single node setup.
objTest(objLayer, FSTestStr, t)
@ -2061,42 +1954,34 @@ func ExecObjectLayerTest(t TestErrHandler, objTest objTestType) {
// ExecObjectLayerTestWithDirs - executes object layer tests.
// Creates single node and XL ObjectLayer instance and runs test for both the layers.
func ExecObjectLayerTestWithDirs(t TestErrHandler, objTest objTestTypeWithDirs) {
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal("Unexpected error", err)
}
defer os.RemoveAll(rootPath)
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatalf("Initialization of object layer failed for single node setup: %s", err)
}
objLayer, fsDirs, err := prepareXL16()
if err != nil {
t.Fatalf("Initialization of object layer failed for XL setup: %s", err)
}
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatal("Unexpected error", err)
}
// Executing the object layer tests for XL.
objTest(objLayer, XLTestStr, fsDirs, t)
defer removeRoots(append(fsDirs, fsDir))
defer removeRoots(fsDirs)
}
// ExecObjectLayerDiskAlteredTest - executes object layer tests while altering
// disks in between tests. Creates XL ObjectLayer instance and runs test for XL layer.
func ExecObjectLayerDiskAlteredTest(t *testing.T, objTest objTestDiskNotFoundType) {
configPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal("Failed to create config directory", err)
}
defer os.RemoveAll(configPath)
objLayer, fsDirs, err := prepareXL16()
if err != nil {
t.Fatalf("Initialization of object layer failed for XL setup: %s", err)
}
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatal("Failed to create config directory", err)
}
// Executing the object layer tests for XL.
objTest(objLayer, XLTestStr, fsDirs, t)
defer removeRoots(fsDirs)
@ -2108,12 +1993,6 @@ type objTestStaleFilesType func(obj ObjectLayer, instanceType string, dirs []str
// ExecObjectLayerStaleFilesTest - executes object layer tests those leaves stale
// files/directories under .minio/tmp. Creates XL ObjectLayer instance and runs test for XL layer.
func ExecObjectLayerStaleFilesTest(t *testing.T, objTest objTestStaleFilesType) {
configPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal("Failed to create config directory", err)
}
defer os.RemoveAll(configPath)
nDisks := 16
erasureDisks, err := getRandomDisks(nDisks)
if err != nil {
@ -2123,6 +2002,10 @@ func ExecObjectLayerStaleFilesTest(t *testing.T, objTest objTestStaleFilesType)
if err != nil {
t.Fatalf("Initialization of object layer failed for XL setup: %s", err)
}
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatal("Failed to create config directory", err)
}
// Executing the object layer tests for XL.
objTest(objLayer, XLTestStr, erasureDisks, t)
defer removeRoots(erasureDisks)
@ -2262,7 +2145,8 @@ func initTestWebRPCEndPoint(objLayer ObjectLayer) http.Handler {
}
func StartTestS3PeerRPCServer(t TestErrHandler) (TestServer, []string) {
root, err := newTestConfig(globalMinioDefaultRegion)
// init disks
objLayer, fsDirs, err := prepareXL16()
if err != nil {
t.Fatalf("%s", err)
}
@ -2270,18 +2154,15 @@ func StartTestS3PeerRPCServer(t TestErrHandler) (TestServer, []string) {
// Create an instance of TestServer.
testRPCServer := TestServer{}
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatalf("%s", err)
}
// Fetch credentials for the test server.
credentials := globalServerConfig.GetCredential()
testRPCServer.Root = root
testRPCServer.AccessKey = credentials.AccessKey
testRPCServer.SecretKey = credentials.SecretKey
// init disks
objLayer, fsDirs, err := prepareXL16()
if err != nil {
t.Fatalf("%s", err)
}
// set object layer
testRPCServer.Obj = objLayer
globalObjLayerMutex.Lock()

8
cmd/ui-errors.go
View file

@ -66,12 +66,18 @@ var (
Secret key should be in between 8 and 40 characters.`,
)
uiErrEnvCredentialsMissing = newUIErrFn(
uiErrEnvCredentialsMissingGateway = newUIErrFn(
"Credentials missing",
"Please provide correct credentials",
`Access key and Secret key should be specified in Gateway mode from environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively.`,
)
uiErrEnvCredentialsMissingServer = newUIErrFn(
"Credentials missing",
"Please provide correct credentials",
`Access key and Secret key should be specified in distributed server mode from environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively.`,
)
uiErrInvalidErasureEndpoints = newUIErrFn(
"Invalid endpoint(s) in erasure mode",
"Please provide correct combination of local/remote paths",

4
cmd/web-handlers.go
View file

@ -520,14 +520,14 @@ func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *Se
prevCred := globalServerConfig.SetCredential(creds)
// Persist updated credentials.
if err = globalServerConfig.Save(getConfigFile()); err != nil {
if err = saveServerConfig(newObjectLayerFn(), globalServerConfig); err != nil {
// Save the current creds when failed to update.
globalServerConfig.SetCredential(prevCred)
logger.LogIf(context.Background(), err)
return toJSONError(err)
}
if errs := globalNotificationSys.SetCredentials(creds); len(errs) != 0 {
if errs := globalNotificationSys.LoadCredentials(); len(errs) != 0 {
reply.PeerErrMsgs = make(map[string]string)
for host, err := range errs {
err = fmt.Errorf("Unable to update credentials on server %v: %v", host, err)

114
cmd/web-handlers_test.go
View file

@ -28,7 +28,6 @@ import (
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"reflect"
"strconv"
"strings"
@ -1506,14 +1505,13 @@ func TestWebCheckAuthorization(t *testing.T) {
// Register the API end points with XL/FS object layer.
apiRouter := initTestWebRPCEndPoint(obj)
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
rootPath, err := newTestConfig(globalMinioDefaultRegion)
err = newTestConfig(globalMinioDefaultRegion, obj)
if err != nil {
t.Fatal("Init Test config failed", err)
}
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
rec := httptest.NewRecorder()
@ -1585,100 +1583,8 @@ func TestWebCheckAuthorization(t *testing.T) {
}
}
// TestWebObjectLayerNotReady - Test RPCs responses when disks are not ready
func TestWebObjectLayerNotReady(t *testing.T) {
// Initialize web rpc endpoint.
apiRouter := initTestWebRPCEndPoint(nil)
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal("Init Test config failed", err)
}
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
rec := httptest.NewRecorder()
credentials := globalServerConfig.GetCredential()
authorization, err := getWebRPCToken(apiRouter, credentials.AccessKey, credentials.SecretKey)
if err != nil {
t.Fatal("Cannot authenticate", err)
}
// Check if web rpc calls return Server not initialized. ServerInfo, GenerateAuth,
// SetAuth and GetAuth are not concerned
webRPCs := []string{"StorageInfo", "MakeBucket", "ListBuckets", "ListObjects", "RemoveObject",
"GetBucketPolicy", "SetBucketPolicy", "ListAllBucketPolicies"}
for _, rpcCall := range webRPCs {
args := &AuthArgs{
RPCVersion: globalRPCAPIVersion,
}
reply := &WebGenericRep{}
req, nerr := newTestWebRPCRequest("Web."+rpcCall, authorization, args)
if nerr != nil {
t.Fatalf("Test %s: Failed to create HTTP request: <ERROR> %v", rpcCall, nerr)
}
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("Test %s: Expected the response status to be 200, but instead found `%d`", rpcCall, rec.Code)
}
err = getTestWebRPCResponse(rec, &reply)
if err == nil {
t.Fatalf("Test %s: Should fail", rpcCall)
} else {
if !strings.EqualFold(err.Error(), errServerNotInitialized.Error()) {
t.Fatalf("Test %s: should fail with %s Found error: %v", rpcCall, errServerNotInitialized, err)
}
}
}
rec = httptest.NewRecorder()
// Test authorization of Web.Download
req, err := http.NewRequest("GET", "/minio/download/bucket/object?token="+authorization, nil)
if err != nil {
t.Fatalf("Cannot create upload request, %v", err)
}
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("Expected the response status to be 503, but instead found `%d`", rec.Code)
}
resp := string(rec.Body.Bytes())
if !strings.EqualFold(resp, errServerNotInitialized.Error()) {
t.Fatalf("Unexpected error message, expected: `%s`, found: `%s`", errServerNotInitialized, resp)
}
rec = httptest.NewRecorder()
// Test authorization of Web.Upload
content := []byte("temporary file's content")
req, err = http.NewRequest("PUT", "/minio/upload/bucket/object", nil)
req.Header.Set("Authorization", "Bearer "+authorization)
req.Header.Set("Content-Length", strconv.Itoa(len(content)))
req.Header.Set("x-amz-date", "20160814T114029Z")
req.Header.Set("Accept", "*/*")
req.Body = ioutil.NopCloser(bytes.NewReader(content))
if err != nil {
t.Fatalf("Cannot create upload request, %v", err)
}
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("Expected the response status to be 503, but instead found `%d`", rec.Code)
}
resp = string(rec.Body.Bytes())
if !strings.EqualFold(resp, errServerNotInitialized.Error()) {
t.Fatalf("Unexpected error message, expected: `%s`, found: `%s`", errServerNotInitialized, resp)
}
}
// TestWebObjectLayerFaultyDisks - Test Web RPC responses with faulty disks
func TestWebObjectLayerFaultyDisks(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
// Prepare XL backend
obj, fsDirs, err := prepareXL16()
if err != nil {
@ -1687,6 +1593,13 @@ func TestWebObjectLayerFaultyDisks(t *testing.T) {
// Executing the object layer tests for XL.
defer removeRoots(fsDirs)
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
err = newTestConfig(globalMinioDefaultRegion, obj)
if err != nil {
t.Fatal("Init Test config failed", err)
}
bucketName := "mybucket"
err = obj.MakeBucketWithLocation(context.Background(), bucketName, "")
if err != nil {
@ -1702,15 +1615,6 @@ func TestWebObjectLayerFaultyDisks(t *testing.T) {
// Initialize web rpc endpoint.
apiRouter := initTestWebRPCEndPoint(obj)
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal("Init Test config failed", err)
}
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
rec := httptest.NewRecorder()
credentials := globalServerConfig.GetCredential()

6
cmd/xl-sets_test.go
View file

@ -101,12 +101,6 @@ func TestNewXLSets(t *testing.T) {
// TestHashedLayer - tests the hashed layer which will be returned
// consistently for a given object name.
func TestHashedLayer(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
var objs []*xlObjects
for i := 0; i < 16; i++ {

6
cmd/xl-v1-common_test.go
View file

@ -25,12 +25,6 @@ import (
// Tests for if parent directory is object
func TestXLParentDirIsObject(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
obj, fsDisks, err := prepareXL16()
if err != nil {
t.Fatalf("Unable to initialize 'XL' object layer.")

13
cmd/xl-v1-healing-common_test.go
View file

@ -19,7 +19,6 @@ package cmd
import (
"bytes"
"context"
"os"
"path/filepath"
"testing"
"time"
@ -111,12 +110,6 @@ func partsMetaFromModTimes(modTimes []time.Time, algorithm BitrotAlgorithm, chec
// TestListOnlineDisks - checks if listOnlineDisks and outDatedDisks
// are consistent with each other.
func TestListOnlineDisks(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatalf("Failed to initialize config - %v", err)
}
defer os.RemoveAll(rootPath)
obj, disks, err := prepareXL16()
if err != nil {
t.Fatalf("Prepare XL backend failed - %v", err)
@ -280,12 +273,6 @@ func TestListOnlineDisks(t *testing.T) {
func TestDisksWithAllParts(t *testing.T) {
ctx := context.Background()
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatalf("Failed to initialize config - %v", err)
}
defer os.RemoveAll(rootPath)
obj, disks, err := prepareXL16()
if err != nil {
t.Fatalf("Prepare XL backend failed - %v", err)

13
cmd/xl-v1-healing_test.go
View file

@ -19,19 +19,12 @@ package cmd
import (
"bytes"
"context"
"os"
"path/filepath"
"testing"
)
// Tests undoes and validates if the undoing completes successfully.
func TestUndoMakeBucket(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
nDisks := 16
fsDirs, err := getRandomDisks(nDisks)
if err != nil {
@ -65,12 +58,6 @@ func TestUndoMakeBucket(t *testing.T) {
// Tests healing of object.
func TestHealObjectXL(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
nDisks := 16
fsDirs, err := getRandomDisks(nDisks)
if err != nil {

8
cmd/xl-v1-multipart_test.go
View file

@ -18,20 +18,12 @@ package cmd
import (
"context"
"os"
"testing"
"time"
)
// Tests cleanup multipart uploads for erasure coded backend.
func TestXLCleanupStaleMultipartUploads(t *testing.T) {
// Initialize configuration
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatalf("%s", err)
}
defer os.RemoveAll(root)
// Create an instance of xl backend
obj, fsDirs, err := prepareXL16()
if err != nil {

6
cmd/xl-v1-object_test.go
View file

@ -262,12 +262,6 @@ func TestPutObjectNoQuorum(t *testing.T) {
// Tests both object and bucket healing.
func TestHealing(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatalf("Failed to initialize test config %v", err)
}
defer os.RemoveAll(rootPath)
obj, fsDirs, err := prepareXL16()
if err != nil {
t.Fatal(err)

64
pkg/madmin/config-commands.go
View file

@ -18,6 +18,8 @@
package madmin
import (
"bytes"
"crypto/rand"
"encoding/json"
"errors"
"fmt"
@ -26,6 +28,8 @@ import (
"net/http"
"github.com/minio/minio/pkg/quick"
"github.com/minio/sio"
"golang.org/x/crypto/argon2"
)
// NodeSummary - represents the result of an operation part of
@ -43,13 +47,48 @@ type SetConfigResult struct {
Status bool `json:"status"`
}
// GetConfig - returns the config.json of a minio setup.
func (adm *AdminClient) GetConfig() ([]byte, error) {
// No TLS?
if !adm.secure {
return nil, fmt.Errorf("credentials/configuration cannot be retrieved over an insecure connection")
// EncryptServerConfigData - encrypts server config data.
func EncryptServerConfigData(password string, data []byte) ([]byte, error) {
salt := make([]byte, 32)
if _, err := io.ReadFull(rand.Reader, salt); err != nil {
return nil, err
}
// derive an encryption key from the master key and the nonce
var key [32]byte
copy(key[:], argon2.IDKey([]byte(password), salt, 1, 64*1024, 4, 32))
encrypted, err := sio.EncryptReader(bytes.NewReader(data), sio.Config{
Key: key[:]},
)
if err != nil {
return nil, err
}
edata, err := ioutil.ReadAll(encrypted)
return append(salt, edata...), err
}
// DecryptServerConfigData - decrypts server config data.
func DecryptServerConfigData(password string, data io.Reader) ([]byte, error) {
salt := make([]byte, 32)
if _, err := io.ReadFull(data, salt); err != nil {
return nil, err
}
// derive an encryption key from the master key and the nonce
var key [32]byte
copy(key[:], argon2.IDKey([]byte(password), salt, 1, 64*1024, 4, 32))
decrypted, err := sio.DecryptReader(data, sio.Config{
Key: key[:]},
)
if err != nil {
return nil, err
}
return ioutil.ReadAll(decrypted)
}
// GetConfig - returns the config.json of a minio setup, incoming data is encrypted.
func (adm *AdminClient) GetConfig() ([]byte, error) {
// Execute GET on /minio/admin/v1/config to get config of a setup.
resp, err := adm.executeMethod("GET",
requestData{relPath: "/v1/config"})
@ -61,19 +100,15 @@ func (adm *AdminClient) GetConfig() ([]byte, error) {
if resp.StatusCode != http.StatusOK {
return nil, httpRespToErrorResponse(resp)
}
defer resp.Body.Close()
// Return the JSON marshaled bytes to user.
return ioutil.ReadAll(resp.Body)
return DecryptServerConfigData(adm.secretAccessKey, resp.Body)
}
// SetConfig - set config supplied as config.json for the setup.
func (adm *AdminClient) SetConfig(config io.Reader) (r SetConfigResult, err error) {
const maxConfigJSONSize = 256 * 1024 // 256KiB
if !adm.secure { // No TLS?
return r, fmt.Errorf("credentials/configuration cannot be updated over an insecure connection")
}
// Read configuration bytes
configBuf := make([]byte, maxConfigJSONSize+1)
n, err := io.ReadFull(config, configBuf)
@ -104,9 +139,14 @@ func (adm *AdminClient) SetConfig(config io.Reader) (r SetConfigResult, err erro
return r, errors.New("Duplicate key in json file: " + err.Error())
}
econfigBytes, err := EncryptServerConfigData(adm.secretAccessKey, configBytes)
if err != nil {
return r, err
}
reqData := requestData{
relPath: "/v1/config",
content: configBytes,
content: econfigBytes,
}
// Execute PUT on /minio/admin/v1/config to set config.

8
pkg/quick/quick.go
View file

@ -153,9 +153,9 @@ func (d config) DeepDiff(c Config) ([]structs.Field, error) {
return fields, nil
}
// checkData - checks the validity of config data. Data should be of
// CheckData - checks the validity of config data. Data should be of
// type struct and contain a string type field called "Version".
func checkData(data interface{}) error {
func CheckData(data interface{}) error {
if !structs.IsStruct(data) {
return fmt.Errorf("interface must be struct type")
}
@ -211,7 +211,7 @@ func LoadConfig(filename string, clnt *etcd.Client, data interface{}) (qc Config
// SaveConfig - saves given configuration data into given file as JSON.
func SaveConfig(data interface{}, filename string, clnt *etcd.Client) (err error) {
if err = checkData(data); err != nil {
if err = CheckData(data); err != nil {
return err
}
var qc Config
@ -225,7 +225,7 @@ func SaveConfig(data interface{}, filename string, clnt *etcd.Client) (err error
// NewConfig loads config from etcd client if provided, otherwise loads from a local filename.
// fails when all else fails.
func NewConfig(data interface{}, clnt *etcd.Client) (cfg Config, err error) {
if err := checkData(data); err != nil {
if err := CheckData(data); err != nil {
return nil, err
}

8
pkg/quick/quick_test.go
View file

@ -106,7 +106,7 @@ func TestSaveFailOnDir(t *testing.T) {
}
func TestCheckData(t *testing.T) {
err := checkData(nil)
err := CheckData(nil)
if err == nil {
t.Fatal("Unexpected should fail")
}
@ -117,7 +117,7 @@ func TestCheckData(t *testing.T) {
Directories []string
}
saveMeBadNoVersion := myStructBadNoVersion{"guest", "nopassword", []string{"Work", "Documents", "Music"}}
err = checkData(&saveMeBadNoVersion)
err = CheckData(&saveMeBadNoVersion)
if err == nil {
t.Fatal("Unexpected should fail if Version is not set")
}
@ -128,7 +128,7 @@ func TestCheckData(t *testing.T) {
Password string
}
saveMeBadVersionInt := myStructBadVersionInt{1, "guest", "nopassword"}
err = checkData(&saveMeBadVersionInt)
err = CheckData(&saveMeBadVersionInt)
if err == nil {
t.Fatal("Unexpected should fail if Version is integer")
}
@ -141,7 +141,7 @@ func TestCheckData(t *testing.T) {
}
saveMeGood := myStructGood{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
err = checkData(&saveMeGood)
err = CheckData(&saveMeGood)
if err != nil {
t.Fatal(err)
}