maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add support for reading and saving config on Gateway. (#4463)

Browse source 
This is also a first step towards supporting bucket
notification for gateway.
This commit is contained in:
Harshavardhana 2017-06-09 19:50:51 -07:00
parent 4fb5fc72d7
commit f99f218999
13 changed files with 325 additions and 420 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

117
cmd/common-main.go Normal file
View file

@ -0,0 +1,117 @@
/*
* Minio Cloud Storage, (C) 2017 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"errors"
"os"
"path/filepath"
"strings"
"time"
"github.com/minio/cli"
)
// Check for updates and print a notification message
func checkUpdate(mode string) {
// Its OK to ignore any errors during getUpdateInfo() here.
if older, downloadURL, err := getUpdateInfo(1*time.Second, mode); err == nil {
if updateMsg := computeUpdateMessage(downloadURL, older); updateMsg != "" {
log.Println(updateMsg)
}
}
}
func enableLoggers() {
fileLogTarget := serverConfig.Logger.GetFile()
if fileLogTarget.Enable {
err := InitFileLogger(&fileLogTarget)
fatalIf(err, "Unable to initialize file logger")
log.AddTarget(fileLogTarget)
}
consoleLogTarget := serverConfig.Logger.GetConsole()
if consoleLogTarget.Enable {
InitConsoleLogger(&consoleLogTarget)
}
log.SetConsoleTarget(consoleLogTarget)
}
func initConfig() {
// Config file does not exist, we create it fresh and return upon success.
if isFile(getConfigFile()) {
fatalIf(migrateConfig(), "Config migration failed.")
fatalIf(loadConfig(), "Unable to load config version: '%s'.", v19)
} else {
fatalIf(newConfig(), "Unable to initialize minio config for the first time.")
log.Println("Created minio configuration file successfully at " + getConfigDir())
}
}
func handleCommonCmdArgs(ctx *cli.Context) {
// Set configuration directory.
{
// Get configuration directory from command line argument.
configDir := ctx.String("config-dir")
if !ctx.IsSet("config-dir") && ctx.GlobalIsSet("config-dir") {
configDir = ctx.GlobalString("config-dir")
}
if configDir == "" {
fatalIf(errors.New("empty directory"), "Configuration directory cannot be empty.")
}
// Disallow relative paths, figure out absolute paths.
configDirAbs, err := filepath.Abs(configDir)
fatalIf(err, "Unable to fetch absolute path for config directory %s", configDir)
setConfigDir(configDirAbs)
}
}
func handleCommonEnvVars() {
// Start profiler if env is set.
if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" {
globalProfiler = startProfiler(profiler)
}
// Check if object cache is disabled.
globalXLObjCacheDisabled = strings.EqualFold(os.Getenv("_MINIO_CACHE"), "off")
accessKey := os.Getenv("MINIO_ACCESS_KEY")
secretKey := os.Getenv("MINIO_SECRET_KEY")
if accessKey != "" && secretKey != "" {
cred, err := createCredential(accessKey, secretKey)
fatalIf(err, "Invalid access/secret Key set in environment.")
// credential Envs are set globally.
globalIsEnvCreds = true
globalActiveCred = cred
}
if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
browserFlag, err := ParseBrowserFlag(browser)
if err != nil {
fatalIf(errors.New("invalid value"), "Unknown value ‘%s’ in MINIO_BROWSER environment variable.", browser)
}
// browser Envs are set globally, this does not represent
// if browser is turned off or on.
globalIsEnvBrowser = true
globalIsBrowserEnabled = bool(browserFlag)
}
}

25
cmd/gateway-azure.go
View file

@ -26,14 +26,12 @@ import (
"io"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"sync"
"time"
"github.com/Azure/azure-sdk-for-go/storage"
"github.com/minio/cli"
"github.com/minio/minio-go/pkg/policy"
"github.com/minio/sha256-simd"
)
@ -157,32 +155,30 @@ func azureToObjectError(err error, params ...string) error {
}
// Inits azure blob storage client and returns AzureObjects.
func newAzureLayer(args cli.Args) (GatewayLayer, error) {
func newAzureLayer(host string) (GatewayLayer, error) {
var err error
// Default endpoint parameters
endPoint := storage.DefaultBaseURL
secure := true
var endpoint = storage.DefaultBaseURL
var secure = true
// If user provided some parameters
if args.Present() {
endPoint, secure, err = parseGatewayEndpoint(args.First())
if host != "" {
endpoint, secure, err = parseGatewayEndpoint(host)
if err != nil {
return nil, err
}
}
account := os.Getenv("MINIO_ACCESS_KEY")
key := os.Getenv("MINIO_SECRET_KEY")
if account == "" || key == "" {
return nil, errors.New("No Azure account and key set")
creds := serverConfig.GetCredential()
if !creds.IsValid() && !globalIsEnvCreds {
return nil, errors.New("Azure backend account and secret keys should be set through ENVs")
}
c, err := storage.NewClient(account, key, endPoint, globalAzureAPIVersion, secure)
c, err := storage.NewClient(creds.AccessKey, creds.SecretKey, endpoint, globalAzureAPIVersion, secure)
if err != nil {
return &azureObjects{}, err
}
return &azureObjects{
client: c.GetBlobService(),
metaInfo: azureMultipartMetaInfo{
@ -195,7 +191,6 @@ func newAzureLayer(args cli.Args) (GatewayLayer, error) {
// Shutdown - save any gateway metadata to disk
// if necessary and reload upon next restart.
func (a *azureObjects) Shutdown() error {
// TODO
return nil
}

6
cmd/gateway-gcs-errors.go
View file

@ -19,9 +19,9 @@ package cmd
import "errors"
var (
// Project is format is not valid
errGCSInvalidProjectID = errors.New("invalid project id")
// ProjectID format is not valid.
errGCSInvalidProjectID = errors.New("GCS project id is either empty or invalid")
// Multipart identifier is not in the correct form
// Multipart identifier is not in the correct form.
errGCSNotValidMultipartIdentifier = errors.New("Not a valid multipart identifier")
)

23
cmd/gateway-gcs.go
View file

@ -33,7 +33,6 @@ import (
"google.golang.org/api/googleapi"
"google.golang.org/api/iterator"
"github.com/minio/cli"
minio "github.com/minio/minio-go"
"github.com/minio/minio-go/pkg/policy"
)
@ -175,29 +174,20 @@ type gcsGateway struct {
ctx context.Context
}
const googleStorageEndpoint = "storage.googleapis.com"
// newGCSGateway returns gcs gatewaylayer
func newGCSGateway(args cli.Args) (GatewayLayer, error) {
if !args.Present() {
return nil, fmt.Errorf("ProjectID expected")
}
endpoint := "storage.googleapis.com"
secure := true
projectID := args.First()
if !isValidGCSProjectID(projectID) {
fatalIf(errGCSInvalidProjectID, "Unable to initialize GCS gateway")
}
func newGCSGateway(projectID string) (GatewayLayer, error) {
ctx := context.Background()
// Creates a client.
// Initialize a GCS client.
client, err := storage.NewClient(ctx)
if err != nil {
return nil, err
}
anonClient, err := minio.NewCore(endpoint, "", "", secure)
// Initialize a anonymous client with minio core APIs.
anonClient, err := minio.NewCore(googleStorageEndpoint, "", "", true)
if err != nil {
return nil, err
}
@ -213,7 +203,6 @@ func newGCSGateway(args cli.Args) (GatewayLayer, error) {
// Shutdown - save any gateway metadata to disk
// if necessary and reload upon next restart.
func (l *gcsGateway) Shutdown() error {
// TODO
return nil
}

140
cmd/gateway-handlers.go
View file

@ -19,12 +19,12 @@ package cmd
import (
"io"
"io/ioutil"
"net"
"net/http"
"strconv"
"encoding/hex"
"encoding/json"
"encoding/xml"
router "github.com/gorilla/mux"
"github.com/minio/minio-go/pkg/policy"
@ -140,7 +140,7 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re
}
// Reads the object at startOffset and writes to mw.
if err := getObject(bucket, object, startOffset, length, writer); err != nil {
if err = getObject(bucket, object, startOffset, length, writer); err != nil {
errorIf(err, "Unable to write to client.")
if !dataWritten {
// Error response only if no data has been written to client yet. i.e if
@ -157,6 +157,23 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re
// call wrter.Write(nil) to set appropriate headers.
writer.Write(nil)
}
// Get host and port from Request.RemoteAddr.
host, port, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
host, port = "", ""
}
// Notify object accessed via a GET request.
eventNotify(eventData{
Type: ObjectAccessedGet,
Bucket: bucket,
ObjInfo: objInfo,
ReqParams: extractReqParams(r),
UserAgent: r.UserAgent(),
Host: host,
Port: port,
})
}
// PutObjectHandler - PUT Object
@ -293,6 +310,23 @@ func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Re
w.Header().Set("ETag", "\""+objInfo.ETag+"\"")
writeSuccessResponseHeadersOnly(w)
// Get host and port from Request.RemoteAddr.
host, port, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
host, port = "", ""
}
// Notify object created event.
eventNotify(eventData{
Type: ObjectCreatedPut,
Bucket: bucket,
ObjInfo: objInfo,
ReqParams: extractReqParams(r),
UserAgent: r.UserAgent(),
Host: host,
Port: port,
})
}
// HeadObjectHandler - HEAD Object
@ -361,97 +395,23 @@ func (api gatewayAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.R
// Successful response.
w.WriteHeader(http.StatusOK)
}
// DeleteMultipleObjectsHandler - deletes multiple objects.
func (api gatewayAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
vars := router.Vars(r)
bucket := vars["bucket"]
objectAPI := api.ObjectAPI()
if objectAPI == nil {
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
return
// Get host and port from Request.RemoteAddr.
host, port, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
host, port = "", ""
}
if s3Error := checkRequestAuthType(r, bucket, "s3:DeleteObject", serverConfig.GetRegion()); s3Error != ErrNone {
writeErrorResponse(w, s3Error, r.URL)
return
}
// Content-Length is required and should be non-zero
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
if r.ContentLength <= 0 {
writeErrorResponse(w, ErrMissingContentLength, r.URL)
return
}
// Content-Md5 is requied should be set
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
if _, ok := r.Header["Content-Md5"]; !ok {
writeErrorResponse(w, ErrMissingContentMD5, r.URL)
return
}
// Allocate incoming content length bytes.
deleteXMLBytes := make([]byte, r.ContentLength)
// Read incoming body XML bytes.
if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
errorIf(err, "Unable to read HTTP body.")
writeErrorResponse(w, ErrInternalError, r.URL)
return
}
// Unmarshal list of keys to be deleted.
deleteObjects := &DeleteObjectsRequest{}
if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
errorIf(err, "Unable to unmarshal delete objects request XML.")
writeErrorResponse(w, ErrMalformedXML, r.URL)
return
}
var dErrs = make([]error, len(deleteObjects.Objects))
// Delete all requested objects in parallel.
for index, object := range deleteObjects.Objects {
dErr := objectAPI.DeleteObject(bucket, object.ObjectName)
if dErr != nil {
dErrs[index] = dErr
}
}
// Collect deleted objects and errors if any.
var deletedObjects []ObjectIdentifier
var deleteErrors []DeleteError
for index, err := range dErrs {
object := deleteObjects.Objects[index]
// Success deleted objects are collected separately.
if err == nil {
deletedObjects = append(deletedObjects, object)
continue
}
if _, ok := errorCause(err).(ObjectNotFound); ok {
// If the object is not found it should be
// accounted as deleted as per S3 spec.
deletedObjects = append(deletedObjects, object)
continue
}
errorIf(err, "Unable to delete object. %s", object.ObjectName)
// Error during delete should be collected separately.
deleteErrors = append(deleteErrors, DeleteError{
Code: errorCodeResponse[toAPIErrorCode(err)].Code,
Message: errorCodeResponse[toAPIErrorCode(err)].Description,
Key: object.ObjectName,
})
}
// Generate response
response := generateMultiDeleteResponse(deleteObjects.Quiet, deletedObjects, deleteErrors)
encodedSuccessResponse := encodeResponse(response)
// Write success response.
writeSuccessResponseXML(w, encodedSuccessResponse)
// Notify object accessed via a HEAD request.
eventNotify(eventData{
Type: ObjectAccessedHead,
Bucket: bucket,
ObjInfo: objInfo,
ReqParams: extractReqParams(r),
UserAgent: r.UserAgent(),
Host: host,
Port: port,
})
}
// PutBucketPolicyHandler - PUT Bucket policy

222
cmd/gateway-main.go
View file

@ -20,7 +20,6 @@ import (
"errors"
"fmt"
"net/url"
"os"
"runtime"
"strings"
@ -60,20 +59,6 @@ EXAMPLES:
$ {{.HelpName}} https://azure.example.com
`
var azureBackendCmd = cli.Command{
Name: "azure",
Usage: "Microsoft Azure Blob Storage.",
Action: azureGatewayMain,
CustomHelpTemplate: azureGatewayTemplate,
Flags: append(serverFlags,
cli.BoolFlag{
Name: "quiet",
Usage: "Disable startup banner.",
},
),
HideHelpCommand: true,
}
const s3GatewayTemplate = `NAME:
{{.HelpName}} - {{.Usage}}
@ -134,40 +119,41 @@ EXAMPLES:
`
var s3BackendCmd = cli.Command{
Name: "s3",
Usage: "Amazon Simple Storage Service (S3).",
Action: s3GatewayMain,
CustomHelpTemplate: s3GatewayTemplate,
Flags: append(serverFlags,
cli.BoolFlag{
Name: "quiet",
Usage: "Disable startup banner.",
},
),
HideHelpCommand: true,
}
var (
azureBackendCmd = cli.Command{
Name: "azure",
Usage: "Microsoft Azure Blob Storage.",
Action: azureGatewayMain,
CustomHelpTemplate: azureGatewayTemplate,
Flags: append(serverFlags, globalFlags...),
HideHelpCommand: true,
}
var gcsBackendCmd = cli.Command{
Name: "gcs",
Usage: "Google Cloud Storage.",
Action: gcsGatewayMain,
CustomHelpTemplate: gcsGatewayTemplate,
Flags: append(serverFlags,
cli.BoolFlag{
Name: "quiet",
Usage: "Disable startup banner.",
},
),
HideHelpCommand: true,
}
s3BackendCmd = cli.Command{
Name: "s3",
Usage: "Amazon Simple Storage Service (S3).",
Action: s3GatewayMain,
CustomHelpTemplate: s3GatewayTemplate,
Flags: append(serverFlags, globalFlags...),
HideHelpCommand: true,
}
gcsBackendCmd = cli.Command{
Name: "gcs",
Usage: "Google Cloud Storage.",
Action: gcsGatewayMain,
CustomHelpTemplate: gcsGatewayTemplate,
Flags: append(serverFlags, globalFlags...),
HideHelpCommand: true,
}
var gatewayCmd = cli.Command{
Name: "gateway",
Usage: "Start object storage gateway.",
HideHelpCommand: true,
Subcommands: []cli.Command{azureBackendCmd, s3BackendCmd, gcsBackendCmd},
}
gatewayCmd = cli.Command{
Name: "gateway",
Usage: "Start object storage gateway.",
Flags: append(serverFlags, globalFlags...),
HideHelpCommand: true,
Subcommands: []cli.Command{azureBackendCmd, s3BackendCmd, gcsBackendCmd},
}
)
// Represents the type of the gateway backend.
type gatewayBackend string
@ -179,38 +165,6 @@ const (
// Add more backends here.
)
// Returns access and secretkey set from environment variables.
func mustGetGatewayConfigFromEnv() (string, string, string) {
// Fetch access keys from environment variables.
accessKey := os.Getenv("MINIO_ACCESS_KEY")
secretKey := os.Getenv("MINIO_SECRET_KEY")
if accessKey == "" || secretKey == "" {
fatalIf(errors.New("Missing credentials"), "Access and secret keys are mandatory to run Minio gateway server.")
}
region := globalMinioDefaultRegion
if v := os.Getenv("MINIO_REGION"); v != "" {
region = v
}
return accessKey, secretKey, region
}
// Set browser setting from environment variables
func mustSetBrowserSettingFromEnv() {
if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
browserFlag, err := ParseBrowserFlag(browser)
if err != nil {
fatalIf(errors.New("invalid value"), "Unknown value ‘%s’ in MINIO_BROWSER environment variable.", browser)
}
// browser Envs are set globally, this does not represent
// if browser is turned off or on.
globalIsEnvBrowser = true
globalIsBrowserEnabled = bool(browserFlag)
}
}
// Initialize gateway layer depending on the backend type.
// Supported backend types are
//
@ -218,46 +172,19 @@ func mustSetBrowserSettingFromEnv() {
// - AWS S3.
// - Google Cloud Storage.
// - Add your favorite backend here.
func newGatewayLayer(backendType string, args cli.Args) (GatewayLayer, error) {
switch gatewayBackend(backendType) {
func newGatewayLayer(backendType gatewayBackend, arg string) (GatewayLayer, error) {
switch backendType {
case azureBackend:
return newAzureLayer(args)
return newAzureLayer(arg)
case s3Backend:
return newS3Gateway(args)
return newS3Gateway(arg)
case gcsBackend:
return newGCSGateway(args)
return newGCSGateway(arg)
}
return nil, fmt.Errorf("Unrecognized backend type %s", backendType)
}
// Initialize a new gateway config.
//
// DO NOT save this config, this is meant to be
// only used in memory.
func newGatewayConfig(accessKey, secretKey, region string) error {
// Initialize server config.
srvCfg := newServerConfigV19()
// If env is set for a fresh start, save them to config file.
srvCfg.SetCredential(credential{
AccessKey: accessKey,
SecretKey: secretKey,
})
// Set custom region.
srvCfg.SetRegion(region)
// hold the mutex lock before a new config is assigned.
// Save the new config globally.
// unlock the mutex.
serverConfigMu.Lock()
serverConfig = srvCfg
serverConfigMu.Unlock()
return nil
}
// Return endpoint.
func parseGatewayEndpoint(arg string) (endPoint string, secure bool, err error) {
schemeSpecified := len(strings.Split(arg, "://")) > 1
@ -317,6 +244,9 @@ func azureGatewayMain(ctx *cli.Context) {
cli.ShowCommandHelpAndExit(ctx, "azure", 1)
}
// Validate gateway arguments.
fatalIf(validateGatewayArguments(ctx.String("address"), ctx.Args().First()), "Invalid argument")
gatewayMain(ctx, azureBackend)
}
@ -326,13 +256,21 @@ func s3GatewayMain(ctx *cli.Context) {
cli.ShowCommandHelpAndExit(ctx, "s3", 1)
}
// Validate gateway arguments.
fatalIf(validateGatewayArguments(ctx.String("address"), ctx.Args().First()), "Invalid argument")
gatewayMain(ctx, s3Backend)
}
// Handler for 'minio gateway gcs' command line
func gcsGatewayMain(ctx *cli.Context) {
if ctx.Args().Present() && ctx.Args().First() == "help" {
cli.ShowCommandHelpAndExit(ctx, "s3", 1)
cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
}
if !isValidGCSProjectID(ctx.Args().First()) {
errorIf(errGCSInvalidProjectID, "Unable to start GCS gateway with %s", ctx.Args().First())
cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
}
gatewayMain(ctx, gcsBackend)
@ -340,43 +278,45 @@ func gcsGatewayMain(ctx *cli.Context) {
// Handler for 'minio gateway'.
func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
// Fetch access and secret key from env.
accessKey, secretKey, region := mustGetGatewayConfigFromEnv()
// Fetch browser env setting
mustSetBrowserSettingFromEnv()
// Initialize new gateway config.
newGatewayConfig(accessKey, secretKey, region)
// Get quiet flag from command line argument.
quietFlag := ctx.Bool("quiet") || ctx.GlobalBool("quiet")
if quietFlag {
log.EnableQuiet()
}
serverAddr := ctx.String("address")
endpointAddr := ctx.Args().First()
err := validateGatewayArguments(serverAddr, endpointAddr)
fatalIf(err, "Invalid argument")
// Handle common command args.
handleCommonCmdArgs(ctx)
// Create certs path for SSL configuration.
fatalIf(createConfigDir(), "Unable to create configuration directory")
// Handle common env vars.
handleCommonEnvVars()
// Create certs path.
fatalIf(createConfigDir(), "Unable to create configuration directories.")
// Initialize gateway config.
initConfig()
// Enable loggers as per configuration file.
enableLoggers()
// Init the error tracing module.
initError()
// Check and load SSL certificates.
var err error
globalPublicCerts, globalRootCAs, globalIsSSL, err = getSSLConfig()
fatalIf(err, "Invalid SSL key file")
initNSLock(false) // Enable local namespace lock.
newObject, err := newGatewayLayer(backendType, ctx.Args()[1:])
newObject, err := newGatewayLayer(backendType, ctx.Args().First())
fatalIf(err, "Unable to initialize gateway layer")
router := mux.NewRouter().SkipClean(true)
// credentials Envs are set globally.
globalIsEnvCreds = true
// Register web router when its enabled.
if globalIsBrowserEnabled {
aerr := registerWebRouter(router)
fatalIf(aerr, "Unable to configure web browser")
fatalIf(registerWebRouter(router), "Unable to configure web browser")
}
registerGatewayAPIRouter(router, newObject)
@ -409,10 +349,7 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
}
apiServer := NewServerMux(serverAddr, registerHandlers(router, handlerFns...))
_, _, globalIsSSL, err = getSSLConfig()
fatalIf(err, "Invalid SSL key file")
apiServer := NewServerMux(ctx.String("address"), registerHandlers(router, handlerFns...))
// Start server, automatically configures TLS if certs are available.
go func() {
@ -420,9 +357,7 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
if globalIsSSL {
cert, key = getPublicCertFile(), getPrivateKeyFile()
}
aerr := apiServer.ListenAndServe(cert, key)
fatalIf(aerr, "Failed to start minio server")
fatalIf(apiServer.ListenAndServe(cert, key), "Failed to start minio server")
}()
// Once endpoints are finalized, initialize the new object api.
@ -441,9 +376,12 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
case s3Backend:
mode = globalMinioModeGatewayS3
}
// Check update mode.
checkUpdate(mode)
apiEndpoints := getAPIEndpoints(apiServer.Addr)
printGatewayStartupMessage(apiEndpoints, accessKey, secretKey, backendType)
// Print gateway startup message.
printGatewayStartupMessage(getAPIEndpoints(apiServer.Addr), backendType)
}
<-globalServiceDoneCh

23
cmd/gateway-main_test.go
View file

@ -17,7 +17,6 @@
package cmd
import (
"os"
"strings"
"testing"
)
@ -53,28 +52,6 @@ func TestParseGatewayEndpoint(t *testing.T) {
}
}
func TestSetBrowserFromEnv(t *testing.T) {
browser := os.Getenv("MINIO_BROWSER")
os.Setenv("MINIO_BROWSER", "on")
mustSetBrowserSettingFromEnv()
if globalIsBrowserEnabled != true {
t.Errorf("Expected the response status to be `%t`, but instead found `%t`", globalIsBrowserEnabled, false)
}
os.Setenv("MINIO_BROWSER", "off")
mustSetBrowserSettingFromEnv()
if globalIsBrowserEnabled != false {
t.Errorf("Expected the response status to be `%t`, but instead found `%t`", globalIsBrowserEnabled, true)
}
os.Setenv("MINIO_BROWSER", "")
mustSetBrowserSettingFromEnv()
if globalIsBrowserEnabled != false {
t.Errorf("Expected the response status to be `%t`, but instead found `%t`", globalIsBrowserEnabled, true)
}
os.Setenv("MINIO_BROWSER", browser)
}
// Test validateGatewayArguments
func TestValidateGatewayArguments(t *testing.T) {
nonLoopBackIPs := localIP4.FuncMatch(func(ip string, matchString string) bool {

32
cmd/gateway-s3.go
View file

@ -20,12 +20,10 @@ import (
"errors"
"io"
"net/http"
"os"
"path"
"encoding/hex"
"github.com/minio/cli"
minio "github.com/minio/minio-go"
"github.com/minio/minio-go/pkg/policy"
)
@ -101,31 +99,33 @@ type s3Objects struct {
}
// newS3Gateway returns s3 gatewaylayer
func newS3Gateway(args cli.Args) (GatewayLayer, error) {
func newS3Gateway(host string) (GatewayLayer, error) {
var err error
var endpoint string
var secure = true
// Default endpoint parameters
endpoint := "s3.amazonaws.com"
secure := true
// Check if user provided some parameters
if args.Present() {
// Override default params if the endpoint is provided
endpoint, secure, err = parseGatewayEndpoint(args.First())
// Validate host parameters.
if host != "" {
// Override default params if the host is provided
endpoint, secure, err = parseGatewayEndpoint(host)
if err != nil {
return nil, err
}
}
accessKey := os.Getenv("MINIO_ACCESS_KEY")
secretKey := os.Getenv("MINIO_SECRET_KEY")
if accessKey == "" || secretKey == "" {
return nil, errors.New("No S3 access and secret key")
// Default endpoint parameters
if endpoint == "" {
endpoint = "s3.amazonaws.com"
}
creds := serverConfig.GetCredential()
if !creds.IsValid() && !globalIsEnvCreds {
return nil, errors.New("S3 backend account and secret keys should be set through ENVs")
}
// Initialize minio client object.
client, err := minio.NewCore(endpoint, accessKey, secretKey, secure)
client, err := minio.NewCore(endpoint, creds.AccessKey, creds.SecretKey, secure)
if err != nil {
return nil, err
}

31
cmd/gateway-startup-msg.go
View file

@ -18,28 +18,19 @@ package cmd
import (
"fmt"
"runtime"
"strings"
)
// Prints the formatted startup message.
func printGatewayStartupMessage(apiEndPoints []string, accessKey, secretKey string, backendType gatewayBackend) {
func printGatewayStartupMessage(apiEndPoints []string, backendType gatewayBackend) {
strippedAPIEndpoints := stripStandardPorts(apiEndPoints)
// Prints credential.
printGatewayCommonMsg(apiEndPoints, accessKey, secretKey)
printGatewayCommonMsg(strippedAPIEndpoints)
// Prints `mc` cli configuration message chooses
// first endpoint as default.
endPoint := apiEndPoints[0]
// Configure 'mc', following block prints platform specific information for minio client.
log.Println(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide)
if runtime.GOOS == globalWindowsOSName {
mcMessage := fmt.Sprintf("$ mc.exe config host add my%s %s %s %s", backendType, endPoint, accessKey, secretKey)
log.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
} else {
mcMessage := fmt.Sprintf("$ mc config host add my%s %s %s %s", backendType, endPoint, accessKey, secretKey)
log.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
}
printCLIAccessMsg(strippedAPIEndpoints[0], fmt.Sprintf("my%s", backendType))
// Prints documentation message.
printObjectAPIMsg()
@ -52,10 +43,16 @@ func printGatewayStartupMessage(apiEndPoints []string, accessKey, secretKey stri
}
// Prints common server startup message. Prints credential, region and browser access.
func printGatewayCommonMsg(apiEndpoints []string, accessKey, secretKey string) {
func printGatewayCommonMsg(apiEndpoints []string) {
// Get saved credentials.
cred := serverConfig.GetCredential()
apiEndpointStr := strings.Join(apiEndpoints, " ")
// Colorize the message and print.
log.Println(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
log.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", accessKey)))
log.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", secretKey)))
log.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
log.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
log.Println(colorBlue("\nBrowser Access:"))
log.Println(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr))
}

20
cmd/gateway-startup-msg_test.go
View file

@ -20,12 +20,24 @@ import "testing"
// Test printing Gateway common message.
func TestPrintGatewayCommonMessage(t *testing.T) {
apiEndpoints := []string{"127.0.0.1:9000"}
printGatewayCommonMsg(apiEndpoints, "abcd1", "abcd123")
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer removeAll(root)
apiEndpoints := []string{"http://127.0.0.1:9000"}
printGatewayCommonMsg(apiEndpoints)
}
// Test print gateway startup message.
func TestPrintGatewayStartupMessage(t *testing.T) {
apiEndpoints := []string{"127.0.0.1:9000"}
printGatewayStartupMessage(apiEndpoints, "abcd1", "abcd123", "azure")
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer removeAll(root)
apiEndpoints := []string{"http://127.0.0.1:9000"}
printGatewayStartupMessage(apiEndpoints, "azure")
}

96
cmd/server-main.go
View file

@ -17,12 +17,8 @@
package cmd
import (
"errors"
"os"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/minio/cli"
"github.com/minio/dsync"
@ -78,61 +74,9 @@ EXAMPLES:
`,
}
// Check for updates and print a notification message
func checkUpdate(mode string) {
// Its OK to ignore any errors during getUpdateInfo() here.
if older, downloadURL, err := getUpdateInfo(1*time.Second, mode); err == nil {
if updateMsg := computeUpdateMessage(downloadURL, older); updateMsg != "" {
log.Println(updateMsg)
}
}
}
func enableLoggers() {
fileLogTarget := serverConfig.Logger.GetFile()
if fileLogTarget.Enable {
err := InitFileLogger(&fileLogTarget)
fatalIf(err, "Unable to initialize file logger")
log.AddTarget(fileLogTarget)
}
consoleLogTarget := serverConfig.Logger.GetConsole()
if consoleLogTarget.Enable {
InitConsoleLogger(&consoleLogTarget)
}
log.SetConsoleTarget(consoleLogTarget)
}
func initConfig() {
// Config file does not exist, we create it fresh and return upon success.
if isFile(getConfigFile()) {
fatalIf(migrateConfig(), "Config migration failed.")
fatalIf(loadConfig(), "Unable to load config version: '%s'.", v19)
} else {
fatalIf(newConfig(), "Unable to initialize minio config for the first time.")
log.Println("Created minio configuration file successfully at " + getConfigDir())
}
}
func serverHandleCmdArgs(ctx *cli.Context) {
// Set configuration directory.
{
// Get configuration directory from command line argument.
configDir := ctx.String("config-dir")
if !ctx.IsSet("config-dir") && ctx.GlobalIsSet("config-dir") {
configDir = ctx.GlobalString("config-dir")
}
if configDir == "" {
fatalIf(errors.New("empty directory"), "Configuration directory cannot be empty.")
}
// Disallow relative paths, figure out absolute paths.
configDirAbs, err := filepath.Abs(configDir)
fatalIf(err, "Unable to fetch absolute path for config directory %s", configDir)
setConfigDir(configDirAbs)
}
// Handle common command args.
handleCommonCmdArgs(ctx)
// Server address.
serverAddr := ctx.String("address")
@ -159,36 +103,8 @@ func serverHandleCmdArgs(ctx *cli.Context) {
}
func serverHandleEnvVars() {
// Start profiler if env is set.
if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" {
globalProfiler = startProfiler(profiler)
}
// Check if object cache is disabled.
globalXLObjCacheDisabled = strings.EqualFold(os.Getenv("_MINIO_CACHE"), "off")
accessKey := os.Getenv("MINIO_ACCESS_KEY")
secretKey := os.Getenv("MINIO_SECRET_KEY")
if accessKey != "" && secretKey != "" {
cred, err := createCredential(accessKey, secretKey)
fatalIf(err, "Invalid access/secret Key set in environment.")
// credential Envs are set globally.
globalIsEnvCreds = true
globalActiveCred = cred
}
if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
browserFlag, err := ParseBrowserFlag(browser)
if err != nil {
fatalIf(errors.New("invalid value"), "Unknown value ‘%s’ in MINIO_BROWSER environment variable.", browser)
}
// browser Envs are set globally, this does not represent
// if browser is turned off or on.
globalIsEnvBrowser = true
globalIsBrowserEnabled = bool(browserFlag)
}
// Handle common environment variables.
handleCommonEnvVars()
if serverRegion := os.Getenv("MINIO_REGION"); serverRegion != "" {
// region Envs are set globally.
@ -210,12 +126,16 @@ func serverMain(ctx *cli.Context) {
log.EnableQuiet()
}
// Handle all server command args.
serverHandleCmdArgs(ctx)
// Handle all server environment vars.
serverHandleEnvVars()
// Create certs path.
fatalIf(createConfigDir(), "Unable to create configuration directories.")
// Initialize server config.
initConfig()
// Enable loggers as per configuration file.

8
cmd/server-startup-msg.go
View file

@ -52,7 +52,7 @@ func printStartupMessage(apiEndPoints []string) {
// Prints `mc` cli configuration message chooses
// first endpoint as default.
printCLIAccessMsg(strippedAPIEndpoints[0])
printCLIAccessMsg(strippedAPIEndpoints[0], "myminio")
// Prints documentation message.
printObjectAPIMsg()
@ -141,17 +141,17 @@ func printEventNotifiers() {
// Prints startup message for command line access. Prints link to our documentation
// and custom platform specific message.
func printCLIAccessMsg(endPoint string) {
func printCLIAccessMsg(endPoint string, alias string) {
// Get saved credentials.
cred := serverConfig.GetCredential()
// Configure 'mc', following block prints platform specific information for minio client.
log.Println(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide)
if runtime.GOOS == globalWindowsOSName {
mcMessage := fmt.Sprintf("$ mc.exe config host add myminio %s %s %s", endPoint, cred.AccessKey, cred.SecretKey)
mcMessage := fmt.Sprintf("$ mc.exe config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey)
log.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
} else {
mcMessage := fmt.Sprintf("$ mc config host add myminio %s %s %s", endPoint, cred.AccessKey, cred.SecretKey)
mcMessage := fmt.Sprintf("$ mc config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey)
log.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
}
}

2
cmd/server-startup-msg_test.go
View file

@ -140,7 +140,7 @@ func TestPrintCLIAccessMsg(t *testing.T) {
defer removeAll(root)
apiEndpoints := []string{"http://127.0.0.1:9000"}
printCLIAccessMsg(apiEndpoints[0])
printCLIAccessMsg(apiEndpoints[0], "myminio")
}
// Test print startup message.