2017-03-16 20:21:58 +01:00
/ *
* Minio Cloud Storage , ( C ) 2017 Minio , Inc .
*
* Licensed under the Apache License , Version 2.0 ( the "License" ) ;
* you may not use this file except in compliance with the License .
* You may obtain a copy of the License at
*
* http : //www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing , software
* distributed under the License is distributed on an "AS IS" BASIS ,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
* See the License for the specific language governing permissions and
* limitations under the License .
* /
package cmd
import (
2017-03-24 00:36:00 +01:00
"errors"
2017-03-16 20:21:58 +01:00
"fmt"
2017-04-12 02:44:26 +02:00
"net/url"
2017-07-13 01:33:21 +02:00
"os"
"os/signal"
2017-06-08 20:20:56 +02:00
"runtime"
2017-04-12 02:44:26 +02:00
"strings"
2017-07-13 01:33:21 +02:00
"syscall"
2017-06-06 00:18:03 +02:00
"github.com/gorilla/mux"
"github.com/minio/cli"
2017-07-13 01:33:21 +02:00
miniohttp "github.com/minio/minio/pkg/http"
2017-03-16 20:21:58 +01:00
)
2017-06-09 08:28:45 +02:00
const azureGatewayTemplate = ` NAME :
2017-03-16 20:21:58 +01:00
{ { . HelpName } } - { { . Usage } }
USAGE :
2017-06-09 08:28:45 +02:00
{ { . HelpName } } { { if . VisibleFlags } } [ FLAGS ] { { end } } [ ENDPOINT ]
2017-03-16 20:21:58 +01:00
{ { if . VisibleFlags } }
FLAGS :
{ { range . VisibleFlags } } { { . } }
{ { end } } { { end } }
2017-06-09 08:28:45 +02:00
ENDPOINT :
Azure server endpoint . Default ENDPOINT is https : //core.windows.net
2017-04-29 01:42:16 +02:00
2017-03-16 20:21:58 +01:00
ENVIRONMENT VARIABLES :
ACCESS :
2017-06-09 08:28:45 +02:00
MINIO_ACCESS_KEY : Username or access key of Azure storage .
MINIO_SECRET_KEY : Password or secret key of Azure storage .
2017-03-16 20:21:58 +01:00
2017-06-01 18:43:20 +02:00
BROWSER :
MINIO_BROWSER : To disable web browser access , set this value to "off" .
2017-03-16 20:21:58 +01:00
EXAMPLES :
1. Start minio gateway server for Azure Blob Storage backend .
2017-04-29 01:42:16 +02:00
$ export MINIO_ACCESS_KEY = azureaccountname
$ export MINIO_SECRET_KEY = azureaccountkey
2017-06-09 08:28:45 +02:00
$ { { . HelpName } }
2017-05-08 21:44:08 +02:00
2017-06-09 08:28:45 +02:00
2. Start minio gateway server for Azure Blob Storage backend on custom endpoint .
$ export MINIO_ACCESS_KEY = azureaccountname
$ export MINIO_SECRET_KEY = azureaccountkey
$ { { . HelpName } } https : //azure.example.com
`
const s3GatewayTemplate = ` NAME :
{ { . HelpName } } - { { . Usage } }
USAGE :
{ { . HelpName } } { { if . VisibleFlags } } [ FLAGS ] { { end } } [ ENDPOINT ]
{ { if . VisibleFlags } }
FLAGS :
{ { range . VisibleFlags } } { { . } }
{ { end } } { { end } }
ENDPOINT :
S3 server endpoint . Default ENDPOINT is https : //s3.amazonaws.com
ENVIRONMENT VARIABLES :
ACCESS :
MINIO_ACCESS_KEY : Username or access key of S3 storage .
MINIO_SECRET_KEY : Password or secret key of S3 storage .
BROWSER :
MINIO_BROWSER : To disable web browser access , set this value to "off" .
2017-05-04 02:55:30 +02:00
2017-06-09 08:28:45 +02:00
EXAMPLES :
1. Start minio gateway server for AWS S3 backend .
2017-05-04 02:55:30 +02:00
$ export MINIO_ACCESS_KEY = accesskey
$ export MINIO_SECRET_KEY = secretkey
2017-06-09 08:28:45 +02:00
$ { { . HelpName } }
2017-05-04 02:55:30 +02:00
2017-06-09 08:28:45 +02:00
2. Start minio gateway server for S3 backend on custom endpoint .
2017-05-04 02:55:30 +02:00
$ export MINIO_ACCESS_KEY = Q3AM3UQ867SPQQA43P2F
$ export MINIO_SECRET_KEY = zuf + tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
2017-06-09 08:28:45 +02:00
$ { { . HelpName } } https : //play.minio.io:9000
2017-03-16 20:21:58 +01:00
`
2017-05-08 21:44:08 +02:00
const gcsGatewayTemplate = ` NAME :
{ { . HelpName } } - { { . Usage } }
USAGE :
2017-10-20 22:59:12 +02:00
{ { . HelpName } } { { if . VisibleFlags } } [ FLAGS ] { { end } } [ PROJECTID ]
2017-05-08 21:44:08 +02:00
{ { if . VisibleFlags } }
FLAGS :
{ { range . VisibleFlags } } { { . } }
{ { end } } { { end } }
PROJECTID :
2017-10-20 22:59:12 +02:00
GCS project - id should be provided if GOOGLE_APPLICATION_CREDENTIALS environmental variable is not set .
2017-05-08 21:44:08 +02:00
ENVIRONMENT VARIABLES :
ACCESS :
2017-06-20 23:25:16 +02:00
MINIO_ACCESS_KEY : Username or access key of GCS .
MINIO_SECRET_KEY : Password or secret key of GCS .
2017-05-08 21:44:08 +02:00
BROWSER :
MINIO_BROWSER : To disable web browser access , set this value to "off" .
2017-10-20 22:59:12 +02:00
GCS credentials file :
GOOGLE_APPLICATION_CREDENTIALS : Path to credentials . json
2017-05-08 21:44:08 +02:00
EXAMPLES :
2017-06-20 23:25:16 +02:00
1. Start minio gateway server for GCS backend .
2017-06-23 22:13:26 +02:00
$ export GOOGLE_APPLICATION_CREDENTIALS = / path / to / credentials . json
( Instructions to generate credentials : https : //developers.google.com/identity/protocols/application-default-credentials)
2017-05-08 21:44:08 +02:00
$ export MINIO_ACCESS_KEY = accesskey
$ export MINIO_SECRET_KEY = secretkey
2017-06-20 23:25:16 +02:00
$ { { . HelpName } } mygcsprojectid
2017-05-24 22:47:39 +02:00
2017-05-08 21:44:08 +02:00
`
2017-10-13 12:56:16 +02:00
const b2GatewayTemplate = ` NAME :
{ { . HelpName } } - { { . Usage } }
USAGE :
{ { . HelpName } } { { if . VisibleFlags } } [ FLAGS ] { { end } }
{ { if . VisibleFlags } }
FLAGS :
{ { range . VisibleFlags } } { { . } }
{ { end } } { { end } }
ENVIRONMENT VARIABLES :
ACCESS :
MINIO_ACCESS_KEY : B2 account id .
MINIO_SECRET_KEY : B2 application key .
BROWSER :
MINIO_BROWSER : To disable web browser access , set this value to "off" .
EXAMPLES :
1. Start minio gateway server for B2 backend .
$ export MINIO_ACCESS_KEY = accountID
$ export MINIO_SECRET_KEY = applicationKey
$ { { . HelpName } }
`
2017-06-10 04:50:51 +02:00
var (
azureBackendCmd = cli . Command {
Name : "azure" ,
Usage : "Microsoft Azure Blob Storage." ,
Action : azureGatewayMain ,
CustomHelpTemplate : azureGatewayTemplate ,
Flags : append ( serverFlags , globalFlags ... ) ,
HideHelpCommand : true ,
}
2017-03-16 20:21:58 +01:00
2017-06-10 04:50:51 +02:00
s3BackendCmd = cli . Command {
Name : "s3" ,
Usage : "Amazon Simple Storage Service (S3)." ,
Action : s3GatewayMain ,
CustomHelpTemplate : s3GatewayTemplate ,
Flags : append ( serverFlags , globalFlags ... ) ,
HideHelpCommand : true ,
}
2017-10-13 12:56:16 +02:00
2017-06-10 04:50:51 +02:00
gcsBackendCmd = cli . Command {
Name : "gcs" ,
Usage : "Google Cloud Storage." ,
Action : gcsGatewayMain ,
CustomHelpTemplate : gcsGatewayTemplate ,
Flags : append ( serverFlags , globalFlags ... ) ,
HideHelpCommand : true ,
}
2017-05-08 21:44:08 +02:00
2017-10-13 12:56:16 +02:00
b2BackendCmd = cli . Command {
Name : "b2" ,
Usage : "Backblaze B2." ,
Action : b2GatewayMain ,
CustomHelpTemplate : b2GatewayTemplate ,
Flags : append ( serverFlags , globalFlags ... ) ,
HideHelpCommand : true ,
}
2017-06-10 04:50:51 +02:00
gatewayCmd = cli . Command {
Name : "gateway" ,
Usage : "Start object storage gateway." ,
Flags : append ( serverFlags , globalFlags ... ) ,
HideHelpCommand : true ,
2017-10-13 12:56:16 +02:00
Subcommands : [ ] cli . Command { azureBackendCmd , s3BackendCmd , gcsBackendCmd , b2BackendCmd } ,
2017-06-10 04:50:51 +02:00
}
)
2017-06-09 08:28:45 +02:00
2017-03-16 20:21:58 +01:00
// Represents the type of the gateway backend.
type gatewayBackend string
const (
azureBackend gatewayBackend = "azure"
2017-04-27 20:26:00 +02:00
s3Backend gatewayBackend = "s3"
2017-05-01 19:59:54 +02:00
gcsBackend gatewayBackend = "gcs"
2017-10-13 12:56:16 +02:00
b2Backend gatewayBackend = "b2"
2017-03-16 20:21:58 +01:00
// Add more backends here.
)
// Initialize gateway layer depending on the backend type.
// Supported backend types are
//
// - Azure Blob Storage.
2017-05-17 14:59:20 +02:00
// - AWS S3.
2017-05-08 21:44:08 +02:00
// - Google Cloud Storage.
2017-10-13 12:56:16 +02:00
// - Backblaze B2.
2017-03-16 20:21:58 +01:00
// - Add your favorite backend here.
2017-06-10 04:50:51 +02:00
func newGatewayLayer ( backendType gatewayBackend , arg string ) ( GatewayLayer , error ) {
switch backendType {
2017-04-27 20:26:00 +02:00
case azureBackend :
2017-06-10 04:50:51 +02:00
return newAzureLayer ( arg )
2017-04-27 20:26:00 +02:00
case s3Backend :
2017-06-10 04:50:51 +02:00
return newS3Gateway ( arg )
2017-05-01 19:59:54 +02:00
case gcsBackend :
2017-06-28 04:44:47 +02:00
// FIXME: The following print command is temporary and
// will be removed when gcs is ready for production use.
log . Println ( colorYellow ( "\n *** Warning: Not Ready for Production ***" ) )
2017-06-10 04:50:51 +02:00
return newGCSGateway ( arg )
2017-10-13 12:56:16 +02:00
case b2Backend :
// FIXME: The following print command is temporary and
// will be removed when B2 is ready for production use.
log . Println ( colorYellow ( "\n *** Warning: Not Ready for Production ***" ) )
return newB2Gateway ( )
2017-03-16 20:21:58 +01:00
}
2017-04-27 20:26:00 +02:00
return nil , fmt . Errorf ( "Unrecognized backend type %s" , backendType )
2017-03-16 20:21:58 +01:00
}
2017-04-12 02:44:26 +02:00
// Return endpoint.
func parseGatewayEndpoint ( arg string ) ( endPoint string , secure bool , err error ) {
schemeSpecified := len ( strings . Split ( arg , "://" ) ) > 1
if ! schemeSpecified {
// Default connection will be "secure".
arg = "https://" + arg
}
2017-04-27 20:26:00 +02:00
2017-04-12 02:44:26 +02:00
u , err := url . Parse ( arg )
if err != nil {
return "" , false , err
}
switch u . Scheme {
case "http" :
return u . Host , false , nil
case "https" :
return u . Host , true , nil
default :
return "" , false , fmt . Errorf ( "Unrecognized scheme %s" , u . Scheme )
}
}
2017-06-08 20:20:56 +02:00
// Validate gateway arguments.
func validateGatewayArguments ( serverAddr , endpointAddr string ) error {
if err := CheckLocalServerAddr ( serverAddr ) ; err != nil {
return err
}
if runtime . GOOS == "darwin" {
_ , port := mustSplitHostPort ( serverAddr )
// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
// to IPv6 address i.e minio will start listening on IPv6 address whereas another
// (non-)minio process is listening on IPv4 of given port.
// To avoid this error situation we check for port availability only for macOS.
if err := checkPortAvailability ( port ) ; err != nil {
return err
}
}
if endpointAddr != "" {
// Reject the endpoint if it points to the gateway handler itself.
sameTarget , err := sameLocalAddrs ( endpointAddr , serverAddr )
if err != nil {
return err
}
if sameTarget {
return errors . New ( "endpoint points to the local gateway" )
}
}
return nil
}
2017-06-09 08:28:45 +02:00
// Handler for 'minio gateway azure' command line.
func azureGatewayMain ( ctx * cli . Context ) {
if ctx . Args ( ) . Present ( ) && ctx . Args ( ) . First ( ) == "help" {
cli . ShowCommandHelpAndExit ( ctx , "azure" , 1 )
}
2017-06-10 04:50:51 +02:00
// Validate gateway arguments.
2017-07-12 03:06:26 +02:00
fatalIf ( validateGatewayArguments ( ctx . GlobalString ( "address" ) , ctx . Args ( ) . First ( ) ) , "Invalid argument" )
2017-06-10 04:50:51 +02:00
2017-06-09 08:28:45 +02:00
gatewayMain ( ctx , azureBackend )
}
// Handler for 'minio gateway s3' command line.
func s3GatewayMain ( ctx * cli . Context ) {
if ctx . Args ( ) . Present ( ) && ctx . Args ( ) . First ( ) == "help" {
cli . ShowCommandHelpAndExit ( ctx , "s3" , 1 )
2017-03-16 20:21:58 +01:00
}
2017-06-10 04:50:51 +02:00
// Validate gateway arguments.
2017-07-12 03:06:26 +02:00
fatalIf ( validateGatewayArguments ( ctx . GlobalString ( "address" ) , ctx . Args ( ) . First ( ) ) , "Invalid argument" )
2017-06-10 04:50:51 +02:00
2017-06-09 08:28:45 +02:00
gatewayMain ( ctx , s3Backend )
}
2017-05-08 21:44:08 +02:00
// Handler for 'minio gateway gcs' command line
func gcsGatewayMain ( ctx * cli . Context ) {
if ctx . Args ( ) . Present ( ) && ctx . Args ( ) . First ( ) == "help" {
2017-06-10 04:50:51 +02:00
cli . ShowCommandHelpAndExit ( ctx , "gcs" , 1 )
}
2017-10-20 22:59:12 +02:00
projectID := ctx . Args ( ) . First ( )
if projectID == "" && os . Getenv ( "GOOGLE_APPLICATION_CREDENTIALS" ) == "" {
errorIf ( errGCSProjectIDNotFound , "project-id should be provided as argument or GOOGLE_APPLICATION_CREDENTIALS should be set with path to credentials.json" )
cli . ShowCommandHelpAndExit ( ctx , "gcs" , 1 )
}
if projectID != "" && ! isValidGCSProjectIDFormat ( projectID ) {
2017-06-10 04:50:51 +02:00
errorIf ( errGCSInvalidProjectID , "Unable to start GCS gateway with %s" , ctx . Args ( ) . First ( ) )
cli . ShowCommandHelpAndExit ( ctx , "gcs" , 1 )
2017-05-08 21:44:08 +02:00
}
gatewayMain ( ctx , gcsBackend )
}
2017-10-13 12:56:16 +02:00
func b2GatewayMain ( ctx * cli . Context ) {
if ctx . Args ( ) . Present ( ) && ctx . Args ( ) . First ( ) == "help" {
cli . ShowCommandHelpAndExit ( ctx , "b2" , 1 )
}
// Validate gateway arguments.
fatalIf ( validateGatewayArguments ( ctx . GlobalString ( "address" ) , ctx . Args ( ) . First ( ) ) , "Invalid argument" )
gatewayMain ( ctx , b2Backend )
}
2017-06-09 08:28:45 +02:00
// Handler for 'minio gateway'.
func gatewayMain ( ctx * cli . Context , backendType gatewayBackend ) {
2017-03-16 20:21:58 +01:00
// Get quiet flag from command line argument.
quietFlag := ctx . Bool ( "quiet" ) || ctx . GlobalBool ( "quiet" )
2017-03-30 20:21:19 +02:00
if quietFlag {
log . EnableQuiet ( )
}
2017-03-16 20:21:58 +01:00
2017-07-21 01:39:11 +02:00
// Fetch address option
gatewayAddr := ctx . GlobalString ( "address" )
if gatewayAddr == ":" + globalMinioPort {
gatewayAddr = ctx . String ( "address" )
}
2017-06-10 04:50:51 +02:00
// Handle common command args.
handleCommonCmdArgs ( ctx )
// Handle common env vars.
handleCommonEnvVars ( )
2017-03-16 20:21:58 +01:00
2017-06-20 04:45:13 +02:00
// Validate if we have access, secret set through environment.
if ! globalIsEnvCreds {
fatalIf ( fmt . Errorf ( "Access and Secret keys should be set through ENVs for backend [%s]" , backendType ) , "" )
}
2017-06-10 04:50:51 +02:00
// Create certs path.
fatalIf ( createConfigDir ( ) , "Unable to create configuration directories." )
// Initialize gateway config.
initConfig ( )
// Enable loggers as per configuration file.
enableLoggers ( )
// Init the error tracing module.
initError ( )
// Check and load SSL certificates.
var err error
2017-07-13 01:33:21 +02:00
globalPublicCerts , globalRootCAs , globalTLSCertificate , globalIsSSL , err = getSSLConfig ( )
fatalIf ( err , "Invalid SSL certificate file" )
2017-03-31 07:26:24 +02:00
2017-05-23 05:02:58 +02:00
initNSLock ( false ) // Enable local namespace lock.
2017-06-10 04:50:51 +02:00
newObject , err := newGatewayLayer ( backendType , ctx . Args ( ) . First ( ) )
2017-03-24 00:36:00 +01:00
fatalIf ( err , "Unable to initialize gateway layer" )
2017-03-16 20:21:58 +01:00
router := mux . NewRouter ( ) . SkipClean ( true )
2017-06-01 18:43:20 +02:00
// Register web router when its enabled.
if globalIsBrowserEnabled {
2017-06-10 04:50:51 +02:00
fatalIf ( registerWebRouter ( router ) , "Unable to configure web browser" )
2017-06-01 18:43:20 +02:00
}
2017-03-16 20:21:58 +01:00
registerGatewayAPIRouter ( router , newObject )
var handlerFns = [ ] HandlerFunc {
2017-04-29 02:17:18 +02:00
// Validate all the incoming paths.
setPathValidityHandler ,
2017-03-16 20:21:58 +01:00
// Limits all requests size to a maximum fixed limit
setRequestSizeLimitHandler ,
// Adds 'crossdomain.xml' policy handler to serve legacy flash clients.
setCrossDomainPolicy ,
// Validates all incoming requests to have a valid date header.
2017-06-01 18:43:20 +02:00
// Redirect some pre-defined browser request paths to a static location prefix.
setBrowserRedirectHandler ,
// Validates if incoming request is for restricted buckets.
2017-09-01 21:16:54 +02:00
setReservedBucketHandler ,
2017-06-01 18:43:20 +02:00
// Adds cache control for all browser requests.
setBrowserCacheControlHandler ,
// Validates all incoming requests to have a valid date header.
2017-03-16 20:21:58 +01:00
setTimeValidityHandler ,
// CORS setting for all browser API requests.
setCorsHandler ,
// Validates all incoming URL resources, for invalid/unsupported
// resources client receives a HTTP error.
setIgnoreResourcesHandler ,
// Auth handler verifies incoming authorization headers and
// routes them accordingly. Client receives a HTTP error for
// invalid/unsupported signatures.
setAuthHandler ,
2017-06-01 18:43:20 +02:00
// Add new handlers here.
2017-03-16 20:21:58 +01:00
}
2017-07-21 01:39:11 +02:00
globalHTTPServer = miniohttp . NewServer ( [ ] string { gatewayAddr } , registerHandlers ( router , handlerFns ... ) , globalTLSCertificate )
2017-03-16 20:21:58 +01:00
// Start server, automatically configures TLS if certs are available.
go func ( ) {
2017-07-13 01:33:21 +02:00
globalHTTPServerErrorCh <- globalHTTPServer . Start ( )
2017-03-16 20:21:58 +01:00
} ( )
2017-07-13 01:33:21 +02:00
signal . Notify ( globalOSSignalCh , os . Interrupt , syscall . SIGTERM )
2017-03-16 20:21:58 +01:00
// Once endpoints are finalized, initialize the new object api.
globalObjLayerMutex . Lock ( )
globalObjectAPI = newObject
globalObjLayerMutex . Unlock ( )
// Prints the formatted startup message once object layer is initialized.
if ! quietFlag {
mode := ""
2017-05-23 05:02:58 +02:00
switch gatewayBackend ( backendType ) {
case azureBackend :
2017-03-16 20:21:58 +01:00
mode = globalMinioModeGatewayAzure
2017-05-23 05:02:58 +02:00
case gcsBackend :
mode = globalMinioModeGatewayGCS
case s3Backend :
2017-04-27 20:26:00 +02:00
mode = globalMinioModeGatewayS3
2017-10-13 12:56:16 +02:00
case b2Backend :
mode = globalMinioModeGatewayB2
2017-03-16 20:21:58 +01:00
}
2017-06-10 04:50:51 +02:00
// Check update mode.
2017-03-16 20:21:58 +01:00
checkUpdate ( mode )
2017-06-10 04:50:51 +02:00
// Print gateway startup message.
2017-07-21 01:39:11 +02:00
printGatewayStartupMessage ( getAPIEndpoints ( gatewayAddr ) , backendType )
2017-03-16 20:21:58 +01:00
}
2017-07-13 01:33:21 +02:00
handleSignals ( )
2017-03-16 20:21:58 +01:00
}