diff --git a/.travis.yml b/.travis.yml index e5a9d12bd..7f66ce3bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ matrix: - make verifiers - make crosscompile - make verify - - cd browser && yarn && yarn test && cd .. + - cd browser && npm install && npm run test && cd .. - bash -c 'shopt -s globstar; shellcheck mint/**/*.sh' - os: windows @@ -56,4 +56,4 @@ before_script: - if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6'; fi before_install: - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install 11.10.1 ; fi + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install stable ; fi diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index 96e724f82..3e738d3c7 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -56,8 +56,12 @@ const ( // - Range over all the available buckets // - Check if a bucket has an entry in etcd backend // -- If no, make an entry -// -- If yes, check if the IP of entry matches local IP. This means entry is for this instance. -// -- If IP of the entry doesn't match, this means entry is for another instance. Log an error to console. +// -- If yes, check if the entry matches local IP check if we +// need to update the entry then proceed to update +// -- If yes, check if the IP of entry matches local IP. +// This means entry is for this instance. +// -- If IP of the entry doesn't match, this means entry is +// for another instance. Log an error to console. func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) { if len(buckets) == 0 { return @@ -85,11 +89,25 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) { } return gerr } - if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() { - // There is already an entry for this bucket, with all IP addresses different. This indicates a bucket name collision. Log an error and continue. - return fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", buckets[index].Name, globalDomainIPs.ToSlice()) + if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() { + if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() { + // No difference in terms of domainIPs and nothing + // has changed so we don't change anything on the etcd. + return nil + } + // if domain IPs intersect then it won't be an empty set. + // such an intersection means that bucket exists on etcd. + // but if we do see a difference with local domain IPs with + // hostSlice from etcd then we should update with newer + // domainIPs, we proceed to do that here. + return globalDNSConfig.Put(buckets[index].Name) } - return nil + // No IPs seem to intersect, this means that bucket exists but has + // different IP addresses perhaps from a different deployment. + // bucket names are globally unique in federation at a given + // path prefix, name collision is not allowed. We simply log + // an error and continue. + return fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", buckets[index].Name, globalDomainIPs.ToSlice()) }, index) } @@ -115,9 +133,11 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) { return nil } - // We go to here, so we know the bucket no longer exists, but is registered in DNS to this server + // We go to here, so we know the bucket no longer exists, + // but is registered in DNS to this server if err := globalDNSConfig.DeleteRecord(dnsBuckets[index]); err != nil { - return fmt.Errorf("Failed to remove DNS entry for %s due to %w", dnsBuckets[index].Key, err) + return fmt.Errorf("Failed to remove DNS entry for %s due to %w", + dnsBuckets[index].Key, err) } return nil diff --git a/cmd/bucket-notification-handlers.go b/cmd/bucket-notification-handlers.go index 5e7a5af4d..79de4f4d0 100644 --- a/cmd/bucket-notification-handlers.go +++ b/cmd/bucket-notification-handlers.go @@ -31,8 +31,6 @@ import ( xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/event" - "github.com/minio/minio/pkg/handlers" - xnet "github.com/minio/minio/pkg/net" "github.com/minio/minio/pkg/policy" ) @@ -232,39 +230,39 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit values := r.URL.Query() var prefix string - if len(values["prefix"]) > 1 { + if len(values[peerRESTListenPrefix]) > 1 { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrFilterNamePrefix), r.URL, guessIsBrowserReq(r)) return } - if len(values["prefix"]) == 1 { - if err := event.ValidateFilterRuleValue(values["prefix"][0]); err != nil { + if len(values[peerRESTListenPrefix]) == 1 { + if err := event.ValidateFilterRuleValue(values[peerRESTListenPrefix][0]); err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) return } - prefix = values["prefix"][0] + prefix = values[peerRESTListenPrefix][0] } var suffix string - if len(values["suffix"]) > 1 { + if len(values[peerRESTListenSuffix]) > 1 { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrFilterNameSuffix), r.URL, guessIsBrowserReq(r)) return } - if len(values["suffix"]) == 1 { - if err := event.ValidateFilterRuleValue(values["suffix"][0]); err != nil { + if len(values[peerRESTListenSuffix]) == 1 { + if err := event.ValidateFilterRuleValue(values[peerRESTListenSuffix][0]); err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) return } - suffix = values["suffix"][0] + suffix = values[peerRESTListenSuffix][0] } pattern := event.NewPattern(prefix, suffix) - eventNames := []event.Name{} - for _, s := range values["events"] { + var eventNames []event.Name + for _, s := range values[peerRESTListenEvents] { eventName, err := event.ParseName(s) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) @@ -279,14 +277,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit return } - host, err := xnet.ParseHost(handlers.GetSourceIP(r)) - if err != nil { - writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) - return - } - - rulesMap := event.NewRulesMap(eventNames, pattern, - event.TargetID{ID: "listen" + "+" + mustGetUUID() + "+" + host.Name, Name: host.Port.String()}) + rulesMap := event.NewRulesMap(eventNames, pattern, event.TargetID{ID: mustGetUUID()}) w.Header().Set(xhttp.ContentType, "text/event-stream") @@ -306,7 +297,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit } objectName, uerr := url.QueryUnescape(ev.S3.Object.Key) if uerr != nil { - return false + objectName = ev.S3.Object.Key } return len(rulesMap.Match(ev.EventName, objectName).ToSlice()) != 0 }) @@ -315,7 +306,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit if peer == nil { continue } - peer.Listen(listenCh, doneCh) + peer.Listen(listenCh, doneCh, r.URL.Query()) } keepAliveTicker := time.NewTicker(500 * time.Millisecond) @@ -326,8 +317,14 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit select { case evI := <-listenCh: ev := evI.(event.Event) - if err := enc.Encode(struct{ Records []event.Event }{[]event.Event{ev}}); err != nil { - return + if len(string(ev.EventName)) > 0 { + if err := enc.Encode(struct{ Records []event.Event }{[]event.Event{ev}}); err != nil { + return + } + } else { + if _, err := w.Write([]byte(" ")); err != nil { + return + } } w.(http.Flusher).Flush() case <-keepAliveTicker.C: diff --git a/cmd/consolelogger.go b/cmd/consolelogger.go index 8f65e70cc..d2f74eb51 100644 --- a/cmd/consolelogger.go +++ b/cmd/consolelogger.go @@ -34,31 +34,40 @@ const defaultLogBufferCount = 10000 //HTTPConsoleLoggerSys holds global console logger state type HTTPConsoleLoggerSys struct { + sync.RWMutex pubsub *pubsub.PubSub console *console.Target nodeName string - // To protect ring buffer. - logBufLk sync.RWMutex logBuf *ring.Ring } -// NewConsoleLogger - creates new HTTPConsoleLoggerSys with all nodes subscribed to -// the console logging pub sub system -func NewConsoleLogger(ctx context.Context, endpointZones EndpointZones) *HTTPConsoleLoggerSys { +func mustGetNodeName(endpointZones EndpointZones) (nodeName string) { host, err := xnet.ParseHost(GetLocalPeer(endpointZones)) if err != nil { logger.FatalIf(err, "Unable to start console logging subsystem") } - var nodeName string if globalIsDistXL { nodeName = host.Name } + return nodeName +} + +// NewConsoleLogger - creates new HTTPConsoleLoggerSys with all nodes subscribed to +// the console logging pub sub system +func NewConsoleLogger(ctx context.Context) *HTTPConsoleLoggerSys { ps := pubsub.New() return &HTTPConsoleLoggerSys{ - ps, nil, nodeName, sync.RWMutex{}, ring.New(defaultLogBufferCount), + pubsub: ps, + console: console.New(), + logBuf: ring.New(defaultLogBufferCount), } } +// SetNodeName - sets the node name if any after distributed setup has initialized +func (sys *HTTPConsoleLoggerSys) SetNodeName(endpointZones EndpointZones) { + sys.nodeName = mustGetNodeName(endpointZones) +} + // HasLogListeners returns true if console log listeners are registered // for this node or peers func (sys *HTTPConsoleLoggerSys) HasLogListeners() bool { @@ -67,9 +76,9 @@ func (sys *HTTPConsoleLoggerSys) HasLogListeners() bool { // Subscribe starts console logging for this node. func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh chan struct{}, node string, last int, logKind string, filter func(entry interface{}) bool) { - // Enable console logging for remote client even if local console logging is disabled in the config. - if !sys.pubsub.HasSubscribers() { - logger.AddTarget(globalConsoleSys.Console()) + // Enable console logging for remote client. + if !sys.HasLogListeners() { + logger.AddTarget(sys) } cnt := 0 @@ -81,14 +90,14 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh chan s } lastN = make([]madmin.LogInfo, last) - sys.logBufLk.RLock() + sys.RLock() sys.logBuf.Do(func(p interface{}) { if p != nil && (p.(madmin.LogInfo)).SendLog(node, logKind) { lastN[cnt%last] = p.(madmin.LogInfo) cnt++ } }) - sys.logBufLk.RUnlock() + sys.RUnlock() // send last n console log messages in order filtered by node if cnt > 0 { for i := 0; i < last; i++ { @@ -106,17 +115,6 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh chan s sys.pubsub.Subscribe(subCh, doneCh, filter) } -// Console returns a console target -func (sys *HTTPConsoleLoggerSys) Console() *HTTPConsoleLoggerSys { - if sys == nil { - return sys - } - if sys.console == nil { - sys.console = console.New() - } - return sys -} - // Send log message 'e' to console and publish to console // log pubsub system func (sys *HTTPConsoleLoggerSys) Send(e interface{}, logKind string) error { @@ -129,11 +127,11 @@ func (sys *HTTPConsoleLoggerSys) Send(e interface{}, logKind string) error { } sys.pubsub.Publish(lg) - sys.logBufLk.Lock() + sys.Lock() // add log to ring buffer sys.logBuf.Value = lg sys.logBuf = sys.logBuf.Next() - sys.logBufLk.Unlock() + sys.Unlock() return sys.console.Send(e, string(logger.All)) } diff --git a/cmd/data-usage.go b/cmd/data-usage.go index 1921afe48..519433e10 100644 --- a/cmd/data-usage.go +++ b/cmd/data-usage.go @@ -68,7 +68,13 @@ func runDataUsageInfoForFS(ctx context.Context, fsObj *FSObjects, endCh <-chan s // Save the data usage in the disk err := storeDataUsageInBackend(ctx, fsObj, usageInfo) if err != nil { - logger.LogIf(ctx, err) + if globalWORMEnabled { + if _, ok := err.(ObjectAlreadyExists); !ok { + logger.LogIf(ctx, err) + } + } else { + logger.LogIf(ctx, err) + } } select { case <-endCh: @@ -97,7 +103,13 @@ func runDataUsageInfoForXLZones(ctx context.Context, z *xlZones, endCh <-chan st usageInfo := z.crawlAndGetDataUsage(ctx, endCh) err := storeDataUsageInBackend(ctx, z, usageInfo) if err != nil { - logger.LogIf(ctx, err) + if globalWORMEnabled { + if _, ok := err.(ObjectAlreadyExists); !ok { + logger.LogIf(ctx, err) + } + } else { + logger.LogIf(ctx, err) + } } select { case <-endCh: diff --git a/cmd/gateway-main.go b/cmd/gateway-main.go index 830d9da9d..34f6ca8d5 100644 --- a/cmd/gateway-main.go +++ b/cmd/gateway-main.go @@ -38,7 +38,10 @@ import ( func init() { logger.Init(GOPATH, GOROOT) logger.RegisterError(config.FmtError) - logger.AddTarget(globalConsoleSys.Console()) + + // Initialize globalConsoleSys system + globalConsoleSys = NewConsoleLogger(context.Background()) + logger.AddTarget(globalConsoleSys) } var ( @@ -145,9 +148,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) { // Set when gateway is enabled globalIsGateway = true - // Initialize globalConsoleSys system - globalConsoleSys = NewConsoleLogger(context.Background(), globalEndpoints) - enableConfigOps := gatewayName == "nas" // TODO: We need to move this code with globalConfigSys.Init() diff --git a/cmd/gateway/azure/gateway-azure.go b/cmd/gateway/azure/gateway-azure.go index f079e539f..09e81ae96 100644 --- a/cmd/gateway/azure/gateway-azure.go +++ b/cmd/gateway/azure/gateway-azure.go @@ -83,23 +83,6 @@ FLAGS: ENDPOINT: Azure server endpoint. Default ENDPOINT is https://core.windows.net -ENVIRONMENT VARIABLES: - ACCESS: - MINIO_ACCESS_KEY: Username or access key of Azure storage. - MINIO_SECRET_KEY: Password or secret key of Azure storage. - - BROWSER: - MINIO_BROWSER: To disable web browser access, set this value to "off". - - DOMAIN: - MINIO_DOMAIN: To enable virtual-host-style requests, set this value to MinIO host domain name. - - CACHE: - MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ",". - MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ",". - MINIO_CACHE_EXPIRY: Cache expiry duration in days. - MINIO_CACHE_QUOTA: Maximum permitted usage of the cache in percentage (0-100). - EXAMPLES: 1. Start minio gateway server for Azure Blob Storage backend on custom endpoint. {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}azureaccountname diff --git a/cmd/gateway/b2/gateway-b2.go b/cmd/gateway/b2/gateway-b2.go index 129b66a12..eafb7aa1b 100644 --- a/cmd/gateway/b2/gateway-b2.go +++ b/cmd/gateway/b2/gateway-b2.go @@ -57,30 +57,14 @@ USAGE: FLAGS: {{range .VisibleFlags}}{{.}} {{end}}{{end}} -ENVIRONMENT VARIABLES: - ACCESS: - MINIO_ACCESS_KEY: B2 account id. - MINIO_SECRET_KEY: B2 application key. - - BROWSER: - MINIO_BROWSER: To disable web browser access, set this value to "off". - - DOMAIN: - MINIO_DOMAIN: To enable virtual-host-style requests, set this value to MinIO host domain name. - - CACHE: - MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ",". - MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ",". - MINIO_CACHE_EXPIRY: Cache expiry duration in days. - MINIO_CACHE_QUOTA: Maximum permitted usage of the cache in percentage (0-100). EXAMPLES: - 1. Start minio gateway server for B2 backend. + 1. Start minio gateway server for B2 backend {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accountID {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}applicationKey {{.Prompt}} {{.HelpName}} - 2. Start minio gateway server for B2 backend with edge caching enabled. + 2. Start minio gateway server for B2 backend with edge caching enabled {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accountID {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}applicationKey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4" diff --git a/cmd/gateway/gcs/gateway-gcs.go b/cmd/gateway/gcs/gateway-gcs.go index 6d54bed23..aa47abb17 100644 --- a/cmd/gateway/gcs/gateway-gcs.go +++ b/cmd/gateway/gcs/gateway-gcs.go @@ -106,37 +106,19 @@ FLAGS: {{range .VisibleFlags}}{{.}} {{end}}{{end}} PROJECTID: - GCS project-id should be provided if GOOGLE_APPLICATION_CREDENTIALS environmental variable is not set. + optional GCS project-id expected GOOGLE_APPLICATION_CREDENTIALS env is not set -ENVIRONMENT VARIABLES: - ACCESS: - MINIO_ACCESS_KEY: Username or access key of GCS. - MINIO_SECRET_KEY: Password or secret key of GCS. - - BROWSER: - MINIO_BROWSER: To disable web browser access, set this value to "off". - - DOMAIN: - MINIO_DOMAIN: To enable virtual-host-style requests, set this value to MinIO host domain name. - - CACHE: - MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ",". - MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ",". - MINIO_CACHE_EXPIRY: Cache expiry duration in days. - MINIO_CACHE_QUOTA: Maximum permitted usage of the cache in percentage (0-100). - - GCS credentials file: - GOOGLE_APPLICATION_CREDENTIALS: Path to credentials.json +GOOGLE_APPLICATION_CREDENTIALS: + path to credentials.json, generated it from here https://developers.google.com/identity/protocols/application-default-credentials EXAMPLES: - 1. Start minio gateway server for GCS backend. + 1. Start minio gateway server for GCS backend {{.Prompt}} {{.EnvVarSetCommand}} GOOGLE_APPLICATION_CREDENTIALS{{.AssignmentOperator}}/path/to/credentials.json - (Instructions to generate credentials : https://developers.google.com/identity/protocols/application-default-credentials) {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey {{.Prompt}} {{.HelpName}} mygcsprojectid - 2. Start minio gateway server for GCS backend with edge caching enabled. + 2. Start minio gateway server for GCS backend with edge caching enabled {{.Prompt}} {{.EnvVarSetCommand}} GOOGLE_APPLICATION_CREDENTIALS{{.AssignmentOperator}}/path/to/credentials.json {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey diff --git a/cmd/gateway/hdfs/gateway-hdfs.go b/cmd/gateway/hdfs/gateway-hdfs.go index ed82013c4..53fb317ac 100644 --- a/cmd/gateway/hdfs/gateway-hdfs.go +++ b/cmd/gateway/hdfs/gateway-hdfs.go @@ -64,30 +64,13 @@ FLAGS: HDFS-NAMENODE: HDFS namenode URI -ENVIRONMENT VARIABLES: - ACCESS: - MINIO_ACCESS_KEY: Username or access key of minimum 3 characters in length. - MINIO_SECRET_KEY: Password or secret key of minimum 8 characters in length. - - BROWSER: - MINIO_BROWSER: To disable web browser access, set this value to "off". - - DOMAIN: - MINIO_DOMAIN: To enable virtual-host-style requests, set this value to Minio host domain name. - - CACHE: - MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ",". - MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ",". - MINIO_CACHE_EXPIRY: Cache expiry duration in days. - MINIO_CACHE_QUOTA: Maximum permitted usage of the cache in percentage (0-100). - EXAMPLES: - 1. Start minio gateway server for HDFS backend. + 1. Start minio gateway server for HDFS backend {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey {{.Prompt}} {{.HelpName}} hdfs://namenode:8200 - 2. Start minio gateway server for HDFS with edge caching enabled. + 2. Start minio gateway server for HDFS with edge caching enabled {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4" diff --git a/cmd/gateway/nas/gateway-nas.go b/cmd/gateway/nas/gateway-nas.go index ec2d9633a..a18f64117 100644 --- a/cmd/gateway/nas/gateway-nas.go +++ b/cmd/gateway/nas/gateway-nas.go @@ -39,32 +39,15 @@ FLAGS: {{range .VisibleFlags}}{{.}} {{end}}{{end}} PATH: - Path to NAS mount point. - -ENVIRONMENT VARIABLES: - ACCESS: - MINIO_ACCESS_KEY: Username or access key of minimum 3 characters in length. - MINIO_SECRET_KEY: Password or secret key of minimum 8 characters in length. - - BROWSER: - MINIO_BROWSER: To disable web browser access, set this value to "off". - - DOMAIN: - MINIO_DOMAIN: To enable virtual-host-style requests, set this value to MinIO host domain name. - - CACHE: - MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ",". - MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ",". - MINIO_CACHE_EXPIRY: Cache expiry duration in days. - MINIO_CACHE_QUOTA: Maximum permitted usage of the cache in percentage (0-100). + path to NAS mount point EXAMPLES: - 1. Start minio gateway server for NAS backend. + 1. Start minio gateway server for NAS backend {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey {{.Prompt}} {{.HelpName}} /shared/nasvol - 2. Start minio gateway server for NAS with edge caching enabled. + 2. Start minio gateway server for NAS with edge caching enabled {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4" diff --git a/cmd/gateway/oss/gateway-oss.go b/cmd/gateway/oss/gateway-oss.go index f129f587b..8443e2042 100644 --- a/cmd/gateway/oss/gateway-oss.go +++ b/cmd/gateway/oss/gateway-oss.go @@ -58,37 +58,15 @@ FLAGS: {{range .VisibleFlags}}{{.}} {{end}}{{end}} ENDPOINT: - OSS server endpoint. Default ENDPOINT is https://oss.aliyuncs.com - -ENVIRONMENT VARIABLES: - ACCESS: - MINIO_ACCESS_KEY: Username or access key of OSS storage. - MINIO_SECRET_KEY: Password or secret key of OSS storage. - - BROWSER: - MINIO_BROWSER: To disable web browser access, set this value to "off". - - DOMAIN: - MINIO_DOMAIN: To enable virtual-host-style requests, set this value to MinIO host domain name. - - CACHE: - MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ",". - MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ",". - MINIO_CACHE_EXPIRY: Cache expiry duration in days. - MINIO_CACHE_QUOTA: Maximum permitted usage of the cache in percentage (0-100). + oss server endpoint. Default ENDPOINT is https://oss.aliyuncs.com EXAMPLES: - 1. Start minio gateway server for Aliyun OSS backend. + 1. Start minio gateway server for Aliyun OSS backend {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey {{.Prompt}} {{.HelpName}} - 2. Start minio gateway server for Aliyun OSS backend on custom endpoint. - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}Q3AM3UQ867SPQQA43P2F - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG - {{.Prompt}} {{.HelpName}} https://oss.example.com - - 3. Start minio gateway server for Aliyun OSS backend with edge caching enabled. + 2. Start minio gateway server for Aliyun OSS backend with edge caching enabled {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4" diff --git a/cmd/gateway/s3/gateway-s3.go b/cmd/gateway/s3/gateway-s3.go index 94fc2fa6f..4fb685426 100644 --- a/cmd/gateway/s3/gateway-s3.go +++ b/cmd/gateway/s3/gateway-s3.go @@ -53,32 +53,15 @@ FLAGS: {{range .VisibleFlags}}{{.}} {{end}}{{end}} ENDPOINT: - S3 server endpoint. Default ENDPOINT is https://s3.amazonaws.com - -ENVIRONMENT VARIABLES: - ACCESS: - MINIO_ACCESS_KEY: Username or access key of S3 storage. - MINIO_SECRET_KEY: Password or secret key of S3 storage. - - BROWSER: - MINIO_BROWSER: To disable web browser access, set this value to "off". - - DOMAIN: - MINIO_DOMAIN: To enable virtual-host-style requests, set this value to MinIO host domain name. - - CACHE: - MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ",". - MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ",". - MINIO_CACHE_EXPIRY: Cache expiry duration in days. - MINIO_CACHE_QUOTA: Maximum permitted usage of the cache in percentage (0-100). + s3 server endpoint. Default ENDPOINT is https://s3.amazonaws.com EXAMPLES: - 1. Start minio gateway server for AWS S3 backend. + 1. Start minio gateway server for AWS S3 backend {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey {{.Prompt}} {{.HelpName}} - 4. Start minio gateway server for AWS S3 backend with edge caching enabled. + 2. Start minio gateway server for AWS S3 backend with edge caching enabled {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4" @@ -86,15 +69,6 @@ EXAMPLES: {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40 {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80 {{.Prompt}} {{.HelpName}} - - 4. Start minio gateway server for AWS S3 backend using AWS environment variables. - NOTE: The access and secret key in this case will authenticate with MinIO instead - of AWS and AWS envs will be used to authenticate to AWS S3. - {{.Prompt}} {{.EnvVarSetCommand}} AWS_ACCESS_KEY_ID{{.AssignmentOperator}}aws_access_key - {{.Prompt}} {{.EnvVarSetCommand}} AWS_SECRET_ACCESS_KEY{{.AssignmentOperator}}aws_secret_key - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey - {{.Prompt}} {{.HelpName}} ` minio.RegisterGatewayCommand(cli.Command{ diff --git a/cmd/peer-rest-client.go b/cmd/peer-rest-client.go index 03657555c..fa6b0176f 100644 --- a/cmd/peer-rest-client.go +++ b/cmd/peer-rest-client.go @@ -649,7 +649,7 @@ func (client *peerRESTClient) doTrace(traceCh chan interface{}, doneCh chan stru } } -func (client *peerRESTClient) doListen(listenCh chan interface{}, doneCh chan struct{}) { +func (client *peerRESTClient) doListen(listenCh chan interface{}, doneCh chan struct{}, v url.Values) { // To cancel the REST request in case doneCh gets closed. ctx, cancel := context.WithCancel(context.Background()) @@ -664,7 +664,7 @@ func (client *peerRESTClient) doListen(listenCh chan interface{}, doneCh chan st cancel() }() - respBody, err := client.callWithContext(ctx, peerRESTMethodListen, nil, nil, -1) + respBody, err := client.callWithContext(ctx, peerRESTMethodListen, v, nil, -1) defer http.DrainBody(respBody) if err != nil { @@ -688,10 +688,10 @@ func (client *peerRESTClient) doListen(listenCh chan interface{}, doneCh chan st } // Listen - listen on peers. -func (client *peerRESTClient) Listen(listenCh chan interface{}, doneCh chan struct{}) { +func (client *peerRESTClient) Listen(listenCh chan interface{}, doneCh chan struct{}, v url.Values) { go func() { for { - client.doListen(listenCh, doneCh) + client.doListen(listenCh, doneCh, v) select { case <-doneCh: return diff --git a/cmd/peer-rest-common.go b/cmd/peer-rest-common.go index a39de6423..ce5761019 100644 --- a/cmd/peer-rest-common.go +++ b/cmd/peer-rest-common.go @@ -81,4 +81,8 @@ const ( peerRESTDryRun = "dry-run" peerRESTTraceAll = "all" peerRESTTraceErr = "err" + + peerRESTListenPrefix = "prefix" + peerRESTListenSuffix = "suffix" + peerRESTListenEvents = "events" ) diff --git a/cmd/peer-rest-server.go b/cmd/peer-rest-server.go index 9fced5d78..372413c50 100644 --- a/cmd/peer-rest-server.go +++ b/cmd/peer-rest-server.go @@ -24,6 +24,7 @@ import ( "io" "io/ioutil" "net/http" + "net/url" "strconv" "strings" "time" @@ -915,6 +916,53 @@ func (s *peerRESTServer) ListenHandler(w http.ResponseWriter, r *http.Request) { return } + values := r.URL.Query() + + var prefix string + if len(values[peerRESTListenPrefix]) > 1 { + s.writeErrorResponse(w, errors.New("Invalid request")) + return + } + + if len(values[peerRESTListenPrefix]) == 1 { + if err := event.ValidateFilterRuleValue(values[peerRESTListenPrefix][0]); err != nil { + s.writeErrorResponse(w, err) + return + } + + prefix = values[peerRESTListenPrefix][0] + } + + var suffix string + if len(values[peerRESTListenSuffix]) > 1 { + s.writeErrorResponse(w, errors.New("Invalid request")) + return + } + + if len(values[peerRESTListenSuffix]) == 1 { + if err := event.ValidateFilterRuleValue(values[peerRESTListenSuffix][0]); err != nil { + s.writeErrorResponse(w, err) + return + } + + suffix = values[peerRESTListenSuffix][0] + } + + pattern := event.NewPattern(prefix, suffix) + + var eventNames []event.Name + for _, ev := range values[peerRESTListenEvents] { + eventName, err := event.ParseName(ev) + if err != nil { + s.writeErrorResponse(w, err) + return + } + + eventNames = append(eventNames, eventName) + } + + rulesMap := event.NewRulesMap(eventNames, pattern, event.TargetID{ID: mustGetUUID()}) + w.WriteHeader(http.StatusOK) w.(http.Flusher).Flush() @@ -925,8 +973,16 @@ func (s *peerRESTServer) ListenHandler(w http.ResponseWriter, r *http.Request) { // Use buffered channel to take care of burst sends or slow w.Write() ch := make(chan interface{}, 2000) - globalHTTPListen.Subscribe(ch, doneCh, func(entry interface{}) bool { - return true + globalHTTPListen.Subscribe(ch, doneCh, func(evI interface{}) bool { + ev, ok := evI.(event.Event) + if !ok { + return false + } + objectName, uerr := url.QueryUnescape(ev.S3.Object.Key) + if uerr != nil { + objectName = ev.S3.Object.Key + } + return len(rulesMap.Match(ev.EventName, objectName).ToSlice()) != 0 }) keepAliveTicker := time.NewTicker(500 * time.Millisecond) @@ -1108,7 +1164,7 @@ func registerPeerRESTHandlers(router *mux.Router) { subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodBackgroundOpsStatus).HandlerFunc(server.BackgroundOpsStatusHandler) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodTrace).HandlerFunc(server.TraceHandler) - subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodListen).HandlerFunc(server.ListenHandler) + subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodListen).HandlerFunc(httpTraceHdrs(server.ListenHandler)) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodBackgroundHealStatus).HandlerFunc(server.BackgroundHealStatusHandler) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLog).HandlerFunc(server.ConsoleLogHandler) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodPutBucketObjectLockConfig).HandlerFunc(httpTraceHdrs(server.PutBucketObjectLockConfigHandler)).Queries(restQueries(peerRESTBucket)...) diff --git a/cmd/server-main.go b/cmd/server-main.go index 16aebcabd..9e574725f 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -40,7 +40,11 @@ import ( func init() { logger.Init(GOPATH, GOROOT) logger.RegisterError(config.FmtError) - logger.AddTarget(globalConsoleSys.Console()) + + // Initialize globalConsoleSys system + globalConsoleSys = NewConsoleLogger(context.Background()) + logger.AddTarget(globalConsoleSys) + gob.Register(VerifyFileError("")) gob.Register(DeleteFileError("")) } @@ -65,6 +69,7 @@ var serverCmd = cli.Command{ USAGE: {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR1 [DIR2..] {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR{1...64} + {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR{1...64} DIR{65...128} DIR: DIR points to a directory on a filesystem. When you want to combine @@ -76,56 +81,21 @@ DIR: FLAGS: {{range .VisibleFlags}}{{.}} {{end}}{{end}} -ENVIRONMENT VARIABLES: - ACCESS: - MINIO_ACCESS_KEY: Custom username or access key of minimum 3 characters in length. - MINIO_SECRET_KEY: Custom password or secret key of minimum 8 characters in length. - - BROWSER: - MINIO_BROWSER: To disable web browser access, set this value to "off". - - DOMAIN: - MINIO_DOMAIN: To enable virtual-host-style requests, set this value to MinIO host domain name. - - WORM: - MINIO_WORM: To turn on Write-Once-Read-Many in server, set this value to "on". - - BUCKET-DNS: - MINIO_DOMAIN: To enable bucket DNS requests, set this value to MinIO host domain name. - MINIO_PUBLIC_IPS: To enable bucket DNS requests, set this value to list of MinIO host public IP(s) delimited by ",". - MINIO_ETCD_ENDPOINTS: To enable bucket DNS requests, set this value to list of etcd endpoints delimited by ",". - - KMS: - MINIO_KMS_VAULT_ENDPOINT: To enable Vault as KMS,set this value to Vault endpoint. - MINIO_KMS_VAULT_APPROLE_ID: To enable Vault as KMS,set this value to Vault AppRole ID. - MINIO_KMS_VAULT_APPROLE_SECRET: To enable Vault as KMS,set this value to Vault AppRole Secret ID. - MINIO_KMS_VAULT_KEY_NAME: To enable Vault as KMS,set this value to Vault encryption key-ring name. EXAMPLES: 1. Start minio server on "/home/shared" directory. {{.Prompt}} {{.HelpName}} /home/shared - 2. Start minio server bound to a specific ADDRESS:PORT. - {{.Prompt}} {{.HelpName}} --address 192.168.1.101:9000 /home/shared - - 3. Start minio server and enable virtual-host-style requests. - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_DOMAIN{{.AssignmentOperator}}mydomain.com - {{.Prompt}} {{.HelpName}} --address mydomain.com:9000 /mnt/export - - 4. Start erasure coded minio server on a node with 64 drives. - {{.Prompt}} {{.HelpName}} /mnt/export{1...64} - - 5. Start distributed minio server on an 32 node setup with 32 drives each. Run following command on all the 32 nodes. + 2. Start distributed minio server on an 32 node setup with 32 drives each, run following command on all the nodes {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}minio {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}miniostorage {{.Prompt}} {{.HelpName}} http://node{1...32}.example.com/mnt/export/{1...32} - 6. Start minio server with KMS enabled. - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_KMS_VAULT_APPROLE_ID{{.AssignmentOperator}}9b56cc08-8258-45d5-24a3-679876769126 - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_KMS_VAULT_APPROLE_SECRET{{.AssignmentOperator}}4e30c52f-13e4-a6f5-0763-d50e8cb4321f - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_KMS_VAULT_ENDPOINT{{.AssignmentOperator}}https://vault-endpoint-ip:8200 - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_KMS_VAULT_KEY_NAME{{.AssignmentOperator}}my-minio-key - {{.Prompt}} {{.HelpName}} /home/shared + 3. Start distributed minio server in an expanded setup, run the following command on all the nodes + {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}minio + {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}miniostorage + {{.Prompt}} {{.HelpName}} http://node{1...16}.example.com/mnt/export/{1...32} \ + http://node{17...64}.example.com/mnt/export/{1...64} `, } @@ -236,7 +206,15 @@ func initSafeMode(buckets []BucketInfo) (err error) { // Migrate all backend configs to encrypted backend configs, optionally // handles rotating keys for encryption. if err = handleEncryptedConfigBackend(newObject, true); err != nil { - return fmt.Errorf("Unable to handle encrypted backend for config, iam and policies: %w", err) + if globalWORMEnabled { + if _, ok := err.(ObjectAlreadyExists); !ok { + return fmt.Errorf("Unable to handle encrypted backend for config, iam and policies: %w", + err) + } + // Ignore ObjectAlreadyExists if globalWORMEnabled is true. + } else { + return fmt.Errorf("Unable to handle encrypted backend for config, iam and policies: %w", err) + } } // **** WARNING **** @@ -298,6 +276,9 @@ func serverMain(ctx *cli.Context) { cli.ShowCommandHelpAndExit(ctx, "server", 1) } + // Initialize globalConsoleSys system + globalConsoleSys = NewConsoleLogger(context.Background()) + signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM) // Handle all server command args. @@ -306,6 +287,9 @@ func serverMain(ctx *cli.Context) { // Handle all server environment vars. serverHandleEnvVars() + // Set node name, only set for distributed setup. + globalConsoleSys.SetNodeName(globalEndpoints) + // Initialize all help initHelp() @@ -349,9 +333,6 @@ func serverMain(ctx *cli.Context) { globalBackgroundHealState = initHealState() } - // Initialize globalConsoleSys system - globalConsoleSys = NewConsoleLogger(context.Background(), globalEndpoints) - // Configure server. var handler http.Handler handler, err = configureServerHandler(globalEndpoints) diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index 67fcc5784..9920be5ff 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -74,6 +74,9 @@ func init() { // Set system resources to maximum. setMaxResources() + // Initialize globalConsoleSys system + globalConsoleSys = NewConsoleLogger(context.Background()) + logger.Disable = true initHelp() @@ -497,9 +500,6 @@ func resetTestGlobals() { // Configure the server for the test run. func newTestConfig(bucketLocation string, obj ObjectLayer) (err error) { - // Initialize globalConsoleSys system - globalConsoleSys = NewConsoleLogger(context.Background(), globalEndpoints) - // Initialize server config. if err = newSrvConfig(obj); err != nil { return err diff --git a/docs/distributed/README.md b/docs/distributed/README.md index 84a8ca348..7f08fccf4 100644 --- a/docs/distributed/README.md +++ b/docs/distributed/README.md @@ -66,7 +66,7 @@ export MINIO_SECRET_KEY= minio server http://host{1...32}/export{1...32} http://host{33...64}/export{1...32} ``` -Now the server has expanded storage of *1024* more disks in total of *2048* disks, new object upload requests automatically start using the least used cluster. This expansion strategy works endlessly, so you can perpetually expand your clusters as needed. +Now the server has expanded storage of *1024* more disks in total of *2048* disks, new object upload requests automatically start using the least used cluster. This expansion strategy works endlessly, so you can perpetually expand your clusters as needed. When you restart, it is immediate and non-disruptive to the applications. Each group of servers in the command-line is called a zone. There are 2 zones in this example. Objects are placed in zones based on which zone has the *fmost free* space. Within each zone, the location of the erasure-set of drives is determined based on a deterministic hashing algorithm. ## 3. Test your setup To test this setup, access the MinIO server via browser or [`mc`](https://docs.min.io/docs/minio-client-quickstart-guide).