simplify context timeout for readiness (#9772)

additionally also add CORS support to restrict
for specific origin, adds a new config and
updated the documentation as well

Makefile

@@ -8,8 +8,6 @@ GOOS := $(shell go env GOOS)
 VERSION ?= $(shell git describe --tags)
 TAG ?= "minio/minio:$(VERSION)"
 
-BUILD_LDFLAGS := '$(LDFLAGS)'
-
 all: build
 
 checks:
@@ -48,19 +46,19 @@ test-race: verifiers build
 # Verify minio binary
 verify:
 	@echo "Verifying build with race"
-	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
 # Verify healing of disks with minio binary
 verify-healing:
 	@echo "Verify healing build with race"
-	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
 
 # Builds minio locally.
 build: checks
 	@echo "Building minio binary to './minio'"
-	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath  --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 
 docker: build
 	@docker build -t $(TAG) . -f Dockerfile.dev

cmd/config-current.go

@@ -365,13 +365,7 @@ func lookupConfigs(s config.Config) {
 		logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
 	}
 
-	apiRequestsMax := apiConfig.APIRequestsMax
+	globalAPIConfig.init(apiConfig)
-	if len(globalEndpoints.Hosts()) > 0 {
-		apiRequestsMax /= len(globalEndpoints.Hosts())
-	}
-
-	globalAPIThrottling.init(apiRequestsMax, apiConfig.APIRequestsDeadline)
-	globalReadyDeadline = apiConfig.APIReadyDeadline
 
 	if globalIsXL {
 		globalStorageClass, err = storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default],

cmd/config/api/api.go

@@ -20,16 +20,24 @@ import (
 	"encoding/json"
 	"errors"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/env"
 )
 
+// API sub-system constants
 const (
 	apiRequestsMax      = "requests_max"
 	apiRequestsDeadline = "requests_deadline"
 	apiReadyDeadline    = "ready_deadline"
+	apiCorsAllowOrigin  = "cors_allow_origin"
+
+	EnvAPIRequestsMax      = "MINIO_API_REQUESTS_MAX"
+	EnvAPIRequestsDeadline = "MINIO_API_REQUESTS_DEADLINE"
+	EnvAPIReadyDeadline    = "MINIO_API_READY_DEADLINE"
+	EnvAPICorsAllowOrigin  = "MINIO_API_CORS_ALLOW_ORIGIN"
 )
 
 // DefaultKVS - default storage class config
@@ -47,6 +55,10 @@ var (
 			Key:   apiReadyDeadline,
 			Value: "10s",
 		},
+		config.KV{
+			Key:   apiCorsAllowOrigin,
+			Value: "*",
+		},
 	}
 )
 
@@ -55,6 +67,7 @@ type Config struct {
 	APIRequestsMax      int           `json:"requests_max"`
 	APIRequestsDeadline time.Duration `json:"requests_deadline"`
 	APIReadyDeadline    time.Duration `json:"ready_deadline"`
+	APICorsAllowOrigin  []string      `json:"cors_allow_origin"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -75,7 +88,7 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 	}
 
 	// Check environment variables parameters
-	requestsMax, err := strconv.Atoi(env.Get(config.EnvAPIRequestsMax, kvs.Get(apiRequestsMax)))
+	requestsMax, err := strconv.Atoi(env.Get(EnvAPIRequestsMax, kvs.Get(apiRequestsMax)))
 	if err != nil {
 		return cfg, err
 	}
@@ -84,20 +97,21 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		return cfg, errors.New("invalid API max requests value")
 	}
 
-	requestsDeadline, err := time.ParseDuration(env.Get(config.EnvAPIRequestsDeadline, kvs.Get(apiRequestsDeadline)))
+	requestsDeadline, err := time.ParseDuration(env.Get(EnvAPIRequestsDeadline, kvs.Get(apiRequestsDeadline)))
 	if err != nil {
 		return cfg, err
 	}
 
-	readyDeadline, err := time.ParseDuration(env.Get(config.EnvAPIReadyDeadline, kvs.Get(apiReadyDeadline)))
+	readyDeadline, err := time.ParseDuration(env.Get(EnvAPIReadyDeadline, kvs.Get(apiReadyDeadline)))
 	if err != nil {
 		return cfg, err
 	}
 
-	cfg = Config{
+	corsAllowOrigin := strings.Split(env.Get(EnvAPICorsAllowOrigin, kvs.Get(apiCorsAllowOrigin)), ",")
+	return Config{
 		APIRequestsMax:      requestsMax,
 		APIRequestsDeadline: requestsDeadline,
 		APIReadyDeadline:    readyDeadline,
-	}
+		APICorsAllowOrigin:  corsAllowOrigin,
-	return cfg, nil
+	}, nil
 }

cmd/config/api/help.go

@@ -39,5 +39,11 @@ var (
 			Optional:    true,
 			Type:        "duration",
 		},
+		config.HelpKV{
+			Key:         apiCorsAllowOrigin,
+			Description: `set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com"`,
+			Optional:    true,
+			Type:        "csv",
+		},
 	}
 )

cmd/config/constants.go

@@ -34,11 +34,6 @@ const (
 	EnvEndpoints    = "MINIO_ENDPOINTS"
 	EnvFSOSync      = "MINIO_FS_OSYNC"
 
-	// API sub-system
-	EnvAPIRequestsMax      = "MINIO_API_REQUESTS_MAX"
-	EnvAPIRequestsDeadline = "MINIO_API_REQUESTS_DEADLINE"
-	EnvAPIReadyDeadline    = "MINIO_API_READY_DEADLINE"
-
 	EnvUpdate = "MINIO_UPDATE"
 
 	EnvWorm   = "MINIO_WORM"   // legacy

cmd/generic-handlers.go

@@ -30,6 +30,7 @@ import (
 	"github.com/minio/minio/cmd/http/stats"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/handlers"
+	"github.com/minio/minio/pkg/wildcard"
 	"github.com/rs/cors"
 )
 
@@ -410,7 +411,14 @@ func setCorsHandler(h http.Handler) http.Handler {
 	}
 
 	c := cors.New(cors.Options{
-		AllowedOrigins: []string{"*"},
+		AllowOriginFunc: func(origin string) bool {
+			for _, allowedOrigin := range globalAPIConfig.getCorsAllowOrigins() {
+				if wildcard.MatchSimple(allowedOrigin, origin) {
+					return true
+				}
+			}
+			return false
+		},
 		AllowedMethods: []string{
 			http.MethodGet,
 			http.MethodPut,

cmd/globals.go

@@ -154,9 +154,9 @@ var (
 	globalLifecycleSys       *LifecycleSys
 	globalBucketSSEConfigSys *BucketSSEConfigSys
 
-	// globalAPIThrottling controls S3 requests throttling when
+	// globalAPIConfig controls S3 API requests throttling,
-	// enabled in the config or in the shell environment.
+	// healthcheck readiness deadlines and cors settings.
-	globalAPIThrottling apiThrottling
+	globalAPIConfig apiConfig
 
 	globalStorageClass storageclass.Config
 	globalLDAPConfig   xldap.Config
@@ -275,8 +275,6 @@ var (
 	// If writes to FS backend should be O_SYNC.
 	globalFSOSync bool
 
-	// Deadline by which /minio/health/ready should respond.
-	globalReadyDeadline time.Duration
 	// Add new variable global values here.
 )
 

cmd/handler-api.go

@@ -20,34 +20,61 @@ import (
 	"net/http"
 	"sync"
 	"time"
+
+	"github.com/minio/minio/cmd/config/api"
 )
 
-type apiThrottling struct {
+type apiConfig struct {
-	mu      sync.RWMutex
+	mu sync.RWMutex
-	enabled bool
 
 	requestsDeadline time.Duration
 	requestsPool     chan struct{}
+	readyDeadline    time.Duration
+	corsAllowOrigins []string
 }
 
-func (t *apiThrottling) init(max int, deadline time.Duration) {
+func (t *apiConfig) init(cfg api.Config) {
-	if max <= 0 {
-		return
-	}
-
 	t.mu.Lock()
 	defer t.mu.Unlock()
 
-	t.requestsPool = make(chan struct{}, max)
+	t.readyDeadline = cfg.APIReadyDeadline
-	t.requestsDeadline = deadline
+	t.corsAllowOrigins = cfg.APICorsAllowOrigin
-	t.enabled = true
+	if cfg.APIRequestsMax <= 0 {
+		return
+	}
+
+	apiRequestsMax := cfg.APIRequestsMax
+	if len(globalEndpoints.Hosts()) > 0 {
+		apiRequestsMax /= len(globalEndpoints.Hosts())
+	}
+
+	t.requestsPool = make(chan struct{}, apiRequestsMax)
+	t.requestsDeadline = cfg.APIRequestsDeadline
 }
 
-func (t *apiThrottling) get() (chan struct{}, <-chan time.Time) {
+func (t *apiConfig) getCorsAllowOrigins() []string {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 
-	if !t.enabled {
+	return t.corsAllowOrigins
+}
+
+func (t *apiConfig) getReadyDeadline() time.Duration {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	if t.readyDeadline == 0 {
+		return 10 * time.Second
+	}
+
+	return t.readyDeadline
+}
+
+func (t *apiConfig) getRequestsPool() (chan struct{}, <-chan time.Time) {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	if t.requestsPool == nil {
 		return nil, nil
 	}
 
@@ -57,7 +84,7 @@ func (t *apiThrottling) get() (chan struct{}, <-chan time.Time) {
 // maxClients throttles the S3 API calls
 func maxClients(f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		pool, deadlineTimer := globalAPIThrottling.get()
+		pool, deadlineTimer := globalAPIConfig.getRequestsPool()
 		if pool == nil {
 			f.ServeHTTP(w, r)
 			return

cmd/healthcheck-handler.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"net/http"
 )
 
@@ -28,7 +29,15 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
 
 	objLayer := newObjectLayerWithoutSafeModeFn()
 	// Service not initialized yet
-	if objLayer == nil || !objLayer.IsReady(ctx) {
+	if objLayer == nil {
+		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getReadyDeadline())
+	defer cancel()
+
+	if !objLayer.IsReady(ctx) {
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}

cmd/notification.go

@@ -1164,7 +1164,7 @@ func (sys *NotificationSys) ServerInfo() []madmin.ServerProperties {
 }
 
 // GetLocalDiskIDs - return disk ids of the local disks of the peers.
-func (sys *NotificationSys) GetLocalDiskIDs() []string {
+func (sys *NotificationSys) GetLocalDiskIDs(ctx context.Context) []string {
 	var diskIDs []string
 	var mu sync.Mutex
 
@@ -1176,7 +1176,7 @@ func (sys *NotificationSys) GetLocalDiskIDs() []string {
 		wg.Add(1)
 		go func(client *peerRESTClient) {
 			defer wg.Done()
-			ids := client.GetLocalDiskIDs()
+			ids := client.GetLocalDiskIDs(ctx)
 			mu.Lock()
 			diskIDs = append(diskIDs, ids...)
 			mu.Unlock()

cmd/peer-rest-client.go

@@ -677,19 +677,7 @@ func (client *peerRESTClient) BackgroundHealStatus() (madmin.BgHealState, error)
 }
 
 // GetLocalDiskIDs - get a peer's local disks' IDs.
-func (client *peerRESTClient) GetLocalDiskIDs() []string {
+func (client *peerRESTClient) GetLocalDiskIDs(ctx context.Context) []string {
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-
-	ctx, cancel := context.WithCancel(GlobalContext)
-	go func() {
-		select {
-		case <-doneCh:
-			return
-		case <-time.After(globalReadyDeadline):
-			cancel()
-		}
-	}()
 	respBody, err := client.callWithContext(ctx, peerRESTMethodGetLocalDiskIDs, nil, nil, -1)
 	if err != nil {
 		return nil

cmd/xl-v1.go

@@ -403,7 +403,7 @@ func (xl xlObjects) crawlAndGetDataUsage(ctx context.Context, buckets []BucketIn
 	return nil
 }
 
-// IsReady - No Op.
+// IsReady - shouldn't be called will panic.
 func (xl xlObjects) IsReady(ctx context.Context) bool {
 	logger.CriticalIf(ctx, NotImplemented{})
 	return true

cmd/xl-zones.go

@@ -1615,7 +1615,7 @@ func (z *xlZones) IsReady(ctx context.Context) bool {
 		erasureSetUpCount[i] = make([]int, len(z.zones[i].sets))
 	}
 
-	diskIDs := globalNotificationSys.GetLocalDiskIDs()
+	diskIDs := globalNotificationSys.GetLocalDiskIDs(ctx)
 
 	diskIDs = append(diskIDs, getLocalDiskIDs(z)...)
 

docs/config/README.md

@@ -172,7 +172,6 @@ MINIO_ETCD_COMMENT          (sentence)  optionally add a comment to this setting
 ```
 
 ### API
-
 By default, there is no limitation on the number of concurrents requests that a server/cluster processes at the same time. However, it is possible to impose such limitation using the API subsystem. Read more about throttling limitation in MinIO server [here](https://github.com/minio/minio/blob/master/docs/throttle/README.md).
 
 ```
@@ -180,16 +179,19 @@ KEY:
 api  manage global HTTP API call specific features, such as throttling, authentication types, etc.
 
 ARGS:
-requests_max       (number)     set the maximum number of concurrent requests
+requests_max       (number)    set the maximum number of concurrent requests, e.g. "1600"
-requests_deadline  (duration)   set the deadline for API requests waiting to be processed
+requests_deadline  (duration)  set the deadline for API requests waiting to be processed e.g. "1m"
+ready_deadline     (duration)  set the deadline for health check API /minio/health/ready e.g. "1m"
+cors_allow_origin  (csv)       set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com"
 ```
 
 or environment variables
 
 ```
-MINIO_API_REQUESTS_MAX        (number)     set the maximum number of concurrent requests
+MINIO_API_REQUESTS_MAX       (number)    set the maximum number of concurrent requests, e.g. "1600"
-MINIO_API_REQUESTS_DEADLINE   (duration)   set the deadline for API requests waiting to be processed
+MINIO_API_REQUESTS_DEADLINE  (duration)  set the deadline for API requests waiting to be processed e.g. "1m"
-
+MINIO_API_READY_DEADLINE     (duration)  set the deadline for health check API /minio/health/ready e.g. "1m"
+MINIO_API_CORS_ALLOW_ORIGIN  (csv)       set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com"
 ```
 
 #### Notifications

docs/metrics/healthcheck/README.md

@@ -16,8 +16,20 @@ This probe is used to identify situations where the server is not ready to accep
 
 Internally, MinIO readiness probe handler checks for backend is alive and in read quorum then the server returns 200 OK, otherwise 503 Service Unavailable.
 
-Platforms like Kubernetes *do not* forward traffic to a pod until its readiness probe is successful. 
+Platforms like Kubernetes *do not* forward traffic to a pod until its readiness probe is successful.
 
 ### Configuration example
 
 Sample `liveness` and `readiness` probe configuration in a Kubernetes `yaml` file can be found [here](https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes/minio-standalone-deployment.yaml).
+
+### Configure readiness deadline
+Readiness checks need to respond faster in orchestrated environments, to facilitate this you can use the following environment variable before starting MinIO
+
+```
+MINIO_API_READY_DEADLINE     (duration)  set the deadline for health check API /minio/health/ready e.g. "1m"
+```
+
+Set a *5s* deadline for MinIO to ensure readiness handler responds with-in 5seconds.
+```
+export MINIO_API_READY_DEADLINE=5s
+```

go.mod

@@ -94,7 +94,7 @@ require (
 	github.com/prometheus/client_golang v0.9.3
 	github.com/rcrowley/go-metrics v0.0.0-20190704165056-9c2d0518ed81 // indirect
 	github.com/rjeczalik/notify v0.9.2
-	github.com/rs/cors v1.6.0
+	github.com/rs/cors v1.7.0
 	github.com/secure-io/sio-go v0.3.0
 	github.com/shirou/gopsutil v2.20.3-0.20200314133625-53cec6b37e6a+incompatible
 	github.com/sirupsen/logrus v1.5.0

go.sum

@@ -377,8 +377,8 @@ github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w
 github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rs/cors v1.6.0 h1:G9tHG9lebljV9mfp9SNPDL36nCDxmo3zTlAf1YgvzmI=
+github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
-github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
 github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=