diff --git a/cmd/config-current.go b/cmd/config-current.go index 1e6f1880e..e6b3ba987 100644 --- a/cmd/config-current.go +++ b/cmd/config-current.go @@ -624,6 +624,8 @@ func applyDynamicConfig(ctx context.Context, objAPI ObjectLayer, s config.Config globalHealConfig = healCfg globalHealConfigMu.Unlock() + // update dynamic scanner values. + scannerCycle.Update(scannerCfg.Cycle) logger.LogIf(ctx, scannerSleeper.Update(scannerCfg.Delay, scannerCfg.MaxWait)) // Update all dynamic config values in memory. diff --git a/cmd/config/scanner/scanner.go b/cmd/config/scanner/scanner.go index 9400a3385..ae87d8402 100644 --- a/cmd/config/scanner/scanner.go +++ b/cmd/config/scanner/scanner.go @@ -1,5 +1,5 @@ /* - * MinIO Cloud Storage, (C) 2020 MinIO, Inc. + * MinIO Cloud Storage, (C) 2020-2021 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,8 +28,10 @@ import ( const ( Delay = "delay" MaxWait = "max_wait" + Cycle = "cycle" EnvDelay = "MINIO_SCANNER_DELAY" + EnvCycle = "MINIO_SCANNER_CYCLE" EnvDelayLegacy = "MINIO_CRAWLER_DELAY" EnvMaxWait = "MINIO_SCANNER_MAX_WAIT" EnvMaxWaitLegacy = "MINIO_CRAWLER_MAX_WAIT" @@ -41,6 +43,8 @@ type Config struct { Delay float64 `json:"delay"` // MaxWait is maximum wait time between operations MaxWait time.Duration + // Cycle is the time.Duration between each scanner cycles + Cycle time.Duration } var ( @@ -54,6 +58,10 @@ var ( Key: MaxWait, Value: "15s", }, + config.KV{ + Key: Cycle, + Value: "1m", + }, } // Help provides help for config values @@ -70,6 +78,12 @@ var ( Optional: true, Type: "duration", }, + config.HelpKV{ + Key: Cycle, + Description: `time duration between scanner cycles, defaults to '1m'`, + Optional: true, + Type: "duration", + }, } ) @@ -94,5 +108,10 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) { if err != nil { return cfg, err } + + cfg.Cycle, err = time.ParseDuration(env.Get(EnvCycle, kvs.Get(Cycle))) + if err != nil { + return cfg, err + } return cfg, nil } diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go index 25b5c5cd0..2332c8cc3 100644 --- a/cmd/data-scanner.go +++ b/cmd/data-scanner.go @@ -44,7 +44,6 @@ import ( const ( dataScannerSleepPerFolder = time.Millisecond // Time to wait between folders. - dataScannerStartDelay = 1 * time.Minute // Time to wait on startup and between cycles. dataUsageUpdateDirCycles = 16 // Visit all folders every n cycles. healDeleteDangling = true @@ -59,6 +58,7 @@ var ( dataScannerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second) // Sleeper values are updated when config is loaded. scannerSleeper = newDynamicSleeper(10, 10*time.Second) + scannerCycle = &safeDuration{} ) // initDataScanner will start the scanner in the background. @@ -66,6 +66,23 @@ func initDataScanner(ctx context.Context, objAPI ObjectLayer) { go runDataScanner(ctx, objAPI) } +type safeDuration struct { + sync.Mutex + t time.Duration +} + +func (s *safeDuration) Update(t time.Duration) { + s.Lock() + defer s.Unlock() + s.t = t +} + +func (s *safeDuration) Get() time.Duration { + s.Lock() + defer s.Unlock() + return s.t +} + // runDataScanner will start a data scanner. // The function will block until the context is canceled. // There should only ever be one scanner running per cluster. @@ -77,7 +94,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) { for { ctx, err = locker.GetLock(ctx, dataScannerLeaderLockTimeout) if err != nil { - time.Sleep(time.Duration(r.Float64() * float64(dataScannerStartDelay))) + time.Sleep(time.Duration(r.Float64() * float64(scannerCycle.Get()))) continue } break @@ -101,7 +118,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) { br.Close() } - scannerTimer := time.NewTimer(dataScannerStartDelay) + scannerTimer := time.NewTimer(scannerCycle.Get()) defer scannerTimer.Stop() for { @@ -110,7 +127,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) { return case <-scannerTimer.C: // Reset the timer for next cycle. - scannerTimer.Reset(dataScannerStartDelay) + scannerTimer.Reset(scannerCycle.Get()) if intDataUpdateTracker.debug { console.Debugln("starting scanner cycle") diff --git a/cmd/logger/target/http/http.go b/cmd/logger/target/http/http.go index fbdde3deb..2d9767161 100644 --- a/cmd/logger/target/http/http.go +++ b/cmd/logger/target/http/http.go @@ -130,8 +130,8 @@ func (h *Target) startHTTPLogger() { resp, err := h.client.Do(req) cancel() if err != nil { - logger.LogIf(ctx, fmt.Errorf("%s returned '%w', please check your endpoint configuration\n", - h.endpoint, err)) + logger.LogOnceIf(ctx, fmt.Errorf("%s returned '%w', please check your endpoint configuration", + h.endpoint, err), h.endpoint) continue } @@ -141,11 +141,11 @@ func (h *Target) startHTTPLogger() { if resp.StatusCode != http.StatusOK { switch resp.StatusCode { case http.StatusForbidden: - logger.LogIf(ctx, fmt.Errorf("%s returned '%s', please check if your auth token is correctly set", - h.endpoint, resp.Status)) + logger.LogOnceIf(ctx, fmt.Errorf("%s returned '%s', please check if your auth token is correctly set", + h.endpoint, resp.Status), h.endpoint) default: - logger.LogIf(ctx, fmt.Errorf("%s returned '%s', please check your endpoint configuration", - h.endpoint, resp.Status)) + logger.LogOnceIf(ctx, fmt.Errorf("%s returned '%s', please check your endpoint configuration", + h.endpoint, resp.Status), h.endpoint) } } }