diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go index a3b91b8e6..32b896df4 100644 --- a/cmd/data-scanner.go +++ b/cmd/data-scanner.go @@ -29,6 +29,7 @@ import ( "path" "strings" "sync" + "sync/atomic" "time" "github.com/bits-and-blooms/bloom/v3" @@ -194,6 +195,22 @@ type folderScanner struct { lastUpdate time.Time } +type scannerStats struct { + // All fields must be accessed atomically and aligned. + + accTotalObjects uint64 + accTotalVersions uint64 + accFolders uint64 + bucketsStarted uint64 + bucketsFinished uint64 + ilmChecks uint64 + + // actions records actions performed. + actions [lifecycle.ActionCount]uint64 +} + +var globalScannerStats scannerStats + // Cache structure and compaction: // // A cache structure will be kept with a tree of usages. @@ -241,6 +258,10 @@ func scanDataFolder(ctx context.Context, basePath string, cache dataUsageCache, logPrefix := color.Green("data-usage: ") logSuffix := color.Blue("- %v + %v", basePath, cache.Info.Name) + atomic.AddUint64(&globalScannerStats.bucketsStarted, 1) + defer func() { + atomic.AddUint64(&globalScannerStats.bucketsFinished, 1) + }() if intDataUpdateTracker.debug { defer func() { console.Debugf(logPrefix+" Scanner time: %v %s\n", time.Since(t), logSuffix) @@ -349,6 +370,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int thisHash := hashPath(folder.name) // Store initial compaction state. wasCompacted := into.Compacted + atomic.AddUint64(&globalScannerStats.accFolders, 1) for { select { @@ -876,6 +898,7 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje return false, size } + atomic.AddUint64(&globalScannerStats.ilmChecks, 1) versionID := oi.VersionID action := i.lifeCycle.ComputeAction( lifecycle.ObjectOpts{ @@ -898,6 +921,8 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), action) } } + atomic.AddUint64(&globalScannerStats.actions[action], 1) + switch action { case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction: return applyLifecycleAction(action, oi), 0 diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index c0863ea2b..5a8672e75 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -393,6 +393,8 @@ func (fs *FSObjects) scanBucket(ctx context.Context, bucket string, cache dataUs } oi := fsMeta.ToObjectInfo(bucket, object, fi) + atomic.AddUint64(&globalScannerStats.accTotalVersions, 1) + atomic.AddUint64(&globalScannerStats.accTotalObjects, 1) sz := item.applyActions(ctx, fs, oi, &sizeSummary{}) if sz >= 0 { return sizeSummary{totalSize: sz, versions: 1}, nil diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go index 68e4e806f..09657cb2d 100644 --- a/cmd/metrics-v2.go +++ b/cmd/metrics-v2.go @@ -24,8 +24,10 @@ import ( "runtime" "strings" "sync" + "sync/atomic" "time" + "github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/logger" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -69,6 +71,7 @@ const ( sysCallSubsystem MetricSubsystem = "syscall" usageSubsystem MetricSubsystem = "usage" ilmSubsystem MetricSubsystem = "ilm" + scannerSubsystem MetricSubsystem = "scanner" ) // MetricName are the individual names for the metric. @@ -255,6 +258,7 @@ func GetGeneratorsForPeer() []MetricsGenerator { getNetworkMetrics, getS3TTFBMetric, getILMNodeMetrics, + getScannerNodeMetrics, } return g } @@ -1066,6 +1070,95 @@ func getILMNodeMetrics() MetricsGroup { } } +func getScannerNodeMetrics() MetricsGroup { + return MetricsGroup{ + id: "ScannerNodeMetrics", + cachedRead: cachedRead, + read: func(_ context.Context) []Metric { + metrics := []Metric{ + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: scannerSubsystem, + Name: "objects_scanned", + Help: "Total number of unique objects scanned since server start.", + Type: counterMetric, + }, + Value: float64(atomic.LoadUint64(&globalScannerStats.accTotalObjects)), + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: scannerSubsystem, + Name: "versions_scanned", + Help: "Total number of object versions scanned since server start.", + Type: counterMetric, + }, + Value: float64(atomic.LoadUint64(&globalScannerStats.accTotalVersions)), + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: scannerSubsystem, + Name: "directories_scanned", + Help: "Total number of directories scanned since server start.", + Type: counterMetric, + }, + Value: float64(atomic.LoadUint64(&globalScannerStats.accFolders)), + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: scannerSubsystem, + Name: "bucket_scans_started", + Help: "Total number of bucket scans started since server start", + Type: counterMetric, + }, + Value: float64(atomic.LoadUint64(&globalScannerStats.bucketsStarted)), + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: scannerSubsystem, + Name: "bucket_scans_finished", + Help: "Total number of bucket scans finished since server start", + Type: counterMetric, + }, + Value: float64(atomic.LoadUint64(&globalScannerStats.bucketsFinished)), + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: ilmSubsystem, + Name: "versions_scanned", + Help: "Total number of object versions checked for ilm actions since server start", + Type: counterMetric, + }, + Value: float64(atomic.LoadUint64(&globalScannerStats.ilmChecks)), + }, + } + for i := range globalScannerStats.actions { + action := lifecycle.Action(i) + v := atomic.LoadUint64(&globalScannerStats.actions[action]) + if v == 0 { + continue + } + metrics = append(metrics, Metric{ + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: ilmSubsystem, + Name: MetricName("action_count_" + toSnake(action.String())), + Help: "Total action outcome of lifecycle checks since server start", + Type: counterMetric, + }, + Value: float64(v), + }) + } + return metrics + }, + } +} + func getMinioVersionMetrics() MetricsGroup { return MetricsGroup{ id: "MinioVersionMetrics", @@ -1753,3 +1846,26 @@ func metricsNodeHandler() http.Handler { }), ) } + +func toSnake(camel string) (snake string) { + var b strings.Builder + l := len(camel) + for i, v := range camel { + // A is 65, a is 97 + if v >= 'a' { + b.WriteRune(v) + continue + } + // v is capital letter here + // disregard first letter + // add underscore if last letter is capital letter + // add underscore when previous letter is lowercase + // add underscore when next letter is lowercase + if (i != 0 || i == l-1) && ((i > 0 && rune(camel[i-1]) >= 'a') || + (i < l-1 && rune(camel[i+1]) >= 'a')) { + b.WriteRune('_') + } + b.WriteRune(v + 'a' - 'A') + } + return b.String() +} diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index 5a095220d..8747c373e 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -33,6 +33,7 @@ import ( "runtime" "strings" "sync" + "sync/atomic" "syscall" "time" @@ -462,7 +463,9 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates return sizeSummary{}, errSkipFile } sizeS := sizeSummary{} + atomic.AddUint64(&globalScannerStats.accTotalObjects, 1) for _, version := range fivs.Versions { + atomic.AddUint64(&globalScannerStats.accTotalVersions, 1) oi := version.ToObjectInfo(item.bucket, item.objectPath()) sz := item.applyActions(ctx, objAPI, oi, &sizeS) if !oi.DeleteMarker && sz == oi.Size { diff --git a/internal/bucket/lifecycle/action_string.go b/internal/bucket/lifecycle/action_string.go index 5d36c14a0..89c3f8f73 100644 --- a/internal/bucket/lifecycle/action_string.go +++ b/internal/bucket/lifecycle/action_string.go @@ -15,11 +15,12 @@ func _() { _ = x[TransitionVersionAction-4] _ = x[DeleteRestoredAction-5] _ = x[DeleteRestoredVersionAction-6] + _ = x[ActionCount-7] } -const _Action_name = "NoneActionDeleteActionDeleteVersionActionTransitionActionTransitionVersionActionDeleteRestoredActionDeleteRestoredVersionAction" +const _Action_name = "NoneActionDeleteActionDeleteVersionActionTransitionActionTransitionVersionActionDeleteRestoredActionDeleteRestoredVersionActionActionCount" -var _Action_index = [...]uint8{0, 10, 22, 41, 57, 80, 100, 127} +var _Action_index = [...]uint8{0, 10, 22, 41, 57, 80, 100, 127, 138} func (i Action) String() string { if i < 0 || i >= Action(len(_Action_index)-1) { diff --git a/internal/bucket/lifecycle/lifecycle.go b/internal/bucket/lifecycle/lifecycle.go index 7570340a3..8ae8d729e 100644 --- a/internal/bucket/lifecycle/lifecycle.go +++ b/internal/bucket/lifecycle/lifecycle.go @@ -63,6 +63,9 @@ const ( DeleteRestoredAction // DeleteRestoredVersionAction deletes a particular version that was temporarily restored DeleteRestoredVersionAction + + // ActionCount must be the last action and shouldn't be used as a regular action. + ActionCount ) // Lifecycle - Configuration for bucket lifecycle.