maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add basic scanner metrics (#13317)

Browse source 
Add number of objects/versions/folders scanned as well as ILM action outcomes.
This commit is contained in:
Klaus Post 2021-10-02 09:31:05 -07:00 committed by GitHub
parent f3aeed77e5
commit 75699a3825
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 152 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
cmd/data-scanner.go
View file

@ -29,6 +29,7 @@ import (
"path"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/bits-and-blooms/bloom/v3"
@ -194,6 +195,22 @@ type folderScanner struct {
lastUpdate time.Time
}
type scannerStats struct {
// All fields must be accessed atomically and aligned.
accTotalObjects uint64
accTotalVersions uint64
accFolders uint64
bucketsStarted uint64
bucketsFinished uint64
ilmChecks uint64
// actions records actions performed.
actions [lifecycle.ActionCount]uint64
}
var globalScannerStats scannerStats
// Cache structure and compaction:
//
// A cache structure will be kept with a tree of usages.
@ -241,6 +258,10 @@ func scanDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
logPrefix := color.Green("data-usage: ")
logSuffix := color.Blue("- %v + %v", basePath, cache.Info.Name)
atomic.AddUint64(&globalScannerStats.bucketsStarted, 1)
defer func() {
atomic.AddUint64(&globalScannerStats.bucketsFinished, 1)
}()
if intDataUpdateTracker.debug {
defer func() {
console.Debugf(logPrefix+" Scanner time: %v %s\n", time.Since(t), logSuffix)
@ -349,6 +370,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
thisHash := hashPath(folder.name)
// Store initial compaction state.
wasCompacted := into.Compacted
atomic.AddUint64(&globalScannerStats.accFolders, 1)
for {
select {
@ -876,6 +898,7 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
return false, size
}
atomic.AddUint64(&globalScannerStats.ilmChecks, 1)
versionID := oi.VersionID
action := i.lifeCycle.ComputeAction(
lifecycle.ObjectOpts{
@ -898,6 +921,8 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), action)
}
}
atomic.AddUint64(&globalScannerStats.actions[action], 1)
switch action {
case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
return applyLifecycleAction(action, oi), 0

2
cmd/fs-v1.go
View file

@ -393,6 +393,8 @@ func (fs *FSObjects) scanBucket(ctx context.Context, bucket string, cache dataUs
}
oi := fsMeta.ToObjectInfo(bucket, object, fi)
atomic.AddUint64(&globalScannerStats.accTotalVersions, 1)
atomic.AddUint64(&globalScannerStats.accTotalObjects, 1)
sz := item.applyActions(ctx, fs, oi, &sizeSummary{})
if sz >= 0 {
return sizeSummary{totalSize: sz, versions: 1}, nil

116
cmd/metrics-v2.go
View file

@ -24,8 +24,10 @@ import (
"runtime"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/minio/minio/internal/bucket/lifecycle"
"github.com/minio/minio/internal/logger"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
@ -69,6 +71,7 @@ const (
sysCallSubsystem MetricSubsystem = "syscall"
usageSubsystem MetricSubsystem = "usage"
ilmSubsystem MetricSubsystem = "ilm"
scannerSubsystem MetricSubsystem = "scanner"
)
// MetricName are the individual names for the metric.
@ -255,6 +258,7 @@ func GetGeneratorsForPeer() []MetricsGenerator {
getNetworkMetrics,
getS3TTFBMetric,
getILMNodeMetrics,
getScannerNodeMetrics,
}
return g
}
@ -1066,6 +1070,95 @@ func getILMNodeMetrics() MetricsGroup {
}
}
func getScannerNodeMetrics() MetricsGroup {
return MetricsGroup{
id: "ScannerNodeMetrics",
cachedRead: cachedRead,
read: func(_ context.Context) []Metric {
metrics := []Metric{
{
Description: MetricDescription{
Namespace: nodeMetricNamespace,
Subsystem: scannerSubsystem,
Name: "objects_scanned",
Help: "Total number of unique objects scanned since server start.",
Type: counterMetric,
},
Value: float64(atomic.LoadUint64(&globalScannerStats.accTotalObjects)),
},
{
Description: MetricDescription{
Namespace: nodeMetricNamespace,
Subsystem: scannerSubsystem,
Name: "versions_scanned",
Help: "Total number of object versions scanned since server start.",
Type: counterMetric,
},
Value: float64(atomic.LoadUint64(&globalScannerStats.accTotalVersions)),
},
{
Description: MetricDescription{
Namespace: nodeMetricNamespace,
Subsystem: scannerSubsystem,
Name: "directories_scanned",
Help: "Total number of directories scanned since server start.",
Type: counterMetric,
},
Value: float64(atomic.LoadUint64(&globalScannerStats.accFolders)),
},
{
Description: MetricDescription{
Namespace: nodeMetricNamespace,
Subsystem: scannerSubsystem,
Name: "bucket_scans_started",
Help: "Total number of bucket scans started since server start",
Type: counterMetric,
},
Value: float64(atomic.LoadUint64(&globalScannerStats.bucketsStarted)),
},
{
Description: MetricDescription{
Namespace: nodeMetricNamespace,
Subsystem: scannerSubsystem,
Name: "bucket_scans_finished",
Help: "Total number of bucket scans finished since server start",
Type: counterMetric,
},
Value: float64(atomic.LoadUint64(&globalScannerStats.bucketsFinished)),
},
{
Description: MetricDescription{
Namespace: nodeMetricNamespace,
Subsystem: ilmSubsystem,
Name: "versions_scanned",
Help: "Total number of object versions checked for ilm actions since server start",
Type: counterMetric,
},
Value: float64(atomic.LoadUint64(&globalScannerStats.ilmChecks)),
},
}
for i := range globalScannerStats.actions {
action := lifecycle.Action(i)
v := atomic.LoadUint64(&globalScannerStats.actions[action])
if v == 0 {
continue
}
metrics = append(metrics, Metric{
Description: MetricDescription{
Namespace: nodeMetricNamespace,
Subsystem: ilmSubsystem,
Name: MetricName("action_count_" + toSnake(action.String())),
Help: "Total action outcome of lifecycle checks since server start",
Type: counterMetric,
},
Value: float64(v),
})
}
return metrics
},
}
}
func getMinioVersionMetrics() MetricsGroup {
return MetricsGroup{
id: "MinioVersionMetrics",
@ -1753,3 +1846,26 @@ func metricsNodeHandler() http.Handler {
}),
)
}
func toSnake(camel string) (snake string) {
var b strings.Builder
l := len(camel)
for i, v := range camel {
// A is 65, a is 97
if v >= 'a' {
b.WriteRune(v)
continue
}
// v is capital letter here
// disregard first letter
// add underscore if last letter is capital letter
// add underscore when previous letter is lowercase
// add underscore when next letter is lowercase
if (i != 0 || i == l-1) && ((i > 0 && rune(camel[i-1]) >= 'a') ||
(i < l-1 && rune(camel[i+1]) >= 'a')) {
b.WriteRune('_')
}
b.WriteRune(v + 'a' - 'A')
}
return b.String()
}

3
cmd/xl-storage.go
View file

@ -33,6 +33,7 @@ import (
"runtime"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
@ -462,7 +463,9 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
return sizeSummary{}, errSkipFile
}
sizeS := sizeSummary{}
atomic.AddUint64(&globalScannerStats.accTotalObjects, 1)
for _, version := range fivs.Versions {
atomic.AddUint64(&globalScannerStats.accTotalVersions, 1)
oi := version.ToObjectInfo(item.bucket, item.objectPath())
sz := item.applyActions(ctx, objAPI, oi, &sizeS)
if !oi.DeleteMarker && sz == oi.Size {

5
internal/bucket/lifecycle/action_string.go
View file

@ -15,11 +15,12 @@ func _() {
_ = x[TransitionVersionAction-4]
_ = x[DeleteRestoredAction-5]
_ = x[DeleteRestoredVersionAction-6]
_ = x[ActionCount-7]
}
const _Action_name = "NoneActionDeleteActionDeleteVersionActionTransitionActionTransitionVersionActionDeleteRestoredActionDeleteRestoredVersionAction"
const _Action_name = "NoneActionDeleteActionDeleteVersionActionTransitionActionTransitionVersionActionDeleteRestoredActionDeleteRestoredVersionActionActionCount"
var _Action_index = [...]uint8{0, 10, 22, 41, 57, 80, 100, 127}
var _Action_index = [...]uint8{0, 10, 22, 41, 57, 80, 100, 127, 138}
func (i Action) String() string {
if i < 0 || i >= Action(len(_Action_index)-1) {

3
internal/bucket/lifecycle/lifecycle.go
View file

@ -63,6 +63,9 @@ const (
DeleteRestoredAction
// DeleteRestoredVersionAction deletes a particular version that was temporarily restored
DeleteRestoredVersionAction
// ActionCount must be the last action and shouldn't be used as a regular action.
ActionCount
)
// Lifecycle - Configuration for bucket lifecycle.