maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

metrics: Use StorageInfo() instead to have consistent info (#9006)

Browse source 
Metrics used to have its own code to calculate offline disks.
StorageInfo() was avoided because it is an expensive operation
by sending calls to all nodes.

To make metrics & server info share the same code, a new
argument `local` is added to StorageInfo() so it will only
query local disks when needed.

Metrics now calls StorageInfo() as server info handler does
but with the local flag set to false.

Co-authored-by: Praveen raj Mani <praveen@minio.io>
Co-authored-by: Harshavardhana <harsha@minio.io>
This commit is contained in:
Anis Elleuch 2020-02-20 04:51:33 +01:00 committed by GitHub
parent 02a59a04d1
commit d4dcf1d722
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 89 additions and 106 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
cmd/admin-handlers.go
View file

@ -276,7 +276,7 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
return
}
storageInfo := objectAPI.StorageInfo(ctx)
storageInfo := objectAPI.StorageInfo(ctx, false)
// Marshal API response
jsonBytes, err := json.Marshal(storageInfo)
@ -897,7 +897,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
}
// find number of disks in the setup
info := objectAPI.StorageInfo(ctx)
info := objectAPI.StorageInfo(ctx, false)
numDisks := info.Backend.OfflineDisks.Sum() + info.Backend.OnlineDisks.Sum()
healPath := pathJoin(hip.bucket, hip.objPrefix)
@ -1410,7 +1410,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
notifyTarget := fetchLambdaInfo(cfg)
// Fetching the Storage information
storageInfo := objectAPI.StorageInfo(ctx)
storageInfo := objectAPI.StorageInfo(ctx, false)
var OnDisks int
var OffDisks int

2
cmd/background-heal-ops.go
View file

@ -127,7 +127,7 @@ func startBackgroundHealing() {
// Launch the background healer sequence to track
// background healing operations
info := objAPI.StorageInfo(ctx)
info := objAPI.StorageInfo(ctx, false)
numDisks := info.Backend.OnlineDisks.Sum() + info.Backend.OfflineDisks.Sum()
nh := newBgHealSequence(numDisks)
globalBackgroundHealState.LaunchNewHealSequence(nh)

2
cmd/fs-v1.go
View file

@ -202,7 +202,7 @@ func (fs *FSObjects) Shutdown(ctx context.Context) error {
}
// StorageInfo - returns underlying storage statistics.
func (fs *FSObjects) StorageInfo(ctx context.Context) StorageInfo {
func (fs *FSObjects) StorageInfo(ctx context.Context, _ bool) StorageInfo {
atomic.AddInt64(&fs.activeIOCount, 1)
defer func() {

2
cmd/gateway/azure/gateway-azure.go
View file

@ -481,7 +481,7 @@ func (a *azureObjects) Shutdown(ctx context.Context) error {
}
// StorageInfo - Not relevant to Azure backend.
func (a *azureObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
func (a *azureObjects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
si.Backend.Type = minio.BackendGateway
si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, a.httpClient, a.endpoint)
return si

2
cmd/gateway/b2/gateway-b2.go
View file

@ -218,7 +218,7 @@ func (l *b2Objects) Shutdown(ctx context.Context) error {
}
// StorageInfo is not relevant to B2 backend.
func (l *b2Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
func (l *b2Objects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
si.Backend.Type = minio.BackendGateway
si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.httpClient, "https://api.backblazeb2.com/b2api/v1")
return si

2
cmd/gateway/gcs/gateway-gcs.go
View file

@ -412,7 +412,7 @@ func (l *gcsGateway) Shutdown(ctx context.Context) error {
}
// StorageInfo - Not relevant to GCS backend.
func (l *gcsGateway) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
func (l *gcsGateway) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
si.Backend.Type = minio.BackendGateway
si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.httpClient, "https://storage.googleapis.com")
return si

2
cmd/gateway/hdfs/gateway-hdfs.go
View file

@ -203,7 +203,7 @@ func (n *hdfsObjects) Shutdown(ctx context.Context) error {
return n.clnt.Close()
}
func (n *hdfsObjects) StorageInfo(ctx context.Context) minio.StorageInfo {
func (n *hdfsObjects) StorageInfo(ctx context.Context, _ bool) minio.StorageInfo {
fsInfo, err := n.clnt.StatFs()
if err != nil {
return minio.StorageInfo{}

6
cmd/gateway/nas/gateway-nas.go
View file

@ -106,8 +106,8 @@ func (n *nasObjects) IsListenBucketSupported() bool {
return false
}
func (n *nasObjects) StorageInfo(ctx context.Context) minio.StorageInfo {
sinfo := n.ObjectLayer.StorageInfo(ctx)
func (n *nasObjects) StorageInfo(ctx context.Context, _ bool) minio.StorageInfo {
sinfo := n.ObjectLayer.StorageInfo(ctx, false)
sinfo.Backend.GatewayOnline = sinfo.Backend.Type == minio.BackendFS
sinfo.Backend.Type = minio.BackendGateway
return sinfo
@ -120,6 +120,6 @@ type nasObjects struct {
// IsReady returns whether the layer is ready to take requests.
func (n *nasObjects) IsReady(ctx context.Context) bool {
sinfo := n.ObjectLayer.StorageInfo(ctx)
sinfo := n.ObjectLayer.StorageInfo(ctx, false)
return sinfo.Backend.Type == minio.BackendFS
}

2
cmd/gateway/oss/gateway-oss.go
View file

@ -320,7 +320,7 @@ func (l *ossObjects) Shutdown(ctx context.Context) error {
}
// StorageInfo is not relevant to OSS backend.
func (l *ossObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
func (l *ossObjects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
si.Backend.Type = minio.BackendGateway
si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.Client.HTTPClient, l.Client.Config.Endpoint)
return si

2
cmd/gateway/s3/gateway-s3.go
View file

@ -272,7 +272,7 @@ func (l *s3Objects) Shutdown(ctx context.Context) error {
}
// StorageInfo is not relevant to S3 backend.
func (l *s3Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
func (l *s3Objects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
si.Backend.Type = minio.BackendGateway
si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.HTTPClient, l.Client.EndpointURL().String())
return si

2
cmd/healthcheck-handler.go
View file

@ -60,7 +60,7 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
}
if !globalIsXL && !globalIsDistXL {
s := objLayer.StorageInfo(ctx)
s := objLayer.StorageInfo(ctx, false)
if s.Backend.Type == BackendGateway {
if !s.Backend.GatewayOnline {
writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)

74
cmd/metrics.go
View file

@ -19,7 +19,6 @@ package cmd
import (
"context"
"net/http"
"strings"
"github.com/minio/minio/cmd/logger"
"github.com/prometheus/client_golang/prometheus"
@ -89,48 +88,35 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
return
}
storageAPIs := []StorageAPI{}
for _, ep := range globalEndpoints {
for _, endpoint := range ep.Endpoints {
if endpoint.IsLocal {
// Construct storageAPIs.
sAPI, _ := newStorageAPI(endpoint)
storageAPIs = append(storageAPIs, sAPI)
}
}
}
storageInfo := objLayer.StorageInfo(context.Background(), true)
disksInfo, onlineDisks, offlineDisks := getDisksInfo(storageAPIs)
offlineDisks := storageInfo.Backend.OfflineDisks
onlineDisks := storageInfo.Backend.OnlineDisks
totalDisks := offlineDisks.Merge(onlineDisks)
for _, offDisks := range offlineDisks {
// MinIO Offline Disks per node
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("minio", "disks", "offline"),
"Total number of offline disks in current MinIO server instance",
nil, nil),
prometheus.GaugeValue,
float64(offDisks),
)
}
// MinIO Offline Disks per node
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("minio", "disks", "offline"),
"Total number of offline disks in current MinIO server instance",
nil, nil),
prometheus.GaugeValue,
float64(offlineDisks.Sum()),
)
for _, totDisks := range totalDisks {
// MinIO Total Disks per node
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("minio", "disks", "total"),
"Total number of disks for current MinIO server instance",
nil, nil),
prometheus.GaugeValue,
float64(totDisks),
)
}
// MinIO Total Disks per node
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("minio", "disks", "total"),
"Total number of disks for current MinIO server instance",
nil, nil),
prometheus.GaugeValue,
float64(totalDisks.Sum()),
)
localPeer := GetLocalPeer(globalEndpoints)
for _, di := range disksInfo {
// Trim the host
absPath := strings.TrimPrefix(di.RelativePath, localPeer)
for i := 0; i < len(storageInfo.Total); i++ {
mountPath, total, free := storageInfo.MountPaths[i], storageInfo.Total[i],
storageInfo.Available[i]
// Total disk usage by the disk
ch <- prometheus.MustNewConstMetric(
@ -139,8 +125,8 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
"Total disk storage used on the disk",
[]string{"disk"}, nil),
prometheus.GaugeValue,
float64(di.Total-di.Free),
absPath,
float64(total-free),
mountPath,
)
// Total available space in the disk
@ -150,8 +136,8 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
"Total available space left on the disk",
[]string{"disk"}, nil),
prometheus.GaugeValue,
float64(di.Free),
absPath,
float64(free),
mountPath,
)
// Total storage space of the disk
@ -161,8 +147,8 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
"Total space on the disk",
[]string{"disk"}, nil),
prometheus.GaugeValue,
float64(di.Total),
absPath,
float64(total),
mountPath,
)
}

2
cmd/object-api-interface.go
View file

@ -60,7 +60,7 @@ type ObjectLayer interface {
// Storage operations.
Shutdown(context.Context) error
CrawlAndGetDataUsage(context.Context, <-chan struct{}) DataUsageInfo
StorageInfo(context.Context) StorageInfo
StorageInfo(ctx context.Context, local bool) StorageInfo // local queries only local disks
// Bucket operations.
MakeBucketWithLocation(ctx context.Context, bucket string, location string) error

25
cmd/posix.go
View file

@ -368,11 +368,11 @@ func (s *posix) CrawlAndGetDataUsage(endCh <-chan struct{}) (DataUsageInfo, erro
// DiskInfo is an extended type which returns current
// disk usage per path.
type DiskInfo struct {
Total uint64
Free uint64
Used uint64
RootDisk bool
RelativePath string
Total uint64
Free uint64
Used uint64
RootDisk bool
MountPath string
}
// DiskInfo provides current information about disk space usage,
@ -398,17 +398,12 @@ func (s *posix) DiskInfo() (info DiskInfo, err error) {
return info, err
}
localPeer := ""
if globalIsDistXL {
localPeer = GetLocalPeer(globalEndpoints)
}
return DiskInfo{
Total: di.Total,
Free: di.Free,
Used: used,
RootDisk: rootDisk,
RelativePath: localPeer + s.diskPath,
Total: di.Total,
Free: di.Free,
Used: used,
RootDisk: rootDisk,
MountPath: s.diskPath,
}, nil
}

4
cmd/server-startup-msg.go
View file

@ -54,7 +54,7 @@ func printStartupSafeModeMessage(apiEndpoints []string, err error) {
// Object layer is initialized then print StorageInfo in safe mode.
objAPI := newObjectLayerWithoutSafeModeFn()
if objAPI != nil {
if msg := getStorageInfoMsgSafeMode(objAPI.StorageInfo(context.Background())); msg != "" {
if msg := getStorageInfoMsgSafeMode(objAPI.StorageInfo(context.Background(), false)); msg != "" {
logStartupMessage(msg)
}
}
@ -116,7 +116,7 @@ func printStartupMessage(apiEndpoints []string) {
// Object layer is initialized then print StorageInfo.
objAPI := newObjectLayerFn()
if objAPI != nil {
printStorageInfo(objAPI.StorageInfo(context.Background()))
printStorageInfo(objAPI.StorageInfo(context.Background(), false))
}
// Prints credential, region and browser access.

2
cmd/web-handlers.go
View file

@ -129,7 +129,7 @@ func (web *webAPIHandlers) StorageInfo(r *http.Request, args *WebGenericArgs, re
if authErr != nil {
return toJSONError(ctx, authErr)
}
reply.StorageInfo = objectAPI.StorageInfo(ctx)
reply.StorageInfo = objectAPI.StorageInfo(ctx, false)
reply.UIVersion = browser.UIVersion
return nil
}

4
cmd/xl-sets.go
View file

@ -340,7 +340,7 @@ func (s *xlSets) NewNSLock(ctx context.Context, bucket string, object string) RW
}
// StorageInfo - combines output of StorageInfo across all erasure coded object sets.
func (s *xlSets) StorageInfo(ctx context.Context) StorageInfo {
func (s *xlSets) StorageInfo(ctx context.Context, local bool) StorageInfo {
var storageInfo StorageInfo
storageInfos := make([]StorageInfo, len(s.sets))
@ -350,7 +350,7 @@ func (s *xlSets) StorageInfo(ctx context.Context) StorageInfo {
for index := range s.sets {
index := index
g.Go(func() error {
storageInfos[index] = s.sets[index].StorageInfo(ctx)
storageInfos[index] = s.sets[index].StorageInfo(ctx, local)
return nil
}, index)
}

46
cmd/xl-v1.go
View file

@ -19,14 +19,12 @@ package cmd
import (
"context"
"sort"
"strings"
"sync"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/bpool"
"github.com/minio/minio/pkg/dsync"
"github.com/minio/minio/pkg/madmin"
xnet "github.com/minio/minio/pkg/net"
"github.com/minio/minio/pkg/sync/errgroup"
)
@ -115,34 +113,27 @@ func getDisksInfo(disks []StorageAPI) (disksInfo []DiskInfo, onlineDisks, offlin
}, index)
}
getPeerAddress := func(diskPath string) (string, error) {
hostPort := strings.Split(diskPath, SlashSeparator)[0]
// Host will be empty for xl/fs disk paths.
if hostPort == "" {
return "", nil
}
thisAddr, err := xnet.ParseHost(hostPort)
if err != nil {
return "", err
}
return thisAddr.String(), nil
}
onlineDisks = make(madmin.BackendDisks)
offlineDisks = make(madmin.BackendDisks)
localNodeAddr := GetLocalPeer(globalEndpoints)
// Wait for the routines.
for i, err := range g.Wait() {
peerAddr, pErr := getPeerAddress(disksInfo[i].RelativePath)
if pErr != nil {
for i, diskInfoErr := range g.Wait() {
if disks[i] == nil {
continue
}
peerAddr := disks[i].Hostname()
if peerAddr == "" {
peerAddr = localNodeAddr
}
if _, ok := offlineDisks[peerAddr]; !ok {
offlineDisks[peerAddr] = 0
}
if _, ok := onlineDisks[peerAddr]; !ok {
onlineDisks[peerAddr] = 0
}
if err != nil {
if diskInfoErr != nil {
offlineDisks[peerAddr]++
continue
}
@ -170,7 +161,7 @@ func getStorageInfo(disks []StorageAPI) StorageInfo {
usedList[i] = di.Used
totalList[i] = di.Total
availableList[i] = di.Free
mountPaths[i] = di.RelativePath
mountPaths[i] = di.MountPath
}
storageInfo := StorageInfo{
@ -188,8 +179,19 @@ func getStorageInfo(disks []StorageAPI) StorageInfo {
}
// StorageInfo - returns underlying storage statistics.
func (xl xlObjects) StorageInfo(ctx context.Context) StorageInfo {
return getStorageInfo(xl.getDisks())
func (xl xlObjects) StorageInfo(ctx context.Context, local bool) StorageInfo {
var disks []StorageAPI
if !local {
disks = xl.getDisks()
} else {
for _, d := range xl.getDisks() {
if d.Hostname() == "" {
// Append this local disk since local flag is true
disks = append(disks, d)
}
}
}
return getStorageInfo(disks)
}
// GetMetrics - is not implemented and shouldn't be called.

8
cmd/xl-zones.go
View file

@ -131,7 +131,7 @@ func (z *xlZones) getZonesAvailableSpace(ctx context.Context) zonesAvailableSpac
for index := range z.zones {
index := index
g.Go(func() error {
storageInfos[index] = z.zones[index].StorageInfo(ctx)
storageInfos[index] = z.zones[index].StorageInfo(ctx, false)
return nil
}, index)
}
@ -176,9 +176,9 @@ func (z *xlZones) Shutdown(ctx context.Context) error {
return nil
}
func (z *xlZones) StorageInfo(ctx context.Context) StorageInfo {
func (z *xlZones) StorageInfo(ctx context.Context, local bool) StorageInfo {
if z.SingleZone() {
return z.zones[0].StorageInfo(ctx)
return z.zones[0].StorageInfo(ctx, local)
}
var storageInfo StorageInfo
@ -188,7 +188,7 @@ func (z *xlZones) StorageInfo(ctx context.Context) StorageInfo {
for index := range z.zones {
index := index
g.Go(func() error {
storageInfos[index] = z.zones[index].StorageInfo(ctx)
storageInfos[index] = z.zones[index].StorageInfo(ctx, local)
return nil
}, index)
}