From 939fbb3c38b7b9c92b0f9b025e90a36ed9123f7f Mon Sep 17 00:00:00 2001 From: Krishnan Parthasarathi Date: Sat, 23 Oct 2021 21:38:33 -0400 Subject: [PATCH] ilm: Make per-tier stats available via admin-tier-info (#13381) --- cmd/admin-router.go | 3 + cmd/data-scanner.go | 1 + cmd/data-usage-cache.go | 186 +++- cmd/data-usage-cache_gen.go | 1579 +++++++++++++++++++++++++----- cmd/data-usage-cache_gen_test.go | 452 ++++++--- cmd/data-usage-utils.go | 48 +- cmd/object-api-datatypes.go | 12 + cmd/tier-handlers.go | 43 + cmd/tier.go | 3 +- cmd/xl-storage.go | 19 + go.mod | 15 +- go.sum | 54 +- 12 files changed, 2029 insertions(+), 386 deletions(-) diff --git a/cmd/admin-router.go b/cmd/admin-router.go index 1df120dbf..256def854 100644 --- a/cmd/admin-router.go +++ b/cmd/admin-router.go @@ -192,6 +192,9 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) { adminRouter.Methods(http.MethodPost).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.EditTierHandler))) adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListTierHandler))) + // Tier stats + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier-stats").HandlerFunc(gz(httpTraceHdrs(adminAPI.TierStatsHandler))) + // Cluster Replication APIs adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/add").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationAdd))) adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/disable").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationDisable))) diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go index 32b896df4..26f097b77 100644 --- a/cmd/data-scanner.go +++ b/cmd/data-scanner.go @@ -838,6 +838,7 @@ type sizeSummary struct { pendingCount uint64 failedCount uint64 replTargetStats map[string]replTargetSizeSummary + tiers map[string]tierStats } // replTargetSizeSummary holds summary of replication stats by target diff --git a/cmd/data-usage-cache.go b/cmd/data-usage-cache.go index eb79a36b0..681b1b8fe 100644 --- a/cmd/data-usage-cache.go +++ b/cmd/data-usage-cache.go @@ -31,6 +31,7 @@ import ( "github.com/cespare/xxhash/v2" "github.com/klauspost/compress/zstd" + "github.com/minio/madmin-go" "github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/hash" "github.com/minio/minio/internal/logger" @@ -45,16 +46,70 @@ type dataUsageHash string // sizeHistogram is a size histogram. type sizeHistogram [dataUsageBucketLen]uint64 -//msgp:tuple dataUsageEntry type dataUsageEntry struct { - Children dataUsageHashMap + Children dataUsageHashMap `msg:"ch"` // These fields do no include any children. - Size int64 - Objects uint64 - Versions uint64 // Versions that are not delete markers. - ObjSizes sizeHistogram - ReplicationStats *replicationAllStats - Compacted bool + Size int64 `msg:"sz"` + Objects uint64 `msg:"os"` + Versions uint64 `msg:"vs"` // Versions that are not delete markers. + ObjSizes sizeHistogram `msg:"szs"` + ReplicationStats *replicationAllStats `msg:"rs,omitempty"` + AllTierStats *allTierStats `msg:"ats,omitempty"` + Compacted bool `msg:"c"` +} + +// allTierStats is a collection of per-tier stats across all configured remote +// tiers. +type allTierStats struct { + Tiers map[string]tierStats `msg:"ts"` +} + +func newAllTierStats() *allTierStats { + return &allTierStats{ + Tiers: make(map[string]tierStats), + } +} + +func (ats *allTierStats) addSizes(sz sizeSummary) { + for tier, st := range sz.tiers { + ats.Tiers[tier] = ats.Tiers[tier].add(st) + } +} + +func (ats *allTierStats) merge(other *allTierStats) { + for tier, st := range other.Tiers { + ats.Tiers[tier] = ats.Tiers[tier].add(st) + } +} + +func (ats *allTierStats) adminStats(stats map[string]madmin.TierStats) map[string]madmin.TierStats { + if ats == nil { + return stats + } + + // Update stats for tiers as they become available. + for tier, st := range ats.Tiers { + stats[tier] = madmin.TierStats{ + TotalSize: st.TotalSize, + NumVersions: st.NumVersions, + NumObjects: st.NumObjects, + } + } + return stats +} + +// tierStats holds per-tier stats of a remote tier. +type tierStats struct { + TotalSize uint64 `msg:"ts"` + NumVersions int `msg:"nv"` + NumObjects int `msg:"no"` +} + +func (ts tierStats) add(u tierStats) tierStats { + ts.TotalSize += u.TotalSize + ts.NumVersions += u.NumVersions + ts.NumObjects += u.NumObjects + return ts } //msgp:tuple replicationStatsV1 @@ -96,14 +151,19 @@ func (rs replicationStats) Empty() bool { rs.FailedCount == 0 } -//msgp:tuple replicationAllStats type replicationAllStats struct { + Targets map[string]replicationStats `msg:"t,omitempty"` + ReplicaSize uint64 `msg:"r,omitempty"` +} + +//msgp:tuple replicationAllStatsV1 +type replicationAllStatsV1 struct { Targets map[string]replicationStats ReplicaSize uint64 `msg:"ReplicaSize,omitempty"` } -//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 -//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 +//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 +//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 //msgp:tuple dataUsageEntryV2 type dataUsageEntryV2 struct { @@ -149,6 +209,18 @@ type dataUsageEntryV5 struct { Compacted bool } +//msgp:tuple dataUsageEntryV6 +type dataUsageEntryV6 struct { + Children dataUsageHashMap + // These fields do no include any children. + Size int64 + Objects uint64 + Versions uint64 // Versions that are not delete markers. + ObjSizes sizeHistogram + ReplicationStats *replicationAllStatsV1 + Compacted bool +} + // dataUsageCache contains a cache of data usage entries latest version. type dataUsageCache struct { Info dataUsageCacheInfo @@ -156,8 +228,8 @@ type dataUsageCache struct { Disks []string } -//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 -//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 +//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 +//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 // dataUsageCacheV2 contains a cache of data usage entries version 2. type dataUsageCacheV2 struct { @@ -166,27 +238,34 @@ type dataUsageCacheV2 struct { Cache map[string]dataUsageEntryV2 } -// dataUsageCache contains a cache of data usage entries version 3. +// dataUsageCacheV3 contains a cache of data usage entries version 3. type dataUsageCacheV3 struct { Info dataUsageCacheInfo Disks []string Cache map[string]dataUsageEntryV3 } -// dataUsageCache contains a cache of data usage entries version 4. +// dataUsageCacheV4 contains a cache of data usage entries version 4. type dataUsageCacheV4 struct { Info dataUsageCacheInfo Disks []string Cache map[string]dataUsageEntryV4 } -// dataUsageCache contains a cache of data usage entries version 5. +// dataUsageCacheV5 contains a cache of data usage entries version 5. type dataUsageCacheV5 struct { Info dataUsageCacheInfo Disks []string Cache map[string]dataUsageEntryV5 } +// dataUsageCacheV6 contains a cache of data usage entries version 6. +type dataUsageCacheV6 struct { + Info dataUsageCacheInfo + Disks []string + Cache map[string]dataUsageEntryV6 +} + //msgp:ignore dataUsageEntryInfo type dataUsageEntryInfo struct { Name string @@ -242,6 +321,12 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) { e.ReplicationStats.Targets[arn] = tgtStat } } + if summary.tiers != nil { + if e.AllTierStats == nil { + e.AllTierStats = newAllTierStats() + } + e.AllTierStats.addSizes(summary) + } } // merge other data usage entry into this, excluding children. @@ -271,6 +356,13 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) { for i, v := range other.ObjSizes[:] { e.ObjSizes[i] += v } + + if other.AllTierStats != nil { + if e.AllTierStats == nil { + e.AllTierStats = newAllTierStats() + } + e.AllTierStats.merge(other.AllTierStats) + } } // mod returns true if the hash mod cycles == cycle. @@ -317,6 +409,11 @@ func (e dataUsageEntry) clone() dataUsageEntry { r := *e.ReplicationStats e.ReplicationStats = &r } + if e.AllTierStats != nil { + ats := newAllTierStats() + ats.merge(e.AllTierStats) + e.AllTierStats = ats + } return e } @@ -438,6 +535,7 @@ func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo { ObjectsTotalSize: uint64(flat.Size), BucketsCount: uint64(len(e.Children)), BucketsUsage: d.bucketsUsageInfo(buckets), + TierStats: d.tiersUsageInfo(buckets), } return dui } @@ -654,6 +752,25 @@ func (h *sizeHistogram) toMap() map[string]uint64 { return res } +func (d *dataUsageCache) tiersUsageInfo(buckets []BucketInfo) *allTierStats { + dst := newAllTierStats() + for _, bucket := range buckets { + e := d.find(bucket.Name) + if e == nil { + continue + } + flat := d.flatten(*e) + if flat.AllTierStats == nil { + continue + } + dst.merge(flat.AllTierStats) + } + if len(dst.Tiers) == 0 { + return nil + } + return dst +} + // bucketsUsageInfo returns the buckets usage info as a map, with // key as bucket name func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]BucketUsageInfo { @@ -857,7 +974,8 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string) // Bumping the cache version will drop data from previous versions // and write new data with the new version. const ( - dataUsageCacheVerCurrent = 6 + dataUsageCacheVerCurrent = 7 + dataUsageCacheVerV6 = 6 dataUsageCacheVerV5 = 5 dataUsageCacheVerV4 = 4 dataUsageCacheVerV3 = 3 @@ -1086,6 +1204,40 @@ func (d *dataUsageCache) deserialize(r io.Reader) error { d.Cache[k] = e } return nil + case dataUsageCacheVerV6: + // Zstd compressed. + dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2)) + if err != nil { + return err + } + defer dec.Close() + dold := &dataUsageCacheV6{} + if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil { + return err + } + d.Info = dold.Info + d.Disks = dold.Disks + d.Cache = make(map[string]dataUsageEntry, len(dold.Cache)) + for k, v := range dold.Cache { + var replicationStats *replicationAllStats + if v.ReplicationStats != nil { + replicationStats = &replicationAllStats{ + Targets: v.ReplicationStats.Targets, + ReplicaSize: v.ReplicationStats.ReplicaSize, + } + } + due := dataUsageEntry{ + Children: v.Children, + Size: v.Size, + Objects: v.Objects, + Versions: v.Versions, + ObjSizes: v.ObjSizes, + ReplicationStats: replicationStats, + Compacted: v.Compacted, + } + d.Cache[k] = due + } + return nil case dataUsageCacheVerCurrent: // Zstd compressed. dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2)) diff --git a/cmd/data-usage-cache_gen.go b/cmd/data-usage-cache_gen.go index 3e0ef1ae7..deb79d3cb 100644 --- a/cmd/data-usage-cache_gen.go +++ b/cmd/data-usage-cache_gen.go @@ -6,6 +6,284 @@ import ( "github.com/tinylib/msgp/msgp" ) +// DecodeMsg implements msgp.Decodable +func (z *allTierStats) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ts": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Tiers") + return + } + if z.Tiers == nil { + z.Tiers = make(map[string]tierStats, zb0002) + } else if len(z.Tiers) > 0 { + for key := range z.Tiers { + delete(z.Tiers, key) + } + } + for zb0002 > 0 { + zb0002-- + var za0001 string + var za0002 tierStats + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Tiers") + return + } + var zb0003 uint32 + zb0003, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001) + return + } + for zb0003 > 0 { + zb0003-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001) + return + } + switch msgp.UnsafeString(field) { + case "ts": + za0002.TotalSize, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001, "TotalSize") + return + } + case "nv": + za0002.NumVersions, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001, "NumVersions") + return + } + case "no": + za0002.NumObjects, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001, "NumObjects") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001) + return + } + } + } + z.Tiers[za0001] = za0002 + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *allTierStats) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "ts" + err = en.Append(0x81, 0xa2, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.Tiers))) + if err != nil { + err = msgp.WrapError(err, "Tiers") + return + } + for za0001, za0002 := range z.Tiers { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Tiers") + return + } + // map header, size 3 + // write "ts" + err = en.Append(0x83, 0xa2, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteUint64(za0002.TotalSize) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001, "TotalSize") + return + } + // write "nv" + err = en.Append(0xa2, 0x6e, 0x76) + if err != nil { + return + } + err = en.WriteInt(za0002.NumVersions) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001, "NumVersions") + return + } + // write "no" + err = en.Append(0xa2, 0x6e, 0x6f) + if err != nil { + return + } + err = en.WriteInt(za0002.NumObjects) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001, "NumObjects") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *allTierStats) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "ts" + o = append(o, 0x81, 0xa2, 0x74, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.Tiers))) + for za0001, za0002 := range z.Tiers { + o = msgp.AppendString(o, za0001) + // map header, size 3 + // string "ts" + o = append(o, 0x83, 0xa2, 0x74, 0x73) + o = msgp.AppendUint64(o, za0002.TotalSize) + // string "nv" + o = append(o, 0xa2, 0x6e, 0x76) + o = msgp.AppendInt(o, za0002.NumVersions) + // string "no" + o = append(o, 0xa2, 0x6e, 0x6f) + o = msgp.AppendInt(o, za0002.NumObjects) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *allTierStats) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ts": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tiers") + return + } + if z.Tiers == nil { + z.Tiers = make(map[string]tierStats, zb0002) + } else if len(z.Tiers) > 0 { + for key := range z.Tiers { + delete(z.Tiers, key) + } + } + for zb0002 > 0 { + var za0001 string + var za0002 tierStats + zb0002-- + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tiers") + return + } + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001) + return + } + for zb0003 > 0 { + zb0003-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001) + return + } + switch msgp.UnsafeString(field) { + case "ts": + za0002.TotalSize, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001, "TotalSize") + return + } + case "nv": + za0002.NumVersions, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001, "NumVersions") + return + } + case "no": + za0002.NumObjects, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001, "NumObjects") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001) + return + } + } + } + z.Tiers[za0001] = za0002 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *allTierStats) Msgsize() (s int) { + s = 1 + 3 + msgp.MapHeaderSize + if z.Tiers != nil { + for za0001, za0002 := range z.Tiers { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + 1 + 3 + msgp.Uint64Size + 3 + msgp.IntSize + 3 + msgp.IntSize + } + } + return +} + // DecodeMsg implements msgp.Decodable func (z *dataUsageCache) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte @@ -1250,83 +1528,326 @@ func (z *dataUsageCacheV5) Msgsize() (s int) { } // DecodeMsg implements msgp.Decodable -func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) { +func (z *dataUsageCacheV6) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field var zb0001 uint32 - zb0001, err = dc.ReadArrayHeader() + zb0001, err = dc.ReadMapHeader() if err != nil { err = msgp.WrapError(err) return } - if zb0001 != 7 { - err = msgp.ArrayError{Wanted: 7, Got: zb0001} - return - } - err = z.Children.DecodeMsg(dc) - if err != nil { - err = msgp.WrapError(err, "Children") - return - } - z.Size, err = dc.ReadInt64() - if err != nil { - err = msgp.WrapError(err, "Size") - return - } - z.Objects, err = dc.ReadUint64() - if err != nil { - err = msgp.WrapError(err, "Objects") - return - } - z.Versions, err = dc.ReadUint64() - if err != nil { - err = msgp.WrapError(err, "Versions") - return - } - var zb0002 uint32 - zb0002, err = dc.ReadArrayHeader() - if err != nil { - err = msgp.WrapError(err, "ObjSizes") - return - } - if zb0002 != uint32(dataUsageBucketLen) { - err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} - return - } - for za0001 := range z.ObjSizes { - z.ObjSizes[za0001], err = dc.ReadUint64() + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() if err != nil { - err = msgp.WrapError(err, "ObjSizes", za0001) + err = msgp.WrapError(err) return } - } - if dc.IsNil() { - err = dc.ReadNil() - if err != nil { - err = msgp.WrapError(err, "ReplicationStats") - return - } - z.ReplicationStats = nil - } else { - if z.ReplicationStats == nil { - z.ReplicationStats = new(replicationAllStats) - } - err = z.ReplicationStats.DecodeMsg(dc) - if err != nil { - err = msgp.WrapError(err, "ReplicationStats") - return + switch msgp.UnsafeString(field) { + case "Info": + err = z.Info.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + case "Disks": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Disks") + return + } + if cap(z.Disks) >= int(zb0002) { + z.Disks = (z.Disks)[:zb0002] + } else { + z.Disks = make([]string, zb0002) + } + for za0001 := range z.Disks { + z.Disks[za0001], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Disks", za0001) + return + } + } + case "Cache": + var zb0003 uint32 + zb0003, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + if z.Cache == nil { + z.Cache = make(map[string]dataUsageEntryV6, zb0003) + } else if len(z.Cache) > 0 { + for key := range z.Cache { + delete(z.Cache, key) + } + } + for zb0003 > 0 { + zb0003-- + var za0002 string + var za0003 dataUsageEntryV6 + za0002, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + err = za0003.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Cache", za0002) + return + } + z.Cache[za0002] = za0003 + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } } } - z.Compacted, err = dc.ReadBool() + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageCacheV6) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) if err != nil { - err = msgp.WrapError(err, "Compacted") + err = msgp.WrapError(err) return } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Info": + bts, err = z.Info.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + case "Disks": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Disks") + return + } + if cap(z.Disks) >= int(zb0002) { + z.Disks = (z.Disks)[:zb0002] + } else { + z.Disks = make([]string, zb0002) + } + for za0001 := range z.Disks { + z.Disks[za0001], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Disks", za0001) + return + } + } + case "Cache": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + if z.Cache == nil { + z.Cache = make(map[string]dataUsageEntryV6, zb0003) + } else if len(z.Cache) > 0 { + for key := range z.Cache { + delete(z.Cache, key) + } + } + for zb0003 > 0 { + var za0002 string + var za0003 dataUsageEntryV6 + zb0003-- + za0002, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + bts, err = za0003.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Cache", za0002) + return + } + z.Cache[za0002] = za0003 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *dataUsageCacheV6) Msgsize() (s int) { + s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.ArrayHeaderSize + for za0001 := range z.Disks { + s += msgp.StringPrefixSize + len(z.Disks[za0001]) + } + s += 6 + msgp.MapHeaderSize + if z.Cache != nil { + for za0002, za0003 := range z.Cache { + _ = za0003 + s += msgp.StringPrefixSize + len(za0002) + za0003.Msgsize() + } + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ch": + err = z.Children.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + case "sz": + z.Size, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "os": + z.Objects, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + case "vs": + z.Versions, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + case "szs": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + case "rs": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "ReplicationStats") + return + } + z.ReplicationStats = nil + } else { + if z.ReplicationStats == nil { + z.ReplicationStats = new(replicationAllStats) + } + err = z.ReplicationStats.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "ReplicationStats") + return + } + } + case "ats": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + z.AllTierStats = nil + } else { + if z.AllTierStats == nil { + z.AllTierStats = new(allTierStats) + } + err = z.AllTierStats.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + } + case "c": + z.Compacted, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Compacted") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } return } // EncodeMsg implements msgp.Encodable func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { - // array header, size 7 - err = en.Append(0x97) + // omitempty: check for empty values + zb0001Len := uint32(8) + var zb0001Mask uint8 /* 8 bits */ + if z.ReplicationStats == nil { + zb0001Len-- + zb0001Mask |= 0x20 + } + if z.AllTierStats == nil { + zb0001Len-- + zb0001Mask |= 0x40 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) + if err != nil { + return + } + if zb0001Len == 0 { + return + } + // write "ch" + err = en.Append(0xa2, 0x63, 0x68) if err != nil { return } @@ -1335,21 +1856,41 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "Children") return } + // write "sz" + err = en.Append(0xa2, 0x73, 0x7a) + if err != nil { + return + } err = en.WriteInt64(z.Size) if err != nil { err = msgp.WrapError(err, "Size") return } + // write "os" + err = en.Append(0xa2, 0x6f, 0x73) + if err != nil { + return + } err = en.WriteUint64(z.Objects) if err != nil { err = msgp.WrapError(err, "Objects") return } + // write "vs" + err = en.Append(0xa2, 0x76, 0x73) + if err != nil { + return + } err = en.WriteUint64(z.Versions) if err != nil { err = msgp.WrapError(err, "Versions") return } + // write "szs" + err = en.Append(0xa3, 0x73, 0x7a, 0x73) + if err != nil { + return + } err = en.WriteArrayHeader(uint32(dataUsageBucketLen)) if err != nil { err = msgp.WrapError(err, "ObjSizes") @@ -1362,17 +1903,48 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { return } } - if z.ReplicationStats == nil { - err = en.WriteNil() + if (zb0001Mask & 0x20) == 0 { // if not empty + // write "rs" + err = en.Append(0xa2, 0x72, 0x73) if err != nil { return } - } else { - err = z.ReplicationStats.EncodeMsg(en) + if z.ReplicationStats == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.ReplicationStats.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "ReplicationStats") + return + } + } + } + if (zb0001Mask & 0x40) == 0 { // if not empty + // write "ats" + err = en.Append(0xa3, 0x61, 0x74, 0x73) if err != nil { - err = msgp.WrapError(err, "ReplicationStats") return } + if z.AllTierStats == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.AllTierStats.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + } + } + // write "c" + err = en.Append(0xa1, 0x63) + if err != nil { + return } err = en.WriteBool(z.Compacted) if err != nil { @@ -1385,102 +1957,183 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { // MarshalMsg implements msgp.Marshaler func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // array header, size 7 - o = append(o, 0x97) + // omitempty: check for empty values + zb0001Len := uint32(8) + var zb0001Mask uint8 /* 8 bits */ + if z.ReplicationStats == nil { + zb0001Len-- + zb0001Mask |= 0x20 + } + if z.AllTierStats == nil { + zb0001Len-- + zb0001Mask |= 0x40 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len == 0 { + return + } + // string "ch" + o = append(o, 0xa2, 0x63, 0x68) o, err = z.Children.MarshalMsg(o) if err != nil { err = msgp.WrapError(err, "Children") return } + // string "sz" + o = append(o, 0xa2, 0x73, 0x7a) o = msgp.AppendInt64(o, z.Size) + // string "os" + o = append(o, 0xa2, 0x6f, 0x73) o = msgp.AppendUint64(o, z.Objects) + // string "vs" + o = append(o, 0xa2, 0x76, 0x73) o = msgp.AppendUint64(o, z.Versions) + // string "szs" + o = append(o, 0xa3, 0x73, 0x7a, 0x73) o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen)) for za0001 := range z.ObjSizes { o = msgp.AppendUint64(o, z.ObjSizes[za0001]) } - if z.ReplicationStats == nil { - o = msgp.AppendNil(o) - } else { - o, err = z.ReplicationStats.MarshalMsg(o) - if err != nil { - err = msgp.WrapError(err, "ReplicationStats") - return + if (zb0001Mask & 0x20) == 0 { // if not empty + // string "rs" + o = append(o, 0xa2, 0x72, 0x73) + if z.ReplicationStats == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.ReplicationStats.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ReplicationStats") + return + } } } + if (zb0001Mask & 0x40) == 0 { // if not empty + // string "ats" + o = append(o, 0xa3, 0x61, 0x74, 0x73) + if z.AllTierStats == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.AllTierStats.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + } + } + // string "c" + o = append(o, 0xa1, 0x63) o = msgp.AppendBool(o, z.Compacted) return } // UnmarshalMsg implements msgp.Unmarshaler func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field var zb0001 uint32 - zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) if err != nil { err = msgp.WrapError(err) return } - if zb0001 != 7 { - err = msgp.ArrayError{Wanted: 7, Got: zb0001} - return - } - bts, err = z.Children.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Children") - return - } - z.Size, bts, err = msgp.ReadInt64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "Size") - return - } - z.Objects, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "Objects") - return - } - z.Versions, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "Versions") - return - } - var zb0002 uint32 - zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "ObjSizes") - return - } - if zb0002 != uint32(dataUsageBucketLen) { - err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} - return - } - for za0001 := range z.ObjSizes { - z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts) + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) if err != nil { - err = msgp.WrapError(err, "ObjSizes", za0001) + err = msgp.WrapError(err) return } - } - if msgp.IsNil(bts) { - bts, err = msgp.ReadNilBytes(bts) - if err != nil { - return + switch msgp.UnsafeString(field) { + case "ch": + bts, err = z.Children.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + case "sz": + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "os": + z.Objects, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + case "vs": + z.Versions, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + case "szs": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + case "rs": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.ReplicationStats = nil + } else { + if z.ReplicationStats == nil { + z.ReplicationStats = new(replicationAllStats) + } + bts, err = z.ReplicationStats.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicationStats") + return + } + } + case "ats": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.AllTierStats = nil + } else { + if z.AllTierStats == nil { + z.AllTierStats = new(allTierStats) + } + bts, err = z.AllTierStats.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + } + case "c": + z.Compacted, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Compacted") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } } - z.ReplicationStats = nil - } else { - if z.ReplicationStats == nil { - z.ReplicationStats = new(replicationAllStats) - } - bts, err = z.ReplicationStats.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "ReplicationStats") - return - } - } - z.Compacted, bts, err = msgp.ReadBoolBytes(bts) - if err != nil { - err = msgp.WrapError(err, "Compacted") - return } o = bts return @@ -1488,13 +2141,19 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *dataUsageEntry) Msgsize() (s int) { - s = 1 + z.Children.Msgsize() + msgp.Int64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 if z.ReplicationStats == nil { s += msgp.NilSize } else { s += z.ReplicationStats.Msgsize() } - s += msgp.BoolSize + s += 4 + if z.AllTierStats == nil { + s += msgp.NilSize + } else { + s += z.AllTierStats.Msgsize() + } + s += 2 + msgp.BoolSize return } @@ -1925,95 +2584,6 @@ func (z *dataUsageEntryV5) DecodeMsg(dc *msgp.Reader) (err error) { return } -// EncodeMsg implements msgp.Encodable -func (z *dataUsageEntryV5) EncodeMsg(en *msgp.Writer) (err error) { - // array header, size 7 - err = en.Append(0x97) - if err != nil { - return - } - err = z.Children.EncodeMsg(en) - if err != nil { - err = msgp.WrapError(err, "Children") - return - } - err = en.WriteInt64(z.Size) - if err != nil { - err = msgp.WrapError(err, "Size") - return - } - err = en.WriteUint64(z.Objects) - if err != nil { - err = msgp.WrapError(err, "Objects") - return - } - err = en.WriteUint64(z.Versions) - if err != nil { - err = msgp.WrapError(err, "Versions") - return - } - err = en.WriteArrayHeader(uint32(dataUsageBucketLen)) - if err != nil { - err = msgp.WrapError(err, "ObjSizes") - return - } - for za0001 := range z.ObjSizes { - err = en.WriteUint64(z.ObjSizes[za0001]) - if err != nil { - err = msgp.WrapError(err, "ObjSizes", za0001) - return - } - } - if z.ReplicationStats == nil { - err = en.WriteNil() - if err != nil { - return - } - } else { - err = z.ReplicationStats.EncodeMsg(en) - if err != nil { - err = msgp.WrapError(err, "ReplicationStats") - return - } - } - err = en.WriteBool(z.Compacted) - if err != nil { - err = msgp.WrapError(err, "Compacted") - return - } - return -} - -// MarshalMsg implements msgp.Marshaler -func (z *dataUsageEntryV5) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - // array header, size 7 - o = append(o, 0x97) - o, err = z.Children.MarshalMsg(o) - if err != nil { - err = msgp.WrapError(err, "Children") - return - } - o = msgp.AppendInt64(o, z.Size) - o = msgp.AppendUint64(o, z.Objects) - o = msgp.AppendUint64(o, z.Versions) - o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen)) - for za0001 := range z.ObjSizes { - o = msgp.AppendUint64(o, z.ObjSizes[za0001]) - } - if z.ReplicationStats == nil { - o = msgp.AppendNil(o) - } else { - o, err = z.ReplicationStats.MarshalMsg(o) - if err != nil { - err = msgp.WrapError(err, "ReplicationStats") - return - } - } - o = msgp.AppendBool(o, z.Compacted) - return -} - // UnmarshalMsg implements msgp.Unmarshaler func (z *dataUsageEntryV5) UnmarshalMsg(bts []byte) (o []byte, err error) { var zb0001 uint32 @@ -2100,6 +2670,166 @@ func (z *dataUsageEntryV5) Msgsize() (s int) { return } +// DecodeMsg implements msgp.Decodable +func (z *dataUsageEntryV6) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0001 uint32 + zb0001, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 7 { + err = msgp.ArrayError{Wanted: 7, Got: zb0001} + return + } + err = z.Children.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + z.Size, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + z.Objects, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + z.Versions, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "ReplicationStats") + return + } + z.ReplicationStats = nil + } else { + if z.ReplicationStats == nil { + z.ReplicationStats = new(replicationAllStatsV1) + } + err = z.ReplicationStats.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "ReplicationStats") + return + } + } + z.Compacted, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Compacted") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageEntryV6) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 7 { + err = msgp.ArrayError{Wanted: 7, Got: zb0001} + return + } + bts, err = z.Children.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + z.Objects, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + z.Versions, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.ReplicationStats = nil + } else { + if z.ReplicationStats == nil { + z.ReplicationStats = new(replicationAllStatsV1) + } + bts, err = z.ReplicationStats.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicationStats") + return + } + } + z.Compacted, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Compacted") + return + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *dataUsageEntryV6) Msgsize() (s int) { + s = 1 + z.Children.Msgsize() + msgp.Int64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + if z.ReplicationStats == nil { + s += msgp.NilSize + } else { + s += z.ReplicationStats.Msgsize() + } + s += msgp.BoolSize + return +} + // DecodeMsg implements msgp.Decodable func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) { { @@ -2154,6 +2884,250 @@ func (z dataUsageHash) Msgsize() (s int) { // DecodeMsg implements msgp.Decodable func (z *replicationAllStats) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "t": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Targets") + return + } + if z.Targets == nil { + z.Targets = make(map[string]replicationStats, zb0002) + } else if len(z.Targets) > 0 { + for key := range z.Targets { + delete(z.Targets, key) + } + } + for zb0002 > 0 { + zb0002-- + var za0001 string + var za0002 replicationStats + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Targets") + return + } + err = za0002.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Targets", za0001) + return + } + z.Targets[za0001] = za0002 + } + case "r": + z.ReplicaSize, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ReplicaSize") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) { + // omitempty: check for empty values + zb0001Len := uint32(2) + var zb0001Mask uint8 /* 2 bits */ + if z.Targets == nil { + zb0001Len-- + zb0001Mask |= 0x1 + } + if z.ReplicaSize == 0 { + zb0001Len-- + zb0001Mask |= 0x2 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) + if err != nil { + return + } + if zb0001Len == 0 { + return + } + if (zb0001Mask & 0x1) == 0 { // if not empty + // write "t" + err = en.Append(0xa1, 0x74) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.Targets))) + if err != nil { + err = msgp.WrapError(err, "Targets") + return + } + for za0001, za0002 := range z.Targets { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Targets") + return + } + err = za0002.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Targets", za0001) + return + } + } + } + if (zb0001Mask & 0x2) == 0 { // if not empty + // write "r" + err = en.Append(0xa1, 0x72) + if err != nil { + return + } + err = en.WriteUint64(z.ReplicaSize) + if err != nil { + err = msgp.WrapError(err, "ReplicaSize") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // omitempty: check for empty values + zb0001Len := uint32(2) + var zb0001Mask uint8 /* 2 bits */ + if z.Targets == nil { + zb0001Len-- + zb0001Mask |= 0x1 + } + if z.ReplicaSize == 0 { + zb0001Len-- + zb0001Mask |= 0x2 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len == 0 { + return + } + if (zb0001Mask & 0x1) == 0 { // if not empty + // string "t" + o = append(o, 0xa1, 0x74) + o = msgp.AppendMapHeader(o, uint32(len(z.Targets))) + for za0001, za0002 := range z.Targets { + o = msgp.AppendString(o, za0001) + o, err = za0002.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Targets", za0001) + return + } + } + } + if (zb0001Mask & 0x2) == 0 { // if not empty + // string "r" + o = append(o, 0xa1, 0x72) + o = msgp.AppendUint64(o, z.ReplicaSize) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *replicationAllStats) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "t": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Targets") + return + } + if z.Targets == nil { + z.Targets = make(map[string]replicationStats, zb0002) + } else if len(z.Targets) > 0 { + for key := range z.Targets { + delete(z.Targets, key) + } + } + for zb0002 > 0 { + var za0001 string + var za0002 replicationStats + zb0002-- + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Targets") + return + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Targets", za0001) + return + } + z.Targets[za0001] = za0002 + } + case "r": + z.ReplicaSize, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicaSize") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *replicationAllStats) Msgsize() (s int) { + s = 1 + 2 + msgp.MapHeaderSize + if z.Targets != nil { + for za0001, za0002 := range z.Targets { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize() + } + } + s += 2 + msgp.Uint64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *replicationAllStatsV1) DecodeMsg(dc *msgp.Reader) (err error) { var zb0001 uint32 zb0001, err = dc.ReadArrayHeader() if err != nil { @@ -2202,7 +3176,7 @@ func (z *replicationAllStats) DecodeMsg(dc *msgp.Reader) (err error) { } // EncodeMsg implements msgp.Encodable -func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) { +func (z *replicationAllStatsV1) EncodeMsg(en *msgp.Writer) (err error) { // array header, size 2 err = en.Append(0x92) if err != nil { @@ -2234,7 +3208,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) { } // MarshalMsg implements msgp.Marshaler -func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) { +func (z *replicationAllStatsV1) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // array header, size 2 o = append(o, 0x92) @@ -2252,7 +3226,7 @@ func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) { } // UnmarshalMsg implements msgp.Unmarshaler -func (z *replicationAllStats) UnmarshalMsg(bts []byte) (o []byte, err error) { +func (z *replicationAllStatsV1) UnmarshalMsg(bts []byte) (o []byte, err error) { var zb0001 uint32 zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { @@ -2302,7 +3276,7 @@ func (z *replicationAllStats) UnmarshalMsg(bts []byte) (o []byte, err error) { } // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *replicationAllStats) Msgsize() (s int) { +func (z *replicationAllStatsV1) Msgsize() (s int) { s = 1 + msgp.MapHeaderSize if z.Targets != nil { for za0001, za0002 := range z.Targets { @@ -2805,3 +3779,156 @@ func (z *sizeHistogram) Msgsize() (s int) { s = msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) return } + +// DecodeMsg implements msgp.Decodable +func (z *tierStats) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ts": + z.TotalSize, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "TotalSize") + return + } + case "nv": + z.NumVersions, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "NumVersions") + return + } + case "no": + z.NumObjects, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "NumObjects") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z tierStats) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "ts" + err = en.Append(0x83, 0xa2, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteUint64(z.TotalSize) + if err != nil { + err = msgp.WrapError(err, "TotalSize") + return + } + // write "nv" + err = en.Append(0xa2, 0x6e, 0x76) + if err != nil { + return + } + err = en.WriteInt(z.NumVersions) + if err != nil { + err = msgp.WrapError(err, "NumVersions") + return + } + // write "no" + err = en.Append(0xa2, 0x6e, 0x6f) + if err != nil { + return + } + err = en.WriteInt(z.NumObjects) + if err != nil { + err = msgp.WrapError(err, "NumObjects") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z tierStats) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "ts" + o = append(o, 0x83, 0xa2, 0x74, 0x73) + o = msgp.AppendUint64(o, z.TotalSize) + // string "nv" + o = append(o, 0xa2, 0x6e, 0x76) + o = msgp.AppendInt(o, z.NumVersions) + // string "no" + o = append(o, 0xa2, 0x6e, 0x6f) + o = msgp.AppendInt(o, z.NumObjects) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *tierStats) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ts": + z.TotalSize, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalSize") + return + } + case "nv": + z.NumVersions, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NumVersions") + return + } + case "no": + z.NumObjects, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NumObjects") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z tierStats) Msgsize() (s int) { + s = 1 + 3 + msgp.Uint64Size + 3 + msgp.IntSize + 3 + msgp.IntSize + return +} diff --git a/cmd/data-usage-cache_gen_test.go b/cmd/data-usage-cache_gen_test.go index 3cc55ac22..5e8f4a3be 100644 --- a/cmd/data-usage-cache_gen_test.go +++ b/cmd/data-usage-cache_gen_test.go @@ -9,6 +9,119 @@ import ( "github.com/tinylib/msgp/msgp" ) +func TestMarshalUnmarshalallTierStats(t *testing.T) { + v := allTierStats{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgallTierStats(b *testing.B) { + v := allTierStats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgallTierStats(b *testing.B) { + v := allTierStats{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalallTierStats(b *testing.B) { + v := allTierStats{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeallTierStats(t *testing.T) { + v := allTierStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeallTierStats Msgsize() is inaccurate") + } + + vn := allTierStats{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeallTierStats(b *testing.B) { + v := allTierStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeallTierStats(b *testing.B) { + v := allTierStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + func TestMarshalUnmarshaldataUsageCache(t *testing.T) { v := dataUsageCache{} bts, err := v.MarshalMsg(nil) @@ -348,119 +461,6 @@ func BenchmarkDecodedataUsageEntry(b *testing.B) { } } -func TestMarshalUnmarshaldataUsageEntryV5(t *testing.T) { - v := dataUsageEntryV5{} - bts, err := v.MarshalMsg(nil) - if err != nil { - t.Fatal(err) - } - left, err := v.UnmarshalMsg(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) - } - - left, err = msgp.Skip(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after Skip(): %q", len(left), left) - } -} - -func BenchmarkMarshalMsgdataUsageEntryV5(b *testing.B) { - v := dataUsageEntryV5{} - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - v.MarshalMsg(nil) - } -} - -func BenchmarkAppendMsgdataUsageEntryV5(b *testing.B) { - v := dataUsageEntryV5{} - bts := make([]byte, 0, v.Msgsize()) - bts, _ = v.MarshalMsg(bts[0:0]) - b.SetBytes(int64(len(bts))) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - bts, _ = v.MarshalMsg(bts[0:0]) - } -} - -func BenchmarkUnmarshaldataUsageEntryV5(b *testing.B) { - v := dataUsageEntryV5{} - bts, _ := v.MarshalMsg(nil) - b.ReportAllocs() - b.SetBytes(int64(len(bts))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, err := v.UnmarshalMsg(bts) - if err != nil { - b.Fatal(err) - } - } -} - -func TestEncodeDecodedataUsageEntryV5(t *testing.T) { - v := dataUsageEntryV5{} - var buf bytes.Buffer - msgp.Encode(&buf, &v) - - m := v.Msgsize() - if buf.Len() > m { - t.Log("WARNING: TestEncodeDecodedataUsageEntryV5 Msgsize() is inaccurate") - } - - vn := dataUsageEntryV5{} - err := msgp.Decode(&buf, &vn) - if err != nil { - t.Error(err) - } - - buf.Reset() - msgp.Encode(&buf, &v) - err = msgp.NewReader(&buf).Skip() - if err != nil { - t.Error(err) - } -} - -func BenchmarkEncodedataUsageEntryV5(b *testing.B) { - v := dataUsageEntryV5{} - var buf bytes.Buffer - msgp.Encode(&buf, &v) - b.SetBytes(int64(buf.Len())) - en := msgp.NewWriter(msgp.Nowhere) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - v.EncodeMsg(en) - } - en.Flush() -} - -func BenchmarkDecodedataUsageEntryV5(b *testing.B) { - v := dataUsageEntryV5{} - var buf bytes.Buffer - msgp.Encode(&buf, &v) - b.SetBytes(int64(buf.Len())) - rd := msgp.NewEndlessReader(buf.Bytes(), b) - dc := msgp.NewReader(rd) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - err := v.DecodeMsg(dc) - if err != nil { - b.Fatal(err) - } - } -} - func TestMarshalUnmarshalreplicationAllStats(t *testing.T) { v := replicationAllStats{} bts, err := v.MarshalMsg(nil) @@ -574,6 +574,119 @@ func BenchmarkDecodereplicationAllStats(b *testing.B) { } } +func TestMarshalUnmarshalreplicationAllStatsV1(t *testing.T) { + v := replicationAllStatsV1{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgreplicationAllStatsV1(b *testing.B) { + v := replicationAllStatsV1{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgreplicationAllStatsV1(b *testing.B) { + v := replicationAllStatsV1{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalreplicationAllStatsV1(b *testing.B) { + v := replicationAllStatsV1{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodereplicationAllStatsV1(t *testing.T) { + v := replicationAllStatsV1{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodereplicationAllStatsV1 Msgsize() is inaccurate") + } + + vn := replicationAllStatsV1{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodereplicationAllStatsV1(b *testing.B) { + v := replicationAllStatsV1{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodereplicationAllStatsV1(b *testing.B) { + v := replicationAllStatsV1{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + func TestMarshalUnmarshalreplicationStats(t *testing.T) { v := replicationStats{} bts, err := v.MarshalMsg(nil) @@ -912,3 +1025,116 @@ func BenchmarkDecodesizeHistogram(b *testing.B) { } } } + +func TestMarshalUnmarshaltierStats(t *testing.T) { + v := tierStats{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgtierStats(b *testing.B) { + v := tierStats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgtierStats(b *testing.B) { + v := tierStats{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshaltierStats(b *testing.B) { + v := tierStats{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodetierStats(t *testing.T) { + v := tierStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodetierStats Msgsize() is inaccurate") + } + + vn := tierStats{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodetierStats(b *testing.B) { + v := tierStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodetierStats(b *testing.B) { + v := tierStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/data-usage-utils.go b/cmd/data-usage-utils.go index e093ae49f..de77a240d 100644 --- a/cmd/data-usage-utils.go +++ b/cmd/data-usage-utils.go @@ -18,7 +18,10 @@ package cmd import ( + "sort" "time" + + "github.com/minio/madmin-go" ) // BucketTargetUsageInfo - bucket target usage info provides @@ -81,7 +84,50 @@ type DataUsageInfo struct { // - total objects in a bucket // - object size histogram per bucket BucketsUsage map[string]BucketUsageInfo `json:"bucketsUsageInfo"` - // Deprecated kept here for backward compatibility reasons. BucketSizes map[string]uint64 `json:"bucketsSizes"` + + // TierStats contains per-tier stats of all configured remote tiers + TierStats *allTierStats `json:"tierStats,omitempty"` +} + +func (dui DataUsageInfo) tierStats() []madmin.TierInfo { + if globalTierConfigMgr.Empty() { + return nil + } + + ts := make(map[string]madmin.TierStats) + // Add configured remote tiers + for tier := range globalTierConfigMgr.Tiers { + ts[tier] = madmin.TierStats{} + } + // Add STANDARD (hot-tier) + ts[minioHotTier] = madmin.TierStats{} + + ts = dui.TierStats.adminStats(ts) + infos := make([]madmin.TierInfo, 0, len(ts)) + for tier, st := range ts { + var tierType string + if tier == minioHotTier { + tierType = "internal" + } else { + tierType = globalTierConfigMgr.Tiers[tier].Type.String() + } + infos = append(infos, madmin.TierInfo{ + Name: tier, + Type: tierType, + Stats: st, + }) + } + + sort.Slice(infos, func(i, j int) bool { + if infos[i].Type == "internal" { + return true + } + if infos[j].Type == "internal" { + return false + } + return infos[i].Name < infos[j].Name + }) + return infos } diff --git a/cmd/object-api-datatypes.go b/cmd/object-api-datatypes.go index 9132ff6c2..6bf4bfd45 100644 --- a/cmd/object-api-datatypes.go +++ b/cmd/object-api-datatypes.go @@ -225,6 +225,18 @@ func (o ObjectInfo) Clone() (cinfo ObjectInfo) { return cinfo } +func (o ObjectInfo) tierStats() tierStats { + ts := tierStats{ + TotalSize: uint64(o.Size), + NumVersions: 1, + } + // the current version of an object is accounted towards objects count + if o.IsLatest { + ts.NumObjects = 1 + } + return ts +} + // ReplicateObjectInfo represents object info to be replicated type ReplicateObjectInfo struct { ObjectInfo diff --git a/cmd/tier-handlers.go b/cmd/tier-handlers.go index bf06e38e8..a51471d63 100644 --- a/cmd/tier-handlers.go +++ b/cmd/tier-handlers.go @@ -25,6 +25,7 @@ import ( "github.com/gorilla/mux" jsoniter "github.com/json-iterator/go" "github.com/minio/madmin-go" + "github.com/minio/minio/internal/config/storageclass" "github.com/minio/minio/internal/logger" iampolicy "github.com/minio/pkg/iam/policy" ) @@ -60,6 +61,12 @@ var ( Message: "Invalid remote tier credentials", StatusCode: http.StatusBadRequest, } + // error returned when reserved internal names are used. + errTierReservedName = AdminError{ + Code: "XMinioAdminTierReserved", + Message: "Cannot use reserved tier name", + StatusCode: http.StatusBadRequest, + } ) func (api adminAPIHandlers) AddTierHandler(w http.ResponseWriter, r *http.Request) { @@ -92,6 +99,12 @@ func (api adminAPIHandlers) AddTierHandler(w http.ResponseWriter, r *http.Reques return } + // Disallow remote tiers with internal storage class names + switch cfg.Name { + case storageclass.STANDARD, storageclass.RRS: + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errTierReservedName), r.URL) + return + } // Refresh from the disk in case we had missed notifications about edits from peers. if err := globalTierConfigMgr.Reload(ctx, objAPI); err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) @@ -191,3 +204,33 @@ func (api adminAPIHandlers) EditTierHandler(w http.ResponseWriter, r *http.Reque writeSuccessNoContent(w) } + +func (api adminAPIHandlers) TierStatsHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "TierStats") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + if !globalIsErasure { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + objAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListTierAction) + if objAPI == nil || globalNotificationSys == nil || globalTierConfigMgr == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + dui, err := loadDataUsageFromBackend(ctx, objAPI) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + data, err := json.Marshal(dui.tierStats()) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + writeSuccessResponseJSON(w, data) +} diff --git a/cmd/tier.go b/cmd/tier.go index 3f847aeae..d4cb3a53e 100644 --- a/cmd/tier.go +++ b/cmd/tier.go @@ -46,6 +46,8 @@ const ( tierConfigFile = "tier-config.bin" tierConfigFormat = 1 tierConfigVersion = 1 + + minioHotTier = "STANDARD" ) // tierConfigPath refers to remote tier config object name @@ -85,7 +87,6 @@ func (config *TierConfigMgr) Add(ctx context.Context, tier madmin.TierConfig) er defer config.Unlock() // check if tier name is in all caps - tierName := tier.Name if tierName != strings.ToUpper(tierName) { return errTierNameNotUppercase diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index 5f8f3e8b7..a2ee4af0a 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -463,6 +463,10 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates return sizeSummary{}, errSkipFile } sizeS := sizeSummary{} + var noTiers bool + if noTiers = globalTierConfigMgr.Empty(); !noTiers { + sizeS.tiers = make(map[string]tierStats) + } atomic.AddUint64(&globalScannerStats.accTotalObjects, 1) for _, version := range fivs.Versions { atomic.AddUint64(&globalScannerStats.accTotalVersions, 1) @@ -472,6 +476,21 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates sizeS.versions++ } sizeS.totalSize += sz + + // Skip tier accounting if, + // 1. no tiers configured + // 2. object version is a delete-marker or a free-version + // tracking deleted transitioned objects + switch { + case noTiers, oi.DeleteMarker, oi.TransitionedObject.FreeVersion: + + continue + } + tier := minioHotTier + if oi.TransitionedObject.Status == lifecycle.TransitionComplete { + tier = oi.TransitionedObject.Tier + } + sizeS.tiers[tier] = sizeS.tiers[tier].add(oi.tierStats()) } return sizeS, nil }) diff --git a/go.mod b/go.mod index 157a4478f..cda7759f9 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 github.com/fatih/color v1.12.0 github.com/go-ldap/ldap/v3 v3.2.4 + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/loads v0.20.2 github.com/go-sql-driver/mysql v1.5.0 github.com/golang-jwt/jwt v3.2.1+incompatible @@ -32,9 +33,9 @@ require ( github.com/google/uuid v1.1.2 github.com/gorilla/mux v1.8.0 github.com/jcmturner/gokrb5/v8 v8.4.2 - github.com/json-iterator/go v1.1.11 + github.com/json-iterator/go v1.1.12 github.com/klauspost/compress v1.13.5 - github.com/klauspost/cpuid/v2 v2.0.6 + github.com/klauspost/cpuid/v2 v2.0.9 github.com/klauspost/pgzip v1.2.5 github.com/klauspost/readahead v1.3.1 github.com/klauspost/reedsolomon v1.9.13 @@ -72,9 +73,9 @@ require ( github.com/rs/cors v1.7.0 github.com/rs/dnscache v0.0.0-20210201191234-295bba877686 github.com/secure-io/sio-go v0.3.1 - github.com/shirou/gopsutil/v3 v3.21.7 + github.com/shirou/gopsutil/v3 v3.21.9 github.com/streadway/amqp v1.0.0 - github.com/tinylib/msgp v1.1.6-0.20210521143832-0becd170c402 + github.com/tinylib/msgp v1.1.6 github.com/valyala/bytebufferpool v1.0.0 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c github.com/yargevad/filepathx v1.0.0 @@ -83,10 +84,12 @@ require ( go.opencensus.io v0.22.5 // indirect go.uber.org/atomic v1.7.0 go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19 - golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e - golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c + golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 + golang.org/x/net v0.0.0-20211020060615-d418f374d309 // indirect + golang.org/x/sys v0.0.0-20211020174200-9d6173849985 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba golang.org/x/tools v0.1.1 // indirect google.golang.org/api v0.31.0 + gopkg.in/ini.v1 v1.63.2 // indirect gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index a8715471d..c5da4dcf4 100644 --- a/go.sum +++ b/go.sum @@ -381,8 +381,9 @@ github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= -github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -661,7 +662,6 @@ github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2c github.com/gookit/color v1.2.4/go.mod h1:AhIE+pS6D4Ql0SQWbBeXPHw7gY0/sjHoA4s/n1KB7xg= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gopherjs/gopherjs v0.0.0-20190328170749-bb2674552d8f h1:4Gslotqbs16iAg+1KR/XdabIfq8TlAWHdwS5QJFksLc= github.com/gopherjs/gopherjs v0.0.0-20190328170749-bb2674552d8f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/goreleaser/goreleaser v0.136.0/go.mod h1:wiKrPUeSNh6Wu8nUHxZydSOVQ/OZvOaO7DTtFqie904= github.com/goreleaser/nfpm v1.2.1/go.mod h1:TtWrABZozuLOttX2uDlYyECfQX7x5XYkVxhjYcR6G9w= @@ -862,12 +862,12 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= @@ -893,8 +893,9 @@ github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/klauspost/cpuid/v2 v2.0.3/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.6 h1:dQ5ueTiftKxp0gyjKSx5+8BtPWkyQbd95m8Gys/RarI= github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/readahead v1.3.1 h1:QqXNYvm+VvqYcbrRT4LojUciM0XrznFRIDrbHiJtu/0= @@ -1026,6 +1027,8 @@ github.com/minio/kes v0.14.0 h1:plCGm4LwR++T1P1sXsJbyFRX54CE1WRuo9PAPj6MC3Q= github.com/minio/kes v0.14.0/go.mod h1:OUensXz2BpgMfiogslKxv7Anyx/wj+6bFC6qA7BQcfA= github.com/minio/madmin-go v1.0.12/go.mod h1:BK+z4XRx7Y1v8SFWXsuLNqQqnq5BO/axJ8IDJfgyvfs= github.com/minio/madmin-go v1.1.6/go.mod h1:vw+c3/u+DeVKqReEavo///Cl2OO8nt5s4ee843hJeLs= +github.com/minio/madmin-go v1.1.9 h1:leOiyGhHL3wbF2FQoI/s/WROEPa3imR3cBP8wg9AVJk= +github.com/minio/madmin-go v1.1.9/go.mod h1:Iu0OnrMWNBYx1lqJTW+BFjBMx0Hi0wjw8VmqhiOs2Jo= github.com/minio/madmin-go v1.1.10 h1:pfMgXkzdwADnNfVdNMJbwok2fjb2sJ7Q76kDt89RGzE= github.com/minio/madmin-go v1.1.10/go.mod h1:Iu0OnrMWNBYx1lqJTW+BFjBMx0Hi0wjw8VmqhiOs2Jo= github.com/minio/mc v0.0.0-20210626002108-cebf3318546f h1:hyFvo5hSFw2K417YvDr/vAKlgCG69uTuhZW/5LNdL0U= @@ -1093,8 +1096,9 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.6.6 h1:Duep6KMIDpY4Yo11iFsvyqJDyfzLF9+sndUKT+v64GQ= github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= @@ -1310,8 +1314,8 @@ github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lz github.com/shirou/gopsutil/v3 v3.21.4/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw= github.com/shirou/gopsutil/v3 v3.21.5/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw= github.com/shirou/gopsutil/v3 v3.21.6/go.mod h1:JfVbDpIBLVzT8oKbvMg9P3wEIMDDpVn+LwHTKj0ST88= -github.com/shirou/gopsutil/v3 v3.21.7 h1:PnTqQamUjwEDSgn+nBGu0qSDV/CfvyiR/gwTH3i7HTU= -github.com/shirou/gopsutil/v3 v3.21.7/go.mod h1:RGl11Y7XMTQPmHh8F0ayC6haKNBgH4PXMJuTAcMOlz4= +github.com/shirou/gopsutil/v3 v3.21.9 h1:Vn4MUz2uXhqLSiCbGFRc0DILbMVLAY92DSkT8bsYrHg= +github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -1330,10 +1334,8 @@ github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= -github.com/smartystreets/assertions v1.1.1 h1:T/YLemO5Yp7KPzS+lVtu+WsHn8yoSwTfItdAd1r3cck= github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -1399,8 +1401,8 @@ github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiff github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/tinylib/msgp v1.1.3/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= -github.com/tinylib/msgp v1.1.6-0.20210521143832-0becd170c402 h1:x5VlSgDgIGXNegkO4gigpYmb/RFkKGgy12Kkrbif7XE= -github.com/tinylib/msgp v1.1.6-0.20210521143832-0becd170c402/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw= +github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw= +github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= @@ -1408,12 +1410,12 @@ github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKw github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= github.com/tklauser/go-sysconf v0.3.6/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= -github.com/tklauser/go-sysconf v0.3.7 h1:HT7h4+536gjqeq1ZIJPgOl1rg1XFatQGVZWp7Py53eg= -github.com/tklauser/go-sysconf v0.3.7/go.mod h1:JZIdXh4RmBvZDBZ41ld2bGxRV3n4daiiqA3skYhAoQ4= +github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= +github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8= github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= -github.com/tklauser/numcpus v0.2.3 h1:nQ0QYpiritP6ViFhrKYsiv6VVxOpum2Gks5GhnJbS/8= -github.com/tklauser/numcpus v0.2.3/go.mod h1:vpEPS/JC+oZGGQ/My/vJnNsvMDQL6PwOqt8dsCw5j+E= +github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= +github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tommy-muehle/go-mnd v1.1.1/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig= @@ -1553,8 +1555,9 @@ golang.org/x/crypto v0.0.0-20201217014255-9d1352758620/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1649,8 +1652,11 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210421230115-4e50805a0758 h1:aEpZnXcAmXkd6AvLb2OPt+EN1Zu/8Ne3pCqPjja5PXY= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI= +golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1757,11 +1763,14 @@ golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c h1:Lyn7+CqXIiC+LOR9aHD6jDK+hPcmAuCfuXztd1v4w1Q= -golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211015200801-69063c4bb744 h1:KzbpndAYEM+4oHRp9JmB2ewj0NHHxO3Z0g7Gus2O1kk= +golang.org/x/sys v0.0.0-20211015200801-69063c4bb744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211020174200-9d6173849985 h1:LOlKVhfDyahgmqa97awczplwkjzNaELFg3zRIJ13RYo= +golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2024,8 +2033,9 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c= +gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM=