From 58934e5881d5da9b4bead4c05d3460ac23dd2ffb Mon Sep 17 00:00:00 2001 From: Krishna Srinivas <634494+krishnasrinivas@users.noreply.github.com> Date: Tue, 2 Nov 2021 15:27:03 -0700 Subject: [PATCH] Support live updates for clients during speedtest (#13566) --- cmd/admin-handlers.go | 50 ++++-------- cmd/utils.go | 180 ++++++++++++++++++++++-------------------- go.mod | 2 +- go.sum | 12 +++ 4 files changed, 126 insertions(+), 118 deletions(-) diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index ae4453c4b..8dd47664b 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -951,47 +951,31 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques duration = time.Second * 10 } - throughputSize := size - keepAliveTicker := time.NewTicker(500 * time.Millisecond) defer keepAliveTicker.Stop() - endBlankRepliesCh := make(chan error) - - go func() { - for { - select { - case <-ctx.Done(): - endBlankRepliesCh <- nil + ch := speedTest(ctx, size, concurrent, duration, autotune) + for { + select { + case <-ctx.Done(): + return + case <-keepAliveTicker.C: + // Write a blank entry to prevent client from disconnecting + if err := json.NewEncoder(w).Encode(madmin.SpeedTestResult{}); err != nil { return - case <-keepAliveTicker.C: - // Write a blank entry to prevent client from disconnecting - if err := json.NewEncoder(w).Encode(madmin.SpeedTestResult{}); err != nil { - endBlankRepliesCh <- err - return - } - w.(http.Flusher).Flush() - case endBlankRepliesCh <- nil: + } + w.(http.Flusher).Flush() + case result, ok := <-ch: + if !ok { + defer objectAPI.DeleteBucket(context.Background(), pathJoin(minioMetaSpeedTestBucket, minioMetaSpeedTestBucketPrefix), DeleteBucketOptions{Force: true, NoRecreate: true}) + return + } + if err := json.NewEncoder(w).Encode(result); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return } } - }() - - result, err := speedTest(ctx, throughputSize, concurrent, duration, autotune) - if <-endBlankRepliesCh != nil { - return } - if err != nil { - writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) - return - } - - if err := json.NewEncoder(w).Encode(result); err != nil { - writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) - return - } - - objectAPI.DeleteBucket(ctx, pathJoin(minioMetaSpeedTestBucket, minioMetaSpeedTestBucketPrefix), DeleteBucketOptions{Force: true, NoRecreate: true}) } // Admin API errors diff --git a/cmd/utils.go b/cmd/utils.go index f2baae254..f3eb2f3be 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -973,99 +973,111 @@ func auditLogInternal(ctx context.Context, bucket, object string, opts AuditLogO } // Get the max throughput and iops numbers. -func speedTest(ctx context.Context, throughputSize, concurrencyStart int, duration time.Duration, autotune bool) (madmin.SpeedTestResult, error) { - var result madmin.SpeedTestResult +func speedTest(ctx context.Context, throughputSize, concurrencyStart int, duration time.Duration, autotune bool) chan madmin.SpeedTestResult { + ch := make(chan madmin.SpeedTestResult, 1) + go func() { + defer close(ch) - objAPI := newObjectLayerFn() - if objAPI == nil { - return result, errServerNotInitialized - } - - concurrency := concurrencyStart - - throughputHighestGet := uint64(0) - throughputHighestPut := uint64(0) - var throughputHighestGetResults []SpeedtestResult - - for { - select { - case <-ctx.Done(): - // If the client got disconnected stop the speedtest. - return result, errUnexpected - default: + objAPI := newObjectLayerFn() + if objAPI == nil { + return } - results := globalNotificationSys.Speedtest(ctx, throughputSize, concurrency, duration) - sort.Slice(results, func(i, j int) bool { - return results[i].Endpoint < results[j].Endpoint - }) - totalPut := uint64(0) - totalGet := uint64(0) - for _, result := range results { - totalPut += result.Uploads - totalGet += result.Downloads + concurrency := concurrencyStart + + throughputHighestGet := uint64(0) + throughputHighestPut := uint64(0) + var throughputHighestGetResults []SpeedtestResult + + sendResult := func() { + var result madmin.SpeedTestResult + + durationSecs := duration.Seconds() + + result.GETStats.ThroughputPerSec = throughputHighestGet / uint64(durationSecs) + result.GETStats.ObjectsPerSec = throughputHighestGet / uint64(throughputSize) / uint64(durationSecs) + result.PUTStats.ThroughputPerSec = throughputHighestPut / uint64(durationSecs) + result.PUTStats.ObjectsPerSec = throughputHighestPut / uint64(throughputSize) / uint64(durationSecs) + for i := 0; i < len(throughputHighestGetResults); i++ { + errStr := "" + if throughputHighestGetResults[i].Error != "" { + errStr = throughputHighestGetResults[i].Error + } + result.PUTStats.Servers = append(result.PUTStats.Servers, madmin.SpeedTestStatServer{ + Endpoint: throughputHighestGetResults[i].Endpoint, + ThroughputPerSec: throughputHighestGetResults[i].Uploads / uint64(durationSecs), + ObjectsPerSec: throughputHighestGetResults[i].Uploads / uint64(throughputSize) / uint64(durationSecs), + Err: errStr, + }) + result.GETStats.Servers = append(result.GETStats.Servers, madmin.SpeedTestStatServer{ + Endpoint: throughputHighestGetResults[i].Endpoint, + ThroughputPerSec: throughputHighestGetResults[i].Downloads / uint64(durationSecs), + ObjectsPerSec: throughputHighestGetResults[i].Downloads / uint64(throughputSize) / uint64(durationSecs), + Err: errStr, + }) + } + + numDisks := 0 + if pools, ok := objAPI.(*erasureServerPools); ok { + for _, set := range pools.serverPools { + numDisks = set.setCount * set.setDriveCount + } + } + result.Disks = numDisks + result.Servers = len(globalNotificationSys.peerClients) + 1 + result.Version = Version + result.Size = throughputSize + result.Concurrent = concurrency + + ch <- result } - if totalGet < throughputHighestGet { - break - } + for { + select { + case <-ctx.Done(): + // If the client got disconnected stop the speedtest. + return + default: + } - doBreak := false - if float64(totalGet-throughputHighestGet)/float64(totalGet) < 0.025 { - doBreak = true - } + results := globalNotificationSys.Speedtest(ctx, throughputSize, concurrency, duration) + sort.Slice(results, func(i, j int) bool { + return results[i].Endpoint < results[j].Endpoint + }) + totalPut := uint64(0) + totalGet := uint64(0) + for _, result := range results { + totalPut += result.Uploads + totalGet += result.Downloads + } + + if totalGet < throughputHighestGet { + sendResult() + break + } + + doBreak := false + if float64(totalGet-throughputHighestGet)/float64(totalGet) < 0.025 { + doBreak = true + } - if totalGet > throughputHighestGet { throughputHighestGet = totalGet throughputHighestGetResults = results throughputHighestPut = totalPut + + if doBreak { + sendResult() + break + } + + if !autotune { + sendResult() + break + } + sendResult() + // Try with a higher concurrency to see if we get better throughput + concurrency += (concurrency + 1) / 2 } - - if doBreak { - break - } - - if !autotune { - break - } - // Try with a higher concurrency to see if we get better throughput - concurrency += (concurrency + 1) / 2 - } - - durationSecs := duration.Seconds() - - result.GETStats.ThroughputPerSec = throughputHighestGet / uint64(durationSecs) - result.GETStats.ObjectsPerSec = throughputHighestGet / uint64(throughputSize) / uint64(durationSecs) - result.PUTStats.ThroughputPerSec = throughputHighestPut / uint64(durationSecs) - result.PUTStats.ObjectsPerSec = throughputHighestPut / uint64(throughputSize) / uint64(durationSecs) - for i := 0; i < len(throughputHighestGetResults); i++ { - errStr := "" - if throughputHighestGetResults[i].Error != "" { - errStr = throughputHighestGetResults[i].Error - } - result.PUTStats.Servers = append(result.PUTStats.Servers, madmin.SpeedTestStatServer{ - Endpoint: throughputHighestGetResults[i].Endpoint, - ThroughputPerSec: throughputHighestGetResults[i].Uploads / uint64(durationSecs), - ObjectsPerSec: throughputHighestGetResults[i].Uploads / uint64(throughputSize) / uint64(durationSecs), - Err: errStr, - }) - result.GETStats.Servers = append(result.GETStats.Servers, madmin.SpeedTestStatServer{ - Endpoint: throughputHighestGetResults[i].Endpoint, - ThroughputPerSec: throughputHighestGetResults[i].Downloads / uint64(durationSecs), - ObjectsPerSec: throughputHighestGetResults[i].Downloads / uint64(throughputSize) / uint64(durationSecs), - Err: errStr, - }) - } - - numDisks := 0 - if pools, ok := objAPI.(*erasureServerPools); ok { - for _, set := range pools.serverPools { - numDisks = set.setCount * set.setDriveCount - } - } - result.Disks = numDisks - result.Servers = len(globalNotificationSys.peerClients) + 1 - result.Version = Version - - return result, nil + }() + return ch } diff --git a/go.mod b/go.mod index 6d21cc496..8e2e4ad45 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/minio/csvparser v1.0.0 github.com/minio/highwayhash v1.0.2 github.com/minio/kes v0.14.0 - github.com/minio/madmin-go v1.1.10 + github.com/minio/madmin-go v1.1.11-0.20211102182201-e51fd3d6b104 github.com/minio/minio-go/v7 v7.0.15 github.com/minio/parquet-go v1.0.0 github.com/minio/pkg v1.1.5 diff --git a/go.sum b/go.sum index 7dde04f90..2b07fd9a8 100644 --- a/go.sum +++ b/go.sum @@ -633,6 +633,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-containerregistry v0.1.2/go.mod h1:GPivBPgdAyd2SU+vf6EpsgOtWDuPqjW0hJZt4rNdTZ4= github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -967,6 +968,8 @@ github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -1053,6 +1056,8 @@ github.com/minio/kes v0.14.0/go.mod h1:OUensXz2BpgMfiogslKxv7Anyx/wj+6bFC6qA7BQc github.com/minio/madmin-go v1.0.12/go.mod h1:BK+z4XRx7Y1v8SFWXsuLNqQqnq5BO/axJ8IDJfgyvfs= github.com/minio/madmin-go v1.1.10 h1:pfMgXkzdwADnNfVdNMJbwok2fjb2sJ7Q76kDt89RGzE= github.com/minio/madmin-go v1.1.10/go.mod h1:Iu0OnrMWNBYx1lqJTW+BFjBMx0Hi0wjw8VmqhiOs2Jo= +github.com/minio/madmin-go v1.1.11-0.20211102182201-e51fd3d6b104 h1:/N0JW/1+vbxdDgCI8+tac7GXKkis6rh8kVsLNejN6Ug= +github.com/minio/madmin-go v1.1.11-0.20211102182201-e51fd3d6b104/go.mod h1:Iu0OnrMWNBYx1lqJTW+BFjBMx0Hi0wjw8VmqhiOs2Jo= github.com/minio/mc v0.0.0-20211027024940-7866f97ef502 h1:7ip9qTspUniv+WDENgOcfUr95IccxG5aDkBM4Z96kQg= github.com/minio/mc v0.0.0-20211027024940-7866f97ef502/go.mod h1:vxztwXLB9Gyl/h3Yh08Mpz1CB/0FO5Es0iQRpzxvS5I= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= @@ -1343,6 +1348,8 @@ github.com/shirou/gopsutil/v3 v3.21.6/go.mod h1:JfVbDpIBLVzT8oKbvMg9P3wEIMDDpVn+ github.com/shirou/gopsutil/v3 v3.21.8/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ= github.com/shirou/gopsutil/v3 v3.21.9 h1:Vn4MUz2uXhqLSiCbGFRc0DILbMVLAY92DSkT8bsYrHg= github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ= +github.com/shirou/gopsutil/v3 v3.21.10 h1:flTg1DrnV/UVrBqjLgVgDJzx6lf+91rC64/dBHmO2IA= +github.com/shirou/gopsutil/v3 v3.21.10/go.mod h1:t75NhzCZ/dYyPQjyQmrAYP6c8+LCdFANeBMdLPCNnew= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -1699,6 +1706,8 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI= golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211101193420-4a448f8816b3 h1:VrJZAjbekhoRn7n5FBujY31gboH+iB3pdLxn3gE9FjU= +golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1813,8 +1822,11 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211020174200-9d6173849985 h1:LOlKVhfDyahgmqa97awczplwkjzNaELFg3zRIJ13RYo= golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211102061401-a2f17f7b995c h1:QOfDMdrf/UwlVR0UBq2Mpr58UzNtvgJRXA4BgPfFACs= +golang.org/x/sys v0.0.0-20211102061401-a2f17f7b995c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=