Support live updates for clients during speedtest (#13566)
This commit is contained in:
Krishna Srinivas
GitHub
parent
ad3f98b8e7
commit
58934e5881
|
|
@ -951,47 +951,31 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
|
|||
duration = time.Second * 10
|
||||
}
|
||||
|
||||
throughputSize := size
|
||||
|
||||
keepAliveTicker := time.NewTicker(500 * time.Millisecond)
|
||||
defer keepAliveTicker.Stop()
|
||||
|
||||
endBlankRepliesCh := make(chan error)
|
||||
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
endBlankRepliesCh <- nil
|
||||
ch := speedTest(ctx, size, concurrent, duration, autotune)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-keepAliveTicker.C:
|
||||
// Write a blank entry to prevent client from disconnecting
|
||||
if err := json.NewEncoder(w).Encode(madmin.SpeedTestResult{}); err != nil {
|
||||
return
|
||||
case <-keepAliveTicker.C:
|
||||
// Write a blank entry to prevent client from disconnecting
|
||||
if err := json.NewEncoder(w).Encode(madmin.SpeedTestResult{}); err != nil {
|
||||
endBlankRepliesCh <- err
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
case endBlankRepliesCh <- nil:
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
case result, ok := <-ch:
|
||||
if !ok {
|
||||
defer objectAPI.DeleteBucket(context.Background(), pathJoin(minioMetaSpeedTestBucket, minioMetaSpeedTestBucketPrefix), DeleteBucketOptions{Force: true, NoRecreate: true})
|
||||
return
|
||||
}
|
||||
if err := json.NewEncoder(w).Encode(result); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
result, err := speedTest(ctx, throughputSize, concurrent, duration, autotune)
|
||||
if <-endBlankRepliesCh != nil {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err := json.NewEncoder(w).Encode(result); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
objectAPI.DeleteBucket(ctx, pathJoin(minioMetaSpeedTestBucket, minioMetaSpeedTestBucketPrefix), DeleteBucketOptions{Force: true, NoRecreate: true})
|
||||
}
|
||||
|
||||
// Admin API errors
|
||||
|
|
|
|||
180
cmd/utils.go
180
cmd/utils.go
|
|
@ -973,99 +973,111 @@ func auditLogInternal(ctx context.Context, bucket, object string, opts AuditLogO
|
|||
}
|
||||
|
||||
// Get the max throughput and iops numbers.
|
||||
func speedTest(ctx context.Context, throughputSize, concurrencyStart int, duration time.Duration, autotune bool) (madmin.SpeedTestResult, error) {
|
||||
var result madmin.SpeedTestResult
|
||||
func speedTest(ctx context.Context, throughputSize, concurrencyStart int, duration time.Duration, autotune bool) chan madmin.SpeedTestResult {
|
||||
ch := make(chan madmin.SpeedTestResult, 1)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return result, errServerNotInitialized
|
||||
}
|
||||
|
||||
concurrency := concurrencyStart
|
||||
|
||||
throughputHighestGet := uint64(0)
|
||||
throughputHighestPut := uint64(0)
|
||||
var throughputHighestGetResults []SpeedtestResult
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// If the client got disconnected stop the speedtest.
|
||||
return result, errUnexpected
|
||||
default:
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
results := globalNotificationSys.Speedtest(ctx, throughputSize, concurrency, duration)
|
||||
sort.Slice(results, func(i, j int) bool {
|
||||
return results[i].Endpoint < results[j].Endpoint
|
||||
})
|
||||
totalPut := uint64(0)
|
||||
totalGet := uint64(0)
|
||||
for _, result := range results {
|
||||
totalPut += result.Uploads
|
||||
totalGet += result.Downloads
|
||||
concurrency := concurrencyStart
|
||||
|
||||
throughputHighestGet := uint64(0)
|
||||
throughputHighestPut := uint64(0)
|
||||
var throughputHighestGetResults []SpeedtestResult
|
||||
|
||||
sendResult := func() {
|
||||
var result madmin.SpeedTestResult
|
||||
|
||||
durationSecs := duration.Seconds()
|
||||
|
||||
result.GETStats.ThroughputPerSec = throughputHighestGet / uint64(durationSecs)
|
||||
result.GETStats.ObjectsPerSec = throughputHighestGet / uint64(throughputSize) / uint64(durationSecs)
|
||||
result.PUTStats.ThroughputPerSec = throughputHighestPut / uint64(durationSecs)
|
||||
result.PUTStats.ObjectsPerSec = throughputHighestPut / uint64(throughputSize) / uint64(durationSecs)
|
||||
for i := 0; i < len(throughputHighestGetResults); i++ {
|
||||
errStr := ""
|
||||
if throughputHighestGetResults[i].Error != "" {
|
||||
errStr = throughputHighestGetResults[i].Error
|
||||
}
|
||||
result.PUTStats.Servers = append(result.PUTStats.Servers, madmin.SpeedTestStatServer{
|
||||
Endpoint: throughputHighestGetResults[i].Endpoint,
|
||||
ThroughputPerSec: throughputHighestGetResults[i].Uploads / uint64(durationSecs),
|
||||
ObjectsPerSec: throughputHighestGetResults[i].Uploads / uint64(throughputSize) / uint64(durationSecs),
|
||||
Err: errStr,
|
||||
})
|
||||
result.GETStats.Servers = append(result.GETStats.Servers, madmin.SpeedTestStatServer{
|
||||
Endpoint: throughputHighestGetResults[i].Endpoint,
|
||||
ThroughputPerSec: throughputHighestGetResults[i].Downloads / uint64(durationSecs),
|
||||
ObjectsPerSec: throughputHighestGetResults[i].Downloads / uint64(throughputSize) / uint64(durationSecs),
|
||||
Err: errStr,
|
||||
})
|
||||
}
|
||||
|
||||
numDisks := 0
|
||||
if pools, ok := objAPI.(*erasureServerPools); ok {
|
||||
for _, set := range pools.serverPools {
|
||||
numDisks = set.setCount * set.setDriveCount
|
||||
}
|
||||
}
|
||||
result.Disks = numDisks
|
||||
result.Servers = len(globalNotificationSys.peerClients) + 1
|
||||
result.Version = Version
|
||||
result.Size = throughputSize
|
||||
result.Concurrent = concurrency
|
||||
|
||||
ch <- result
|
||||
}
|
||||
|
||||
if totalGet < throughputHighestGet {
|
||||
break
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// If the client got disconnected stop the speedtest.
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
doBreak := false
|
||||
if float64(totalGet-throughputHighestGet)/float64(totalGet) < 0.025 {
|
||||
doBreak = true
|
||||
}
|
||||
results := globalNotificationSys.Speedtest(ctx, throughputSize, concurrency, duration)
|
||||
sort.Slice(results, func(i, j int) bool {
|
||||
return results[i].Endpoint < results[j].Endpoint
|
||||
})
|
||||
totalPut := uint64(0)
|
||||
totalGet := uint64(0)
|
||||
for _, result := range results {
|
||||
totalPut += result.Uploads
|
||||
totalGet += result.Downloads
|
||||
}
|
||||
|
||||
if totalGet < throughputHighestGet {
|
||||
sendResult()
|
||||
break
|
||||
}
|
||||
|
||||
doBreak := false
|
||||
if float64(totalGet-throughputHighestGet)/float64(totalGet) < 0.025 {
|
||||
doBreak = true
|
||||
}
|
||||
|
||||
if totalGet > throughputHighestGet {
|
||||
throughputHighestGet = totalGet
|
||||
throughputHighestGetResults = results
|
||||
throughputHighestPut = totalPut
|
||||
|
||||
if doBreak {
|
||||
sendResult()
|
||||
break
|
||||
}
|
||||
|
||||
if !autotune {
|
||||
sendResult()
|
||||
break
|
||||
}
|
||||
sendResult()
|
||||
// Try with a higher concurrency to see if we get better throughput
|
||||
concurrency += (concurrency + 1) / 2
|
||||
}
|
||||
|
||||
if doBreak {
|
||||
break
|
||||
}
|
||||
|
||||
if !autotune {
|
||||
break
|
||||
}
|
||||
// Try with a higher concurrency to see if we get better throughput
|
||||
concurrency += (concurrency + 1) / 2
|
||||
}
|
||||
|
||||
durationSecs := duration.Seconds()
|
||||
|
||||
result.GETStats.ThroughputPerSec = throughputHighestGet / uint64(durationSecs)
|
||||
result.GETStats.ObjectsPerSec = throughputHighestGet / uint64(throughputSize) / uint64(durationSecs)
|
||||
result.PUTStats.ThroughputPerSec = throughputHighestPut / uint64(durationSecs)
|
||||
result.PUTStats.ObjectsPerSec = throughputHighestPut / uint64(throughputSize) / uint64(durationSecs)
|
||||
for i := 0; i < len(throughputHighestGetResults); i++ {
|
||||
errStr := ""
|
||||
if throughputHighestGetResults[i].Error != "" {
|
||||
errStr = throughputHighestGetResults[i].Error
|
||||
}
|
||||
result.PUTStats.Servers = append(result.PUTStats.Servers, madmin.SpeedTestStatServer{
|
||||
Endpoint: throughputHighestGetResults[i].Endpoint,
|
||||
ThroughputPerSec: throughputHighestGetResults[i].Uploads / uint64(durationSecs),
|
||||
ObjectsPerSec: throughputHighestGetResults[i].Uploads / uint64(throughputSize) / uint64(durationSecs),
|
||||
Err: errStr,
|
||||
})
|
||||
result.GETStats.Servers = append(result.GETStats.Servers, madmin.SpeedTestStatServer{
|
||||
Endpoint: throughputHighestGetResults[i].Endpoint,
|
||||
ThroughputPerSec: throughputHighestGetResults[i].Downloads / uint64(durationSecs),
|
||||
ObjectsPerSec: throughputHighestGetResults[i].Downloads / uint64(throughputSize) / uint64(durationSecs),
|
||||
Err: errStr,
|
||||
})
|
||||
}
|
||||
|
||||
numDisks := 0
|
||||
if pools, ok := objAPI.(*erasureServerPools); ok {
|
||||
for _, set := range pools.serverPools {
|
||||
numDisks = set.setCount * set.setDriveCount
|
||||
}
|
||||
}
|
||||
result.Disks = numDisks
|
||||
result.Servers = len(globalNotificationSys.peerClients) + 1
|
||||
result.Version = Version
|
||||
|
||||
return result, nil
|
||||
}()
|
||||
return ch
|
||||
}
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -46,7 +46,7 @@ require (
|
|||
github.com/minio/csvparser v1.0.0
|
||||
github.com/minio/highwayhash v1.0.2
|
||||
github.com/minio/kes v0.14.0
|
||||
github.com/minio/madmin-go v1.1.10
|
||||
github.com/minio/madmin-go v1.1.11-0.20211102182201-e51fd3d6b104
|
||||
github.com/minio/minio-go/v7 v7.0.15
|
||||
github.com/minio/parquet-go v1.0.0
|
||||
github.com/minio/pkg v1.1.5
|
||||
|
|
|
|||
12
go.sum
12
go.sum
|
|
@ -633,6 +633,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
|||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-containerregistry v0.1.2/go.mod h1:GPivBPgdAyd2SU+vf6EpsgOtWDuPqjW0hJZt4rNdTZ4=
|
||||
github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
|
|
@ -967,6 +968,8 @@ github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
|||
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
|
||||
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
|
||||
github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
||||
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
|
||||
github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
|
|
@ -1053,6 +1056,8 @@ github.com/minio/kes v0.14.0/go.mod h1:OUensXz2BpgMfiogslKxv7Anyx/wj+6bFC6qA7BQc
|
|||
github.com/minio/madmin-go v1.0.12/go.mod h1:BK+z4XRx7Y1v8SFWXsuLNqQqnq5BO/axJ8IDJfgyvfs=
|
||||
github.com/minio/madmin-go v1.1.10 h1:pfMgXkzdwADnNfVdNMJbwok2fjb2sJ7Q76kDt89RGzE=
|
||||
github.com/minio/madmin-go v1.1.10/go.mod h1:Iu0OnrMWNBYx1lqJTW+BFjBMx0Hi0wjw8VmqhiOs2Jo=
|
||||
github.com/minio/madmin-go v1.1.11-0.20211102182201-e51fd3d6b104 h1:/N0JW/1+vbxdDgCI8+tac7GXKkis6rh8kVsLNejN6Ug=
|
||||
github.com/minio/madmin-go v1.1.11-0.20211102182201-e51fd3d6b104/go.mod h1:Iu0OnrMWNBYx1lqJTW+BFjBMx0Hi0wjw8VmqhiOs2Jo=
|
||||
github.com/minio/mc v0.0.0-20211027024940-7866f97ef502 h1:7ip9qTspUniv+WDENgOcfUr95IccxG5aDkBM4Z96kQg=
|
||||
github.com/minio/mc v0.0.0-20211027024940-7866f97ef502/go.mod h1:vxztwXLB9Gyl/h3Yh08Mpz1CB/0FO5Es0iQRpzxvS5I=
|
||||
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
|
||||
|
|
@ -1343,6 +1348,8 @@ github.com/shirou/gopsutil/v3 v3.21.6/go.mod h1:JfVbDpIBLVzT8oKbvMg9P3wEIMDDpVn+
|
|||
github.com/shirou/gopsutil/v3 v3.21.8/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ=
|
||||
github.com/shirou/gopsutil/v3 v3.21.9 h1:Vn4MUz2uXhqLSiCbGFRc0DILbMVLAY92DSkT8bsYrHg=
|
||||
github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ=
|
||||
github.com/shirou/gopsutil/v3 v3.21.10 h1:flTg1DrnV/UVrBqjLgVgDJzx6lf+91rC64/dBHmO2IA=
|
||||
github.com/shirou/gopsutil/v3 v3.21.10/go.mod h1:t75NhzCZ/dYyPQjyQmrAYP6c8+LCdFANeBMdLPCNnew=
|
||||
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
|
|
@ -1699,6 +1706,8 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx
|
|||
golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI=
|
||||
golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211101193420-4a448f8816b3 h1:VrJZAjbekhoRn7n5FBujY31gboH+iB3pdLxn3gE9FjU=
|
||||
golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
|
|
@ -1813,8 +1822,11 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211020174200-9d6173849985 h1:LOlKVhfDyahgmqa97awczplwkjzNaELFg3zRIJ13RYo=
|
||||
golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211102061401-a2f17f7b995c h1:QOfDMdrf/UwlVR0UBq2Mpr58UzNtvgJRXA4BgPfFACs=
|
||||
golang.org/x/sys v0.0.0-20211102061401-a2f17f7b995c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
|
|
|
|||
Loading…
Reference in New Issue