benchmarks: add parallel benchmarks for PutObject/GetObject. (#2159)
Fixes #2092
This commit is contained in:
Bala FA
Harshavardhana
parent
bc8720406d
commit
2a95eabb8a
|
|
@ -20,6 +20,7 @@ import (
|
|||
"bytes"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"math/rand"
|
||||
"strconv"
|
||||
|
|
@ -261,3 +262,123 @@ func returnGetObjectBenchmark(objSize int) func(*testing.B, ObjectLayer) {
|
|||
runGetObjectBenchmark(b, obj, objSize)
|
||||
}
|
||||
}
|
||||
|
||||
// Parallel benchmark utility functions for ObjectLayer.PutObject().
|
||||
// Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark.
|
||||
func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
var err error
|
||||
// obtains random bucket name.
|
||||
bucket := getRandomBucketName()
|
||||
// create bucket.
|
||||
err = obj.MakeBucket(bucket)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
// PutObject returns md5Sum of the object inserted.
|
||||
// md5Sum variable is assigned with that value.
|
||||
var md5Sum string
|
||||
// get text data generated for number of bytes equal to object size.
|
||||
textData := generateBytesData(objSize)
|
||||
// generate md5sum for the generated data.
|
||||
// md5sum of the data to written is required as input for PutObject.
|
||||
hasher := md5.New()
|
||||
hasher.Write([]byte(textData))
|
||||
metadata := make(map[string]string)
|
||||
metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
|
||||
// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
|
||||
b.ReportAllocs()
|
||||
// the actual benchmark for PutObject starts here. Reset the benchmark timer.
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
i := 0
|
||||
for pb.Next() {
|
||||
// insert the object.
|
||||
md5Sum, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if md5Sum != metadata["md5Sum"] {
|
||||
b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", md5Sum, metadata["md5Sum"])
|
||||
}
|
||||
i++
|
||||
}
|
||||
})
|
||||
|
||||
// Benchmark ends here. Stop timer.
|
||||
b.StopTimer()
|
||||
}
|
||||
|
||||
// closure for returning the put object benchmark executor for given object size in bytes.
|
||||
func returnPutObjectBenchmarkParallel(objSize int) func(*testing.B, ObjectLayer) {
|
||||
// FIXME: Avoid closure.
|
||||
return func(b *testing.B, obj ObjectLayer) {
|
||||
runPutObjectBenchmarkParallel(b, obj, objSize)
|
||||
}
|
||||
}
|
||||
|
||||
// Parallel benchmark utility functions for ObjectLayer.GetObject().
|
||||
// Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark.
|
||||
func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
var err error
|
||||
// obtains random bucket name.
|
||||
bucket := getRandomBucketName()
|
||||
// create bucket.
|
||||
err = obj.MakeBucket(bucket)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
// PutObject returns md5Sum of the object inserted.
|
||||
// md5Sum variable is assigned with that value.
|
||||
var md5Sum string
|
||||
for i := 0; i < 10; i++ {
|
||||
// get text data generated for number of bytes equal to object size.
|
||||
textData := generateBytesData(objSize)
|
||||
// generate md5sum for the generated data.
|
||||
// md5sum of the data to written is required as input for PutObject.
|
||||
// PutObject is the functions which writes the data onto the FS/XL backend.
|
||||
hasher := md5.New()
|
||||
hasher.Write([]byte(textData))
|
||||
metadata := make(map[string]string)
|
||||
metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
|
||||
// insert the object.
|
||||
md5Sum, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if md5Sum != metadata["md5Sum"] {
|
||||
b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"])
|
||||
}
|
||||
}
|
||||
|
||||
// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
|
||||
b.ReportAllocs()
|
||||
// the actual benchmark for GetObject starts here. Reset the benchmark timer.
|
||||
b.ResetTimer()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
i := 0
|
||||
for pb.Next() {
|
||||
err = obj.GetObject(bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard)
|
||||
if err != nil {
|
||||
b.Error(err)
|
||||
}
|
||||
i++
|
||||
if i == 10 {
|
||||
i = 0
|
||||
}
|
||||
}
|
||||
})
|
||||
// Benchmark ends here. Stop timer.
|
||||
b.StopTimer()
|
||||
|
||||
}
|
||||
|
||||
// closure for returning the get object benchmark executor for given object size in bytes.
|
||||
// FIXME: Avoid closure.
|
||||
func returnGetObjectBenchmarkParallel(objSize int) func(*testing.B, ObjectLayer) {
|
||||
return func(b *testing.B, obj ObjectLayer) {
|
||||
runGetObjectBenchmarkParallel(b, obj, objSize)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -298,3 +298,127 @@ func BenchmarkGetObject1GbFS(b *testing.B) {
|
|||
func BenchmarkGetObject1GbXL(b *testing.B) {
|
||||
benchmarkGetObject(b, "XL", returnGetObjectBenchmark(1024*1024*1024))
|
||||
}
|
||||
|
||||
// The intent is to benchamrk GetObject for various sizes ranging from few bytes to 100MB.
|
||||
// Also each of these BenchmarkParallels are run both XL and FS backends.
|
||||
|
||||
// BenchmarkParallelGetObjectVerySmallFS - BenchmarkParallel FS.GetObject() for object size of 10 bytes.
|
||||
func BenchmarkParallelGetObjectVerySmallFS(b *testing.B) {
|
||||
benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(10))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObjectVerySmallXL - BenchmarkParallel XL.GetObject() for object size of 10 bytes.
|
||||
func BenchmarkParallelGetObjectVerySmallXL(b *testing.B) {
|
||||
benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(10))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject10KbFS - BenchmarkParallel FS.GetObject() for object size of 10KB.
|
||||
func BenchmarkParallelGetObject10KbFS(b *testing.B) {
|
||||
benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(10*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject10KbXL - BenchmarkParallel XL.GetObject() for object size of 10KB.
|
||||
func BenchmarkParallelGetObject10KbXL(b *testing.B) {
|
||||
benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(10*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject100KbFS - BenchmarkParallel FS.GetObject() for object size of 100KB.
|
||||
func BenchmarkParallelGetObject100KbFS(b *testing.B) {
|
||||
benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(100*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject100KbXL - BenchmarkParallel XL.GetObject() for object size of 100KB.
|
||||
func BenchmarkParallelGetObject100KbXL(b *testing.B) {
|
||||
benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(100*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject1MbFS - BenchmarkParallel FS.GetObject() for object size of 1MB.
|
||||
func BenchmarkParallelGetObject1MbFS(b *testing.B) {
|
||||
benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject1MbXL - BenchmarkParallel XL.GetObject() for object size of 1MB.
|
||||
func BenchmarkParallelGetObject1MbXL(b *testing.B) {
|
||||
benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject5MbFS - BenchmarkParallel FS.GetObject() for object size of 5MB.
|
||||
func BenchmarkParallelGetObject5MbFS(b *testing.B) {
|
||||
benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(5*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject5MbXL - BenchmarkParallel XL.GetObject() for object size of 5MB.
|
||||
func BenchmarkParallelGetObject5MbXL(b *testing.B) {
|
||||
benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(5*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject10MbFS - BenchmarkParallel FS.GetObject() for object size of 10MB.
|
||||
func BenchmarkParallelGetObject10MbFS(b *testing.B) {
|
||||
benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(10*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject10MbXL - BenchmarkParallel XL.GetObject() for object size of 10MB.
|
||||
func BenchmarkParallelGetObject10MbXL(b *testing.B) {
|
||||
benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(10*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject25MbFS - BenchmarkParallel FS.GetObject() for object size of 25MB.
|
||||
func BenchmarkParallelGetObject25MbFS(b *testing.B) {
|
||||
benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(25*1024*1024))
|
||||
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject25MbXL - BenchmarkParallel XL.GetObject() for object size of 25MB.
|
||||
func BenchmarkParallelGetObject25MbXL(b *testing.B) {
|
||||
benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(25*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject50MbFS - BenchmarkParallel FS.GetObject() for object size of 50MB.
|
||||
func BenchmarkParallelGetObject50MbFS(b *testing.B) {
|
||||
benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(50*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject50MbXL - BenchmarkParallel XL.GetObject() for object size of 50MB.
|
||||
func BenchmarkParallelGetObject50MbXL(b *testing.B) {
|
||||
benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(50*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject100MbFS - BenchmarkParallel FS.GetObject() for object size of 100MB.
|
||||
func BenchmarkParallelGetObject100MbFS(b *testing.B) {
|
||||
benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(100*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject100MbXL - BenchmarkParallel XL.GetObject() for object size of 100MB.
|
||||
func BenchmarkParallelGetObject100MbXL(b *testing.B) {
|
||||
benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(100*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject200MbFS - BenchmarkParallel FS.GetObject() for object size of 200MB.
|
||||
func BenchmarkParallelGetObject200MbFS(b *testing.B) {
|
||||
benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(200*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject200MbXL - BenchmarkParallel XL.GetObject() for object size of 200MB.
|
||||
func BenchmarkParallelGetObject200MbXL(b *testing.B) {
|
||||
benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(200*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject500MbFS - BenchmarkParallel FS.GetObject() for object size of 500MB.
|
||||
func BenchmarkParallelGetObject500MbFS(b *testing.B) {
|
||||
benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(500*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject500MbXL - BenchmarkParallel XL.GetObject() for object size of 500MB.
|
||||
func BenchmarkParallelGetObject500MbXL(b *testing.B) {
|
||||
benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(500*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObject1GbFS - BenchmarkParallel FS.GetObject() for object size of 1GB.
|
||||
func BenchmarkParallelGetObject1GbFS(b *testing.B) {
|
||||
benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(1024*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelGetObjectGbXL - BenchmarkParallel XL.GetObject() for object size of 1GB.
|
||||
func BenchmarkParallelGetObject1GbXL(b *testing.B) {
|
||||
benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(1024*1024*1024))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -469,3 +469,123 @@ func BenchmarkPutObject1GbFS(b *testing.B) {
|
|||
func BenchmarkPutObject1GbXL(b *testing.B) {
|
||||
benchmarkPutObject(b, "XL", returnPutObjectBenchmark(1024*1024*1024))
|
||||
}
|
||||
|
||||
func BenchmarkParallelPutObjectVerySmallFS(b *testing.B) {
|
||||
benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(10))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObjectVerySmallXL - BenchmarkParallel XL.PutObject() for object size of 10 bytes.
|
||||
func BenchmarkParallelPutObjectVerySmallXL(b *testing.B) {
|
||||
benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(10))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject10KbFS - BenchmarkParallel FS.PutObject() for object size of 10KB.
|
||||
func BenchmarkParallelPutObject10KbFS(b *testing.B) {
|
||||
benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(10*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject10KbXL - BenchmarkParallel XL.PutObject() for object size of 10KB.
|
||||
func BenchmarkParallelPutObject10KbXL(b *testing.B) {
|
||||
benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(10*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject100KbFS - BenchmarkParallel FS.PutObject() for object size of 100KB.
|
||||
func BenchmarkParallelPutObject100KbFS(b *testing.B) {
|
||||
benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(100*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject100KbXL - BenchmarkParallel XL.PutObject() for object size of 100KB.
|
||||
func BenchmarkParallelPutObject100KbXL(b *testing.B) {
|
||||
benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(100*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject1MbFS - BenchmarkParallel FS.PutObject() for object size of 1MB.
|
||||
func BenchmarkParallelPutObject1MbFS(b *testing.B) {
|
||||
benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject1MbXL - BenchmarkParallel XL.PutObject() for object size of 1MB.
|
||||
func BenchmarkParallelPutObject1MbXL(b *testing.B) {
|
||||
benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject5MbFS - BenchmarkParallel FS.PutObject() for object size of 5MB.
|
||||
func BenchmarkParallelPutObject5MbFS(b *testing.B) {
|
||||
benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(5*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject5MbXL - BenchmarkParallel XL.PutObject() for object size of 5MB.
|
||||
func BenchmarkParallelPutObject5MbXL(b *testing.B) {
|
||||
benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(5*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject10MbFS - BenchmarkParallel FS.PutObject() for object size of 10MB.
|
||||
func BenchmarkParallelPutObject10MbFS(b *testing.B) {
|
||||
benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(10*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject10MbXL - BenchmarkParallel XL.PutObject() for object size of 10MB.
|
||||
func BenchmarkParallelPutObject10MbXL(b *testing.B) {
|
||||
benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(10*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject25MbFS - BenchmarkParallel FS.PutObject() for object size of 25MB.
|
||||
func BenchmarkParallelPutObject25MbFS(b *testing.B) {
|
||||
benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(25*1024*1024))
|
||||
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject25MbXL - BenchmarkParallel XL.PutObject() for object size of 25MB.
|
||||
func BenchmarkParallelPutObject25MbXL(b *testing.B) {
|
||||
benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(25*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject50MbFS - BenchmarkParallel FS.PutObject() for object size of 50MB.
|
||||
func BenchmarkParallelPutObject50MbFS(b *testing.B) {
|
||||
benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(50*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject50MbXL - BenchmarkParallel XL.PutObject() for object size of 50MB.
|
||||
func BenchmarkParallelPutObject50MbXL(b *testing.B) {
|
||||
benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(50*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject100MbFS - BenchmarkParallel FS.PutObject() for object size of 100MB.
|
||||
func BenchmarkParallelPutObject100MbFS(b *testing.B) {
|
||||
benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(100*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject100MbXL - BenchmarkParallel XL.PutObject() for object size of 100MB.
|
||||
func BenchmarkParallelPutObject100MbXL(b *testing.B) {
|
||||
benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(100*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject200MbFS - BenchmarkParallel FS.PutObject() for object size of 200MB.
|
||||
func BenchmarkParallelPutObject200MbFS(b *testing.B) {
|
||||
benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(200*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject200MbXL - BenchmarkParallel XL.PutObject() for object size of 200MB.
|
||||
func BenchmarkParallelPutObject200MbXL(b *testing.B) {
|
||||
benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(200*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject500MbFS - BenchmarkParallel FS.PutObject() for object size of 500MB.
|
||||
func BenchmarkParallelPutObject500MbFS(b *testing.B) {
|
||||
benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(500*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject500MbXL - BenchmarkParallel XL.PutObject() for object size of 500MB.
|
||||
func BenchmarkParallelPutObject500MbXL(b *testing.B) {
|
||||
benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(500*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObject1GbFS - BenchmarkParallel FS.PutObject() for object size of 1GB.
|
||||
func BenchmarkParallelPutObject1GbFS(b *testing.B) {
|
||||
benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(1024*1024*1024))
|
||||
}
|
||||
|
||||
// BenchmarkParallelPutObjectGbXL - BenchmarkParallel XL.PutObject() for object size of 1GB.
|
||||
func BenchmarkParallelPutObject1GbXL(b *testing.B) {
|
||||
benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(1024*1024*1024))
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue