2016-07-09 03:26:04 +02:00
/ *
* Minio Cloud Storage , ( C ) 2016 Minio , Inc .
*
* Licensed under the Apache License , Version 2.0 ( the "License" ) ;
* you may not use this file except in compliance with the License .
* You may obtain a copy of the License at
*
* http : //www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing , software
* distributed under the License is distributed on an "AS IS" BASIS ,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
* See the License for the specific language governing permissions and
* limitations under the License .
* /
package main
import (
"bytes"
"fmt"
"io"
"strings"
"testing"
)
// Wrapper for calling GetObject tests for both XL multiple disks and single node setup.
func TestGetObject ( t * testing . T ) {
ExecObjectLayerTest ( t , testGetObject )
}
// ObjectLayer.GetObject is called with series of cases for valid and erroneous inputs and the result is validated.
func testGetObject ( obj ObjectLayer , instanceType string , t TestErrHandler ) {
// Setup for the tests.
bucketName := getRandomBucketName ( )
objectName := "test-object"
// create bucket.
err := obj . MakeBucket ( bucketName )
// Stop the test if creation of the bucket fails.
if err != nil {
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
// set of byte data for PutObject.
// object has to be inserted before running tests for GetObject.
// this is required even to assert the GetObject data,
// since dataInserted === dataFetched back is a primary criteria for any object storage this assertion is critical.
bytesData := [ ] struct {
byteData [ ] byte
} {
{ generateBytesData ( 6 * 1024 * 1024 ) } ,
}
// set of inputs for uploading the objects before tests for downloading is done.
putObjectInputs := [ ] struct {
bucketName string
objectName string
contentLength int64
textData [ ] byte
metaData map [ string ] string
} {
// case - 1.
{ bucketName , objectName , int64 ( len ( bytesData [ 0 ] . byteData ) ) , bytesData [ 0 ] . byteData , make ( map [ string ] string ) } ,
}
// iterate through the above set of inputs and upkoad the object.
for i , input := range putObjectInputs {
// uploading the object.
_ , err = obj . PutObject ( input . bucketName , input . objectName , input . contentLength , bytes . NewBuffer ( input . textData ) , input . metaData )
// if object upload fails stop the test.
if err != nil {
t . Fatalf ( "Put Object case %d: Error uploading object: <ERROR> %v" , i + 1 , err )
}
}
// set of empty buffers used to fill GetObject data.
buffers := [ ] * bytes . Buffer {
new ( bytes . Buffer ) ,
new ( bytes . Buffer ) ,
}
// test cases with set of inputs
testCases := [ ] struct {
bucketName string
objectName string
startOffset int64
length int64
// data obtained/fetched from GetObject.
getObjectData * bytes . Buffer
// writer which governs the write into the `getObjectData`.
writer io . Writer
// flag indicating whether the test for given ase should pass.
shouldPass bool
// expected Result.
expectedData [ ] byte
err error
} {
// Test case 1-4.
// Cases with invalid bucket names.
{ ".test" , "obj" , 0 , 0 , nil , nil , false , [ ] byte ( "" ) , fmt . Errorf ( "%s" , "Bucket name invalid: .test" ) } ,
{ "------" , "obj" , 0 , 0 , nil , nil , false , [ ] byte ( "" ) , fmt . Errorf ( "%s" , "Bucket name invalid: ------" ) } ,
{ "$this-is-not-valid-too" , "obj" , 0 , 0 , nil , nil , false ,
[ ] byte ( "" ) , fmt . Errorf ( "%s" , "Bucket name invalid: $this-is-not-valid-too" ) } ,
{ "a" , "obj" , 0 , 0 , nil , nil , false , [ ] byte ( "" ) , fmt . Errorf ( "%s" , "Bucket name invalid: a" ) } ,
// Test case - 5.
// Case with invalid object names.
{ bucketName , "" , 0 , 0 , nil , nil , false , [ ] byte ( "" ) , fmt . Errorf ( "%s" , "Object name invalid: " + bucketName + "#" ) } ,
// Test case - 6.
// Valid object and bucket names but non-existent bucket.
// {"abc", "def", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket not found: abc")},
// A custom writer is sent as an argument.
// Its designed to return a EOF error after reading `n` bytes, where `n` is the argument when initializing the EOF writer.
// This is to simulate the case of cache not filling up completly, since the EOFWriter doesn't allow the write to complete,
// the cache gets filled up with partial data. The following up test case will read the object completly, tests the
// purging of the cache during the incomplete write.
// Test case - 7.
{ bucketName , objectName , 0 , int64 ( len ( bytesData [ 0 ] . byteData ) ) , buffers [ 0 ] , NewEOFWriter ( buffers [ 0 ] , 100 ) , false , [ ] byte { } , io . EOF } ,
// Test case with start offset set to 0 and length set to size of the object.
// Fetching the entire object.
// Test case - 8.
{ bucketName , objectName , 0 , int64 ( len ( bytesData [ 0 ] . byteData ) ) , buffers [ 1 ] , buffers [ 1 ] , true , bytesData [ 0 ] . byteData , nil } ,
// Test case with content-range 1 to objectSize .
// Test case - 9.
{ bucketName , objectName , 1 , int64 ( len ( bytesData [ 0 ] . byteData ) - 1 ) , buffers [ 1 ] , buffers [ 1 ] , true , bytesData [ 0 ] . byteData [ 1 : ] , nil } ,
// Test case with content-range 100 to objectSize - 100.
// Test case - 10.
{ bucketName , objectName , 100 , int64 ( len ( bytesData [ 0 ] . byteData ) - 200 ) , buffers [ 1 ] , buffers [ 1 ] , true ,
bytesData [ 0 ] . byteData [ 100 : len ( bytesData [ 0 ] . byteData ) - 100 ] , nil } ,
// Test case with offset greater than the size of the object
// Test case - 11.
{ bucketName , objectName , int64 ( len ( bytesData [ 0 ] . byteData ) + 1 ) , int64 ( len ( bytesData [ 0 ] . byteData ) ) , buffers [ 0 ] ,
NewEOFWriter ( buffers [ 0 ] , 100 ) , false , [ ] byte { } ,
InvalidRange { int64 ( len ( bytesData [ 0 ] . byteData ) + 1 ) , int64 ( len ( bytesData [ 0 ] . byteData ) ) , int64 ( len ( bytesData [ 0 ] . byteData ) ) } } ,
// Test case with offset greater than the size of the object.
// Test case - 12.
{ bucketName , objectName , - 1 , int64 ( len ( bytesData [ 0 ] . byteData ) ) , buffers [ 0 ] , new ( bytes . Buffer ) , false , [ ] byte { } , errUnexpected } ,
// Test case length parameter is more than the object size.
// Test case - 13.
{ bucketName , objectName , 0 , int64 ( len ( bytesData [ 0 ] . byteData ) + 1 ) , buffers [ 1 ] , buffers [ 1 ] , false , bytesData [ 0 ] . byteData ,
InvalidRange { 0 , int64 ( len ( bytesData [ 0 ] . byteData ) + 1 ) , int64 ( len ( bytesData [ 0 ] . byteData ) ) } } ,
// Test case with `length` parameter set to a negative value.
// Test case - 14.
{ bucketName , objectName , 0 , int64 ( - 1 ) , buffers [ 1 ] , buffers [ 1 ] , false , bytesData [ 0 ] . byteData , errUnexpected } ,
// Test case with offset + length > objectSize parameter set to a negative value.
// Test case - 15.
{ bucketName , objectName , 2 , int64 ( len ( bytesData [ 0 ] . byteData ) ) , buffers [ 1 ] , buffers [ 1 ] , false , bytesData [ 0 ] . byteData ,
InvalidRange { 2 , int64 ( len ( bytesData [ 0 ] . byteData ) ) , int64 ( len ( bytesData [ 0 ] . byteData ) ) } } ,
// Test case with the writer set to nil.
// Test case - 16.
{ bucketName , objectName , 0 , int64 ( len ( bytesData [ 0 ] . byteData ) ) , buffers [ 1 ] , nil , false , bytesData [ 0 ] . byteData , errUnexpected } ,
}
for i , testCase := range testCases {
err = obj . GetObject ( testCase . bucketName , testCase . objectName , testCase . startOffset , testCase . length , testCase . writer )
if err != nil && testCase . shouldPass {
t . Errorf ( "Test %d: %s: Expected to pass, but failed with: <ERROR> %s" , i + 1 , instanceType , err . Error ( ) )
}
if err == nil && ! testCase . shouldPass {
t . Errorf ( "Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead." , i + 1 , instanceType , testCase . err . Error ( ) )
}
// Failed as expected, but does it fail for the expected reason.
if err != nil && ! testCase . shouldPass {
if ! strings . Contains ( err . Error ( ) , testCase . err . Error ( ) ) {
t . Errorf ( "Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead." , i + 1 , instanceType , testCase . err . Error ( ) , err . Error ( ) )
}
}
// Since there are cases for which GetObject fails, this is
// necessary. Test passes as expected, but the output values
// are verified for correctness here.
if err == nil && testCase . shouldPass {
if ! bytes . Equal ( testCase . expectedData , testCase . getObjectData . Bytes ( ) ) {
t . Errorf ( "Test %d: %s: Data Mismatch: Expected data and the fetched data from GetObject doesn't match." , i + 1 , instanceType )
}
// empty the buffer so that it can be used to further cases.
testCase . getObjectData . Reset ( )
}
}
}
2016-07-09 09:45:49 +02:00
// Benchmarks for ObjectLayer.GetObject().
// The intent is to benchamrk GetObject for various sizes ranging from few bytes to 100MB.
// Also each of these Benchmarks are run both XL and FS backends.
// BenchmarkGetObjectVerySmallFS - Benchmark FS.GetObject() for object size of 10 bytes.
func BenchmarkGetObjectVerySmallFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmark ( 10 ) )
}
// BenchmarkGetObjectVerySmallXL - Benchmark XL.GetObject() for object size of 10 bytes.
func BenchmarkGetObjectVerySmallXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmark ( 10 ) )
}
// BenchmarkGetObject10KbFS - Benchmark FS.GetObject() for object size of 10KB.
func BenchmarkGetObject10KbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmark ( 10 * 1024 ) )
}
// BenchmarkGetObject10KbXL - Benchmark XL.GetObject() for object size of 10KB.
func BenchmarkGetObject10KbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmark ( 10 * 1024 ) )
}
// BenchmarkGetObject100KbFS - Benchmark FS.GetObject() for object size of 100KB.
func BenchmarkGetObject100KbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmark ( 100 * 1024 ) )
}
// BenchmarkGetObject100KbXL - Benchmark XL.GetObject() for object size of 100KB.
func BenchmarkGetObject100KbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmark ( 100 * 1024 ) )
}
// BenchmarkGetObject1MbFS - Benchmark FS.GetObject() for object size of 1MB.
func BenchmarkGetObject1MbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmark ( 1024 * 1024 ) )
}
// BenchmarkGetObject1MbXL - Benchmark XL.GetObject() for object size of 1MB.
func BenchmarkGetObject1MbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmark ( 1024 * 1024 ) )
}
// BenchmarkGetObject5MbFS - Benchmark FS.GetObject() for object size of 5MB.
func BenchmarkGetObject5MbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmark ( 5 * 1024 * 1024 ) )
}
// BenchmarkGetObject5MbXL - Benchmark XL.GetObject() for object size of 5MB.
func BenchmarkGetObject5MbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmark ( 5 * 1024 * 1024 ) )
}
// BenchmarkGetObject10MbFS - Benchmark FS.GetObject() for object size of 10MB.
func BenchmarkGetObject10MbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmark ( 10 * 1024 * 1024 ) )
}
// BenchmarkGetObject10MbXL - Benchmark XL.GetObject() for object size of 10MB.
func BenchmarkGetObject10MbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmark ( 10 * 1024 * 1024 ) )
}
// BenchmarkGetObject25MbFS - Benchmark FS.GetObject() for object size of 25MB.
func BenchmarkGetObject25MbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmark ( 25 * 1024 * 1024 ) )
}
// BenchmarkGetObject25MbXL - Benchmark XL.GetObject() for object size of 25MB.
func BenchmarkGetObject25MbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmark ( 25 * 1024 * 1024 ) )
}
// BenchmarkGetObject50MbFS - Benchmark FS.GetObject() for object size of 50MB.
func BenchmarkGetObject50MbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmark ( 50 * 1024 * 1024 ) )
}
// BenchmarkGetObject50MbXL - Benchmark XL.GetObject() for object size of 50MB.
func BenchmarkGetObject50MbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmark ( 50 * 1024 * 1024 ) )
}
// BenchmarkGetObject100MbFS - Benchmark FS.GetObject() for object size of 100MB.
func BenchmarkGetObject100MbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmark ( 100 * 1024 * 1024 ) )
}
// BenchmarkGetObject100MbXL - Benchmark XL.GetObject() for object size of 100MB.
func BenchmarkGetObject100MbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmark ( 100 * 1024 * 1024 ) )
}
// BenchmarkGetObject200MbFS - Benchmark FS.GetObject() for object size of 200MB.
func BenchmarkGetObject200MbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmark ( 200 * 1024 * 1024 ) )
}
// BenchmarkGetObject200MbXL - Benchmark XL.GetObject() for object size of 200MB.
func BenchmarkGetObject200MbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmark ( 200 * 1024 * 1024 ) )
}
// BenchmarkGetObject500MbFS - Benchmark FS.GetObject() for object size of 500MB.
func BenchmarkGetObject500MbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmark ( 500 * 1024 * 1024 ) )
}
// BenchmarkGetObject500MbXL - Benchmark XL.GetObject() for object size of 500MB.
func BenchmarkGetObject500MbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmark ( 500 * 1024 * 1024 ) )
}
// BenchmarkGetObject1GbFS - Benchmark FS.GetObject() for object size of 1GB.
func BenchmarkGetObject1GbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmark ( 1024 * 1024 * 1024 ) )
}
// BenchmarkGetObjectGbXL - Benchmark XL.GetObject() for object size of 1GB.
func BenchmarkGetObject1GbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmark ( 1024 * 1024 * 1024 ) )
}
2016-07-10 20:08:45 +02:00
// The intent is to benchamrk GetObject for various sizes ranging from few bytes to 100MB.
// Also each of these BenchmarkParallels are run both XL and FS backends.
// BenchmarkParallelGetObjectVerySmallFS - BenchmarkParallel FS.GetObject() for object size of 10 bytes.
func BenchmarkParallelGetObjectVerySmallFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmarkParallel ( 10 ) )
}
// BenchmarkParallelGetObjectVerySmallXL - BenchmarkParallel XL.GetObject() for object size of 10 bytes.
func BenchmarkParallelGetObjectVerySmallXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmarkParallel ( 10 ) )
}
// BenchmarkParallelGetObject10KbFS - BenchmarkParallel FS.GetObject() for object size of 10KB.
func BenchmarkParallelGetObject10KbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmarkParallel ( 10 * 1024 ) )
}
// BenchmarkParallelGetObject10KbXL - BenchmarkParallel XL.GetObject() for object size of 10KB.
func BenchmarkParallelGetObject10KbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmarkParallel ( 10 * 1024 ) )
}
// BenchmarkParallelGetObject100KbFS - BenchmarkParallel FS.GetObject() for object size of 100KB.
func BenchmarkParallelGetObject100KbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmarkParallel ( 100 * 1024 ) )
}
// BenchmarkParallelGetObject100KbXL - BenchmarkParallel XL.GetObject() for object size of 100KB.
func BenchmarkParallelGetObject100KbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmarkParallel ( 100 * 1024 ) )
}
// BenchmarkParallelGetObject1MbFS - BenchmarkParallel FS.GetObject() for object size of 1MB.
func BenchmarkParallelGetObject1MbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmarkParallel ( 1024 * 1024 ) )
}
// BenchmarkParallelGetObject1MbXL - BenchmarkParallel XL.GetObject() for object size of 1MB.
func BenchmarkParallelGetObject1MbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmarkParallel ( 1024 * 1024 ) )
}
// BenchmarkParallelGetObject5MbFS - BenchmarkParallel FS.GetObject() for object size of 5MB.
func BenchmarkParallelGetObject5MbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmarkParallel ( 5 * 1024 * 1024 ) )
}
// BenchmarkParallelGetObject5MbXL - BenchmarkParallel XL.GetObject() for object size of 5MB.
func BenchmarkParallelGetObject5MbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmarkParallel ( 5 * 1024 * 1024 ) )
}
// BenchmarkParallelGetObject10MbFS - BenchmarkParallel FS.GetObject() for object size of 10MB.
func BenchmarkParallelGetObject10MbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmarkParallel ( 10 * 1024 * 1024 ) )
}
// BenchmarkParallelGetObject10MbXL - BenchmarkParallel XL.GetObject() for object size of 10MB.
func BenchmarkParallelGetObject10MbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmarkParallel ( 10 * 1024 * 1024 ) )
}
// BenchmarkParallelGetObject25MbFS - BenchmarkParallel FS.GetObject() for object size of 25MB.
func BenchmarkParallelGetObject25MbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmarkParallel ( 25 * 1024 * 1024 ) )
}
// BenchmarkParallelGetObject25MbXL - BenchmarkParallel XL.GetObject() for object size of 25MB.
func BenchmarkParallelGetObject25MbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmarkParallel ( 25 * 1024 * 1024 ) )
}
// BenchmarkParallelGetObject50MbFS - BenchmarkParallel FS.GetObject() for object size of 50MB.
func BenchmarkParallelGetObject50MbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmarkParallel ( 50 * 1024 * 1024 ) )
}
// BenchmarkParallelGetObject50MbXL - BenchmarkParallel XL.GetObject() for object size of 50MB.
func BenchmarkParallelGetObject50MbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmarkParallel ( 50 * 1024 * 1024 ) )
}
// BenchmarkParallelGetObject100MbFS - BenchmarkParallel FS.GetObject() for object size of 100MB.
func BenchmarkParallelGetObject100MbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmarkParallel ( 100 * 1024 * 1024 ) )
}
// BenchmarkParallelGetObject100MbXL - BenchmarkParallel XL.GetObject() for object size of 100MB.
func BenchmarkParallelGetObject100MbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmarkParallel ( 100 * 1024 * 1024 ) )
}
// BenchmarkParallelGetObject200MbFS - BenchmarkParallel FS.GetObject() for object size of 200MB.
func BenchmarkParallelGetObject200MbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmarkParallel ( 200 * 1024 * 1024 ) )
}
// BenchmarkParallelGetObject200MbXL - BenchmarkParallel XL.GetObject() for object size of 200MB.
func BenchmarkParallelGetObject200MbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmarkParallel ( 200 * 1024 * 1024 ) )
}
// BenchmarkParallelGetObject500MbFS - BenchmarkParallel FS.GetObject() for object size of 500MB.
func BenchmarkParallelGetObject500MbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmarkParallel ( 500 * 1024 * 1024 ) )
}
// BenchmarkParallelGetObject500MbXL - BenchmarkParallel XL.GetObject() for object size of 500MB.
func BenchmarkParallelGetObject500MbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmarkParallel ( 500 * 1024 * 1024 ) )
}
// BenchmarkParallelGetObject1GbFS - BenchmarkParallel FS.GetObject() for object size of 1GB.
func BenchmarkParallelGetObject1GbFS ( b * testing . B ) {
benchmarkGetObject ( b , "FS" , returnGetObjectBenchmarkParallel ( 1024 * 1024 * 1024 ) )
}
// BenchmarkParallelGetObjectGbXL - BenchmarkParallel XL.GetObject() for object size of 1GB.
func BenchmarkParallelGetObject1GbXL ( b * testing . B ) {
benchmarkGetObject ( b , "XL" , returnGetObjectBenchmarkParallel ( 1024 * 1024 * 1024 ) )
}