2016-06-27 19:01:09 +02:00
/ *
2019-04-09 20:39:42 +02:00
* MinIO Cloud Storage , ( C ) 2016 MinIO , Inc .
2016-06-27 19:01:09 +02:00
*
* Licensed under the Apache License , Version 2.0 ( the "License" ) ;
* you may not use this file except in compliance with the License .
* You may obtain a copy of the License at
*
* http : //www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing , software
* distributed under the License is distributed on an "AS IS" BASIS ,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
* See the License for the specific language governing permissions and
* limitations under the License .
* /
2016-08-19 01:23:42 +02:00
package cmd
2016-06-27 19:01:09 +02:00
import (
"bytes"
2018-03-15 21:27:16 +01:00
"context"
2016-06-29 11:28:46 +02:00
"crypto/md5"
"encoding/hex"
"io/ioutil"
"os"
"path"
2016-06-27 19:01:09 +02:00
"testing"
2016-11-23 03:18:22 +01:00
humanize "github.com/dustin/go-humanize"
2017-10-22 07:30:34 +02:00
"github.com/minio/minio/pkg/hash"
2016-06-27 19:01:09 +02:00
)
2016-07-21 22:15:54 +02:00
func md5Header ( data [ ] byte ) map [ string ] string {
2017-05-14 21:05:51 +02:00
return map [ string ] string { "etag" : getMD5Hash ( [ ] byte ( data ) ) }
2016-07-21 22:15:54 +02:00
}
2016-06-27 19:01:09 +02:00
// Wrapper for calling PutObject tests for both XL multiple disks and single node setup.
2017-10-22 07:30:34 +02:00
func TestObjectAPIPutObjectSingle ( t * testing . T ) {
2016-06-27 19:01:09 +02:00
ExecObjectLayerTest ( t , testObjectAPIPutObject )
}
// Tests validate correctness of PutObject.
2016-07-08 00:05:51 +02:00
func testObjectAPIPutObject ( obj ObjectLayer , instanceType string , t TestErrHandler ) {
2016-06-27 19:01:09 +02:00
// Generating cases for which the PutObject fails.
bucket := "minio-bucket"
object := "minio-object"
// Create bucket.
2020-05-08 22:44:44 +02:00
err := obj . MakeBucketWithLocation ( context . Background ( ) , bucket , "" , false )
2016-06-27 19:01:09 +02:00
if err != nil {
// Failed to create newbucket, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
// Creating a dummy bucket for tests.
2020-05-08 22:44:44 +02:00
err = obj . MakeBucketWithLocation ( context . Background ( ) , "unused-bucket" , "" , false )
2016-06-27 19:01:09 +02:00
if err != nil {
// Failed to create newbucket, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
2016-07-21 22:15:54 +02:00
var (
nilBytes [ ] byte
data = [ ] byte ( "hello" )
2016-11-23 03:18:22 +01:00
fiveMBBytes = bytes . Repeat ( [ ] byte ( "a" ) , 5 * humanize . MiByte )
2016-07-21 22:15:54 +02:00
)
2016-11-21 22:51:05 +01:00
invalidMD5 := getMD5Hash ( [ ] byte ( "meh" ) )
2016-07-21 22:15:54 +02:00
invalidMD5Header := md5Header ( [ ] byte ( "meh" ) )
testCases := [ ] struct {
bucketName string
objName string
inputData [ ] byte
inputMeta map [ string ] string
2016-10-03 00:51:49 +02:00
inputSHA256 string
2016-07-21 22:15:54 +02:00
intputDataSize int64
2016-06-27 19:01:09 +02:00
// expected error output.
expectedMd5 string
expectedError error
} {
// Test case 1-4.
// Cases with invalid bucket name.
2018-04-24 05:27:33 +02:00
{ ".test" , "obj" , [ ] byte ( "" ) , nil , "" , 0 , "" , BucketNotFound { Bucket : ".test" } } ,
{ "------" , "obj" , [ ] byte ( "" ) , nil , "" , 0 , "" , BucketNotFound { Bucket : "------" } } ,
2016-10-03 00:51:49 +02:00
{ "$this-is-not-valid-too" , "obj" , [ ] byte ( "" ) , nil , "" , 0 , "" ,
2018-04-24 05:27:33 +02:00
BucketNotFound { Bucket : "$this-is-not-valid-too" } } ,
{ "a" , "obj" , [ ] byte ( "" ) , nil , "" , 0 , "" , BucketNotFound { Bucket : "a" } } ,
2016-07-21 22:15:54 +02:00
2016-06-27 19:01:09 +02:00
// Test case - 5.
// Case with invalid object names.
2016-10-03 00:51:49 +02:00
{ bucket , "" , [ ] byte ( "" ) , nil , "" , 0 , "" , ObjectNameInvalid { Bucket : bucket , Object : "" } } ,
2016-07-21 22:15:54 +02:00
2016-06-27 19:01:09 +02:00
// Test case - 6.
// Valid object and bucket names but non-existent bucket.
2016-10-03 00:51:49 +02:00
{ "abc" , "def" , [ ] byte ( "" ) , nil , "" , 0 , "" , BucketNotFound { Bucket : "abc" } } ,
2016-07-21 22:15:54 +02:00
2016-06-27 19:01:09 +02:00
// Test case - 7.
// Input to replicate Md5 mismatch.
2017-10-22 07:30:34 +02:00
{ bucket , object , [ ] byte ( "" ) , map [ string ] string { "etag" : "d41d8cd98f00b204e9800998ecf8427f" } , "" , 0 , "" ,
2018-11-14 19:23:44 +01:00
hash . BadDigest { ExpectedMD5 : "d41d8cd98f00b204e9800998ecf8427f" , CalculatedMD5 : "d41d8cd98f00b204e9800998ecf8427e" } } ,
2016-07-21 22:15:54 +02:00
2016-06-27 19:01:09 +02:00
// Test case - 8.
2016-10-03 00:51:49 +02:00
// With incorrect sha256.
2017-10-22 07:30:34 +02:00
{ bucket , object , [ ] byte ( "abcd" ) , map [ string ] string { "etag" : "e2fc714c4727ee9395f324cd2e7f331f" } ,
"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580" , int64 ( len ( "abcd" ) ) ,
2018-11-14 19:23:44 +01:00
"" , hash . SHA256Mismatch { ExpectedSHA256 : "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580" ,
CalculatedSHA256 : "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589" } } ,
2016-10-03 00:51:49 +02:00
// Test case - 9.
2016-06-27 19:01:09 +02:00
// Input with size more than the size of actual data inside the reader.
2017-10-22 07:30:34 +02:00
{ bucket , object , [ ] byte ( "abcd" ) , map [ string ] string { "etag" : "e2fc714c4727ee9395f324cd2e7f331e" } , "" , int64 ( len ( "abcd" ) + 1 ) , "" ,
2018-11-14 19:23:44 +01:00
hash . BadDigest { ExpectedMD5 : "e2fc714c4727ee9395f324cd2e7f331e" , CalculatedMD5 : "e2fc714c4727ee9395f324cd2e7f331f" } } ,
2016-07-21 22:15:54 +02:00
2016-10-03 00:51:49 +02:00
// Test case - 10.
2016-06-27 19:01:09 +02:00
// Input with size less than the size of actual data inside the reader.
2017-10-22 07:30:34 +02:00
{ bucket , object , [ ] byte ( "abcd" ) , map [ string ] string { "etag" : "900150983cd24fb0d6963f7d28e17f73" } , "" , int64 ( len ( "abcd" ) - 1 ) , "" ,
2018-11-14 19:23:44 +01:00
hash . BadDigest { ExpectedMD5 : "900150983cd24fb0d6963f7d28e17f73" , CalculatedMD5 : "900150983cd24fb0d6963f7d28e17f72" } } ,
2016-07-21 22:15:54 +02:00
2016-10-03 00:51:49 +02:00
// Test case - 11-14.
2016-06-27 19:01:09 +02:00
// Validating for success cases.
2017-05-14 21:05:51 +02:00
{ bucket , object , [ ] byte ( "abcd" ) , map [ string ] string { "etag" : "e2fc714c4727ee9395f324cd2e7f331f" } , "" , int64 ( len ( "abcd" ) ) , "" , nil } ,
{ bucket , object , [ ] byte ( "efgh" ) , map [ string ] string { "etag" : "1f7690ebdd9b4caf8fab49ca1757bf27" } , "" , int64 ( len ( "efgh" ) ) , "" , nil } ,
{ bucket , object , [ ] byte ( "ijkl" ) , map [ string ] string { "etag" : "09a0877d04abf8759f99adec02baf579" } , "" , int64 ( len ( "ijkl" ) ) , "" , nil } ,
{ bucket , object , [ ] byte ( "mnop" ) , map [ string ] string { "etag" : "e132e96a5ddad6da8b07bba6f6131fef" } , "" , int64 ( len ( "mnop" ) ) , "" , nil } ,
2016-06-27 19:01:09 +02:00
2016-10-03 00:51:49 +02:00
// Test case 15-17.
2016-07-21 22:15:54 +02:00
// With no metadata
2016-11-21 22:51:05 +01:00
{ bucket , object , data , nil , "" , int64 ( len ( data ) ) , getMD5Hash ( data ) , nil } ,
{ bucket , object , nilBytes , nil , "" , int64 ( len ( nilBytes ) ) , getMD5Hash ( nilBytes ) , nil } ,
{ bucket , object , fiveMBBytes , nil , "" , int64 ( len ( fiveMBBytes ) ) , getMD5Hash ( fiveMBBytes ) , nil } ,
2016-06-27 19:01:09 +02:00
2016-10-03 00:51:49 +02:00
// Test case 18-20.
2016-07-21 22:15:54 +02:00
// With arbitrary metadata
2016-11-21 22:51:05 +01:00
{ bucket , object , data , map [ string ] string { "answer" : "42" } , "" , int64 ( len ( data ) ) , getMD5Hash ( data ) , nil } ,
{ bucket , object , nilBytes , map [ string ] string { "answer" : "42" } , "" , int64 ( len ( nilBytes ) ) , getMD5Hash ( nilBytes ) , nil } ,
{ bucket , object , fiveMBBytes , map [ string ] string { "answer" : "42" } , "" , int64 ( len ( fiveMBBytes ) ) , getMD5Hash ( fiveMBBytes ) , nil } ,
2016-07-21 22:15:54 +02:00
2016-10-03 00:51:49 +02:00
// Test case 21-23.
// With valid md5sum and sha256.
2016-11-21 22:51:05 +01:00
{ bucket , object , data , md5Header ( data ) , getSHA256Hash ( data ) , int64 ( len ( data ) ) , getMD5Hash ( data ) , nil } ,
{ bucket , object , nilBytes , md5Header ( nilBytes ) , getSHA256Hash ( nilBytes ) , int64 ( len ( nilBytes ) ) , getMD5Hash ( nilBytes ) , nil } ,
{ bucket , object , fiveMBBytes , md5Header ( fiveMBBytes ) , getSHA256Hash ( fiveMBBytes ) , int64 ( len ( fiveMBBytes ) ) , getMD5Hash ( fiveMBBytes ) , nil } ,
2016-07-21 22:15:54 +02:00
2016-10-03 00:51:49 +02:00
// Test case 24-26.
2016-07-21 22:15:54 +02:00
// data with invalid md5sum in header
2017-10-22 07:30:34 +02:00
{ bucket , object , data , invalidMD5Header , "" , int64 ( len ( data ) ) , getMD5Hash ( data ) ,
2018-11-14 19:23:44 +01:00
hash . BadDigest { ExpectedMD5 : invalidMD5 , CalculatedMD5 : getMD5Hash ( data ) } } ,
2017-10-22 07:30:34 +02:00
{ bucket , object , nilBytes , invalidMD5Header , "" , int64 ( len ( nilBytes ) ) , getMD5Hash ( nilBytes ) ,
2018-11-14 19:23:44 +01:00
hash . BadDigest { ExpectedMD5 : invalidMD5 , CalculatedMD5 : getMD5Hash ( nilBytes ) } } ,
2017-10-22 07:30:34 +02:00
{ bucket , object , fiveMBBytes , invalidMD5Header , "" , int64 ( len ( fiveMBBytes ) ) , getMD5Hash ( fiveMBBytes ) ,
2018-11-14 19:23:44 +01:00
hash . BadDigest { ExpectedMD5 : invalidMD5 , CalculatedMD5 : getMD5Hash ( fiveMBBytes ) } } ,
2016-07-21 22:15:54 +02:00
2016-10-03 00:51:49 +02:00
// Test case 27-29.
2016-07-21 22:15:54 +02:00
// data with size different from the actual number of bytes available in the reader
2016-11-21 22:51:05 +01:00
{ bucket , object , data , nil , "" , int64 ( len ( data ) - 1 ) , getMD5Hash ( data [ : len ( data ) - 1 ] ) , nil } ,
{ bucket , object , nilBytes , nil , "" , int64 ( len ( nilBytes ) + 1 ) , getMD5Hash ( nilBytes ) , IncompleteBody { } } ,
2017-02-24 18:20:40 +01:00
{ bucket , object , fiveMBBytes , nil , "" , 0 , getMD5Hash ( fiveMBBytes ) , nil } ,
2016-09-16 22:06:49 +02:00
2016-10-03 00:51:49 +02:00
// Test case 30
2016-09-16 22:06:49 +02:00
// valid data with X-Amz-Meta- meta
2016-11-21 22:51:05 +01:00
{ bucket , object , data , map [ string ] string { "X-Amz-Meta-AppID" : "a42" } , "" , int64 ( len ( data ) ) , getMD5Hash ( data ) , nil } ,
2018-10-18 01:37:02 +02:00
// Test case 31
// Put an empty object with a trailing slash
{ bucket , "emptydir/" , [ ] byte { } , nil , "" , 0 , getMD5Hash ( [ ] byte { } ) , nil } ,
// Test case 32
// Put an object inside the empty directory
{ bucket , "emptydir/" + object , data , nil , "" , int64 ( len ( data ) ) , getMD5Hash ( data ) , nil } ,
// Test case 33
// Put the empty object with a trailing slash again (refer to Test case 31), this needs to succeed
{ bucket , "emptydir/" , [ ] byte { } , nil , "" , 0 , getMD5Hash ( [ ] byte { } ) , nil } ,
2016-07-21 22:15:54 +02:00
}
for i , testCase := range testCases {
2020-05-14 23:01:31 +02:00
in := mustGetPutObjReader ( t , bytes . NewReader ( testCase . inputData ) , testCase . intputDataSize , testCase . inputMeta [ "etag" ] , testCase . inputSHA256 )
objInfo , actualErr := obj . PutObject ( context . Background ( ) , testCase . bucketName , testCase . objName , in , ObjectOptions { UserDefined : testCase . inputMeta } )
2016-07-21 22:15:54 +02:00
if actualErr != nil && testCase . expectedError == nil {
t . Errorf ( "Test %d: %s: Expected to pass, but failed with: error %s." , i + 1 , instanceType , actualErr . Error ( ) )
2020-05-14 23:01:31 +02:00
continue
2016-06-27 19:01:09 +02:00
}
2016-07-21 22:15:54 +02:00
if actualErr == nil && testCase . expectedError != nil {
t . Errorf ( "Test %d: %s: Expected to fail with error \"%s\", but passed instead." , i + 1 , instanceType , testCase . expectedError . Error ( ) )
2020-05-14 23:01:31 +02:00
continue
2016-06-27 19:01:09 +02:00
}
// Failed as expected, but does it fail for the expected reason.
2016-08-25 18:39:01 +02:00
if actualErr != nil && actualErr != testCase . expectedError {
2018-10-18 01:37:02 +02:00
t . Errorf ( "Test %d: %s: Expected to fail with error \"%v\", but instead failed with error \"%v\" instead." , i + 1 , instanceType , testCase . expectedError , actualErr )
2020-05-14 23:01:31 +02:00
continue
2016-06-27 19:01:09 +02:00
}
// Test passes as expected, but the output values are verified for correctness here.
2016-07-21 22:15:54 +02:00
if actualErr == nil {
2016-06-27 19:01:09 +02:00
// Asserting whether the md5 output is correct.
2017-05-14 21:05:51 +02:00
if expectedMD5 , ok := testCase . inputMeta [ "etag" ] ; ok && expectedMD5 != objInfo . ETag {
t . Errorf ( "Test %d: %s: Calculated Md5 different from the actual one %s." , i + 1 , instanceType , objInfo . ETag )
2020-05-14 23:01:31 +02:00
continue
2016-06-27 19:01:09 +02:00
}
}
}
}
// Wrapper for calling PutObject tests for both XL multiple disks case
2016-06-28 08:54:56 +02:00
// when quorum is not available.
2016-06-27 19:01:09 +02:00
func TestObjectAPIPutObjectDiskNotFound ( t * testing . T ) {
2017-03-07 23:48:56 +01:00
ExecObjectLayerDiskAlteredTest ( t , testObjectAPIPutObjectDiskNotFound )
2016-06-27 19:01:09 +02:00
}
// Tests validate correctness of PutObject.
2017-03-07 23:48:56 +01:00
func testObjectAPIPutObjectDiskNotFound ( obj ObjectLayer , instanceType string , disks [ ] string , t * testing . T ) {
2016-06-27 19:01:09 +02:00
// Generating cases for which the PutObject fails.
bucket := "minio-bucket"
object := "minio-object"
// Create bucket.
2020-05-08 22:44:44 +02:00
err := obj . MakeBucketWithLocation ( context . Background ( ) , bucket , "" , false )
2016-06-27 19:01:09 +02:00
if err != nil {
// Failed to create newbucket, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
// Creating a dummy bucket for tests.
2020-05-08 22:44:44 +02:00
err = obj . MakeBucketWithLocation ( context . Background ( ) , "unused-bucket" , "" , false )
2016-06-27 19:01:09 +02:00
if err != nil {
// Failed to create newbucket, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
2016-07-22 04:06:50 +02:00
// Take 8 disks down, one more we loose quorum on 16 disk node.
2016-07-28 11:20:34 +02:00
for _ , disk := range disks [ : 7 ] {
2017-08-13 04:25:43 +02:00
os . RemoveAll ( disk )
2016-06-27 19:01:09 +02:00
}
testCases := [ ] struct {
2016-07-21 22:15:54 +02:00
bucketName string
objName string
inputData [ ] byte
inputMeta map [ string ] string
intputDataSize int64
2016-06-27 19:01:09 +02:00
// flag indicating whether the test should pass.
shouldPass bool
// expected error output.
expectedMd5 string
expectedError error
} {
// Validating for success cases.
2017-05-14 21:05:51 +02:00
{ bucket , object , [ ] byte ( "abcd" ) , map [ string ] string { "etag" : "e2fc714c4727ee9395f324cd2e7f331f" } , int64 ( len ( "abcd" ) ) , true , "" , nil } ,
{ bucket , object , [ ] byte ( "efgh" ) , map [ string ] string { "etag" : "1f7690ebdd9b4caf8fab49ca1757bf27" } , int64 ( len ( "efgh" ) ) , true , "" , nil } ,
{ bucket , object , [ ] byte ( "ijkl" ) , map [ string ] string { "etag" : "09a0877d04abf8759f99adec02baf579" } , int64 ( len ( "ijkl" ) ) , true , "" , nil } ,
{ bucket , object , [ ] byte ( "mnop" ) , map [ string ] string { "etag" : "e132e96a5ddad6da8b07bba6f6131fef" } , int64 ( len ( "mnop" ) ) , true , "" , nil } ,
2016-06-27 19:01:09 +02:00
}
2016-10-03 00:51:49 +02:00
sha256sum := ""
2016-06-27 19:01:09 +02:00
for i , testCase := range testCases {
2019-02-09 06:31:06 +01:00
objInfo , actualErr := obj . PutObject ( context . Background ( ) , testCase . bucketName , testCase . objName , mustGetPutObjReader ( t , bytes . NewReader ( testCase . inputData ) , testCase . intputDataSize , testCase . inputMeta [ "etag" ] , sha256sum ) , ObjectOptions { UserDefined : testCase . inputMeta } )
2016-06-27 19:01:09 +02:00
if actualErr != nil && testCase . shouldPass {
t . Errorf ( "Test %d: %s: Expected to pass, but failed with: <ERROR> %s." , i + 1 , instanceType , actualErr . Error ( ) )
}
if actualErr == nil && ! testCase . shouldPass {
t . Errorf ( "Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead." , i + 1 , instanceType , testCase . expectedError . Error ( ) )
}
// Failed as expected, but does it fail for the expected reason.
if actualErr != nil && ! testCase . shouldPass {
if testCase . expectedError . Error ( ) != actualErr . Error ( ) {
t . Errorf ( "Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead." , i + 1 ,
instanceType , testCase . expectedError . Error ( ) , actualErr . Error ( ) )
}
}
// Test passes as expected, but the output values are verified for correctness here.
if actualErr == nil && testCase . shouldPass {
// Asserting whether the md5 output is correct.
2017-05-14 21:05:51 +02:00
if testCase . inputMeta [ "etag" ] != objInfo . ETag {
t . Errorf ( "Test %d: %s: Calculated Md5 different from the actual one %s." , i + 1 , instanceType , objInfo . ETag )
2016-06-27 19:01:09 +02:00
}
}
}
// This causes quorum failure verify.
2017-08-13 04:25:43 +02:00
os . RemoveAll ( disks [ len ( disks ) - 1 ] )
2016-06-27 19:01:09 +02:00
// Validate the last test.
testCase := struct {
2016-07-21 22:15:54 +02:00
bucketName string
objName string
inputData [ ] byte
inputMeta map [ string ] string
intputDataSize int64
2016-06-27 19:01:09 +02:00
// flag indicating whether the test should pass.
shouldPass bool
// expected error output.
expectedMd5 string
expectedError error
} {
bucket ,
object ,
2016-07-21 22:15:54 +02:00
[ ] byte ( "mnop" ) ,
2017-05-14 21:05:51 +02:00
map [ string ] string { "etag" : "e132e96a5ddad6da8b07bba6f6131fef" } ,
2016-06-27 19:01:09 +02:00
int64 ( len ( "mnop" ) ) ,
false ,
"" ,
InsufficientWriteQuorum { } ,
}
2016-10-03 00:51:49 +02:00
2019-02-09 06:31:06 +01:00
_ , actualErr := obj . PutObject ( context . Background ( ) , testCase . bucketName , testCase . objName , mustGetPutObjReader ( t , bytes . NewReader ( testCase . inputData ) , testCase . intputDataSize , testCase . inputMeta [ "etag" ] , sha256sum ) , ObjectOptions { UserDefined : testCase . inputMeta } )
2016-06-27 19:01:09 +02:00
if actualErr != nil && testCase . shouldPass {
t . Errorf ( "Test %d: %s: Expected to pass, but failed with: <ERROR> %s." , len ( testCases ) + 1 , instanceType , actualErr . Error ( ) )
}
// Failed as expected, but does it fail for the expected reason.
if actualErr != nil && ! testCase . shouldPass {
if testCase . expectedError . Error ( ) != actualErr . Error ( ) {
t . Errorf ( "Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead." , len ( testCases ) + 1 , instanceType , testCase . expectedError . Error ( ) , actualErr . Error ( ) )
}
}
}
2016-06-29 11:28:46 +02:00
// Wrapper for calling PutObject tests for both XL multiple disks and single node setup.
func TestObjectAPIPutObjectStaleFiles ( t * testing . T ) {
ExecObjectLayerStaleFilesTest ( t , testObjectAPIPutObjectStaleFiles )
}
// Tests validate correctness of PutObject.
func testObjectAPIPutObjectStaleFiles ( obj ObjectLayer , instanceType string , disks [ ] string , t * testing . T ) {
// Generating cases for which the PutObject fails.
bucket := "minio-bucket"
object := "minio-object"
// Create bucket.
2020-05-08 22:44:44 +02:00
err := obj . MakeBucketWithLocation ( context . Background ( ) , bucket , "" , false )
2016-06-29 11:28:46 +02:00
if err != nil {
// Failed to create newbucket, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
data := [ ] byte ( "hello, world" )
// Create object.
2019-02-09 06:31:06 +01:00
_ , err = obj . PutObject ( context . Background ( ) , bucket , object , mustGetPutObjReader ( t , bytes . NewReader ( data ) , int64 ( len ( data ) ) , "" , "" ) , ObjectOptions { } )
2016-06-29 11:28:46 +02:00
if err != nil {
// Failed to create object, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
for _ , disk := range disks {
2016-11-20 23:25:43 +01:00
tmpMetaDir := path . Join ( disk , minioMetaTmpBucket )
2016-06-29 11:28:46 +02:00
if ! isDirEmpty ( tmpMetaDir ) {
2016-11-20 23:25:43 +01:00
t . Fatalf ( "%s: expected: empty, got: non-empty" , minioMetaTmpBucket )
2016-06-29 11:28:46 +02:00
}
}
}
// Wrapper for calling Multipart PutObject tests for both XL multiple disks and single node setup.
func TestObjectAPIMultipartPutObjectStaleFiles ( t * testing . T ) {
ExecObjectLayerStaleFilesTest ( t , testObjectAPIMultipartPutObjectStaleFiles )
}
// Tests validate correctness of PutObject.
func testObjectAPIMultipartPutObjectStaleFiles ( obj ObjectLayer , instanceType string , disks [ ] string , t * testing . T ) {
// Generating cases for which the PutObject fails.
bucket := "minio-bucket"
object := "minio-object"
// Create bucket.
2020-05-08 22:44:44 +02:00
err := obj . MakeBucketWithLocation ( context . Background ( ) , bucket , "" , false )
2016-06-29 11:28:46 +02:00
if err != nil {
// Failed to create newbucket, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
2018-09-10 18:42:43 +02:00
opts := ObjectOptions { }
2016-06-29 11:28:46 +02:00
// Initiate Multipart Upload on the above created bucket.
2019-02-09 06:31:06 +01:00
uploadID , err := obj . NewMultipartUpload ( context . Background ( ) , bucket , object , opts )
2016-06-29 11:28:46 +02:00
if err != nil {
// Failed to create NewMultipartUpload, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
// Upload part1.
2016-11-23 03:18:22 +01:00
fiveMBBytes := bytes . Repeat ( [ ] byte ( "a" ) , 5 * humanize . MiByte )
2016-06-29 11:28:46 +02:00
md5Writer := md5 . New ( )
md5Writer . Write ( fiveMBBytes )
etag1 := hex . EncodeToString ( md5Writer . Sum ( nil ) )
2016-10-03 00:51:49 +02:00
sha256sum := ""
2018-11-15 02:36:41 +01:00
_ , err = obj . PutObjectPart ( context . Background ( ) , bucket , object , uploadID , 1 , mustGetPutObjReader ( t , bytes . NewReader ( fiveMBBytes ) , int64 ( len ( fiveMBBytes ) ) , etag1 , sha256sum ) , opts )
2016-06-29 11:28:46 +02:00
if err != nil {
// Failed to upload object part, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
// Upload part2.
data := [ ] byte ( "hello, world" )
md5Writer = md5 . New ( )
md5Writer . Write ( data )
etag2 := hex . EncodeToString ( md5Writer . Sum ( nil ) )
2018-11-15 02:36:41 +01:00
_ , err = obj . PutObjectPart ( context . Background ( ) , bucket , object , uploadID , 2 , mustGetPutObjReader ( t , bytes . NewReader ( data ) , int64 ( len ( data ) ) , etag2 , sha256sum ) , opts )
2016-06-29 11:28:46 +02:00
if err != nil {
// Failed to upload object part, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
// Complete multipart.
2017-11-14 09:25:10 +01:00
parts := [ ] CompletePart {
2016-06-29 11:28:46 +02:00
{ ETag : etag1 , PartNumber : 1 } ,
{ ETag : etag2 , PartNumber : 2 } ,
}
2018-11-15 02:36:41 +01:00
_ , err = obj . CompleteMultipartUpload ( context . Background ( ) , bucket , object , uploadID , parts , ObjectOptions { } )
2016-06-29 11:28:46 +02:00
if err != nil {
// Failed to complete multipart upload, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
for _ , disk := range disks {
2016-11-20 23:25:43 +01:00
tmpMetaDir := path . Join ( disk , minioMetaTmpBucket )
2016-06-29 11:28:46 +02:00
files , err := ioutil . ReadDir ( tmpMetaDir )
if err != nil {
// Its OK to have non-existen tmpMetaDir.
if os . IsNotExist ( err ) {
continue
}
// Print the error
t . Errorf ( "%s" , err )
}
if len ( files ) != 0 {
t . Fatalf ( "%s: expected: empty, got: non-empty. content: %s" , tmpMetaDir , files )
}
}
}
2016-07-09 09:45:49 +02:00
// Benchmarks for ObjectLayer.PutObject().
2016-08-15 11:44:48 +02:00
// The intent is to benchmark PutObject for various sizes ranging from few bytes to 100MB.
2016-07-09 09:45:49 +02:00
// Also each of these Benchmarks are run both XL and FS backends.
// BenchmarkPutObjectVerySmallFS - Benchmark FS.PutObject() for object size of 10 bytes.
func BenchmarkPutObjectVerySmallFS ( b * testing . B ) {
2016-07-21 20:17:28 +02:00
benchmarkPutObject ( b , "FS" , 10 )
2016-07-09 09:45:49 +02:00
}
// BenchmarkPutObjectVerySmallXL - Benchmark XL.PutObject() for object size of 10 bytes.
func BenchmarkPutObjectVerySmallXL ( b * testing . B ) {
2016-07-21 20:17:28 +02:00
benchmarkPutObject ( b , "XL" , 10 )
2016-07-09 09:45:49 +02:00
}
// BenchmarkPutObject10KbFS - Benchmark FS.PutObject() for object size of 10KB.
func BenchmarkPutObject10KbFS ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObject ( b , "FS" , 10 * humanize . KiByte )
2016-07-09 09:45:49 +02:00
}
// BenchmarkPutObject10KbXL - Benchmark XL.PutObject() for object size of 10KB.
func BenchmarkPutObject10KbXL ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObject ( b , "XL" , 10 * humanize . KiByte )
2016-07-09 09:45:49 +02:00
}
// BenchmarkPutObject100KbFS - Benchmark FS.PutObject() for object size of 100KB.
func BenchmarkPutObject100KbFS ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObject ( b , "FS" , 100 * humanize . KiByte )
2016-07-09 09:45:49 +02:00
}
// BenchmarkPutObject100KbXL - Benchmark XL.PutObject() for object size of 100KB.
func BenchmarkPutObject100KbXL ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObject ( b , "XL" , 100 * humanize . KiByte )
2016-07-09 09:45:49 +02:00
}
// BenchmarkPutObject1MbFS - Benchmark FS.PutObject() for object size of 1MB.
func BenchmarkPutObject1MbFS ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObject ( b , "FS" , 1 * humanize . MiByte )
2016-07-09 09:45:49 +02:00
}
// BenchmarkPutObject1MbXL - Benchmark XL.PutObject() for object size of 1MB.
func BenchmarkPutObject1MbXL ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObject ( b , "XL" , 1 * humanize . MiByte )
2016-07-09 09:45:49 +02:00
}
// BenchmarkPutObject5MbFS - Benchmark FS.PutObject() for object size of 5MB.
func BenchmarkPutObject5MbFS ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObject ( b , "FS" , 5 * humanize . MiByte )
2016-07-09 09:45:49 +02:00
}
// BenchmarkPutObject5MbXL - Benchmark XL.PutObject() for object size of 5MB.
func BenchmarkPutObject5MbXL ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObject ( b , "XL" , 5 * humanize . MiByte )
2016-07-09 09:45:49 +02:00
}
// BenchmarkPutObject10MbFS - Benchmark FS.PutObject() for object size of 10MB.
func BenchmarkPutObject10MbFS ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObject ( b , "FS" , 10 * humanize . MiByte )
2016-07-09 09:45:49 +02:00
}
// BenchmarkPutObject10MbXL - Benchmark XL.PutObject() for object size of 10MB.
func BenchmarkPutObject10MbXL ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObject ( b , "XL" , 10 * humanize . MiByte )
2016-07-09 09:45:49 +02:00
}
// BenchmarkPutObject25MbFS - Benchmark FS.PutObject() for object size of 25MB.
func BenchmarkPutObject25MbFS ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObject ( b , "FS" , 25 * humanize . MiByte )
2016-07-09 09:45:49 +02:00
}
// BenchmarkPutObject25MbXL - Benchmark XL.PutObject() for object size of 25MB.
func BenchmarkPutObject25MbXL ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObject ( b , "XL" , 25 * humanize . MiByte )
2016-07-09 09:45:49 +02:00
}
// BenchmarkPutObject50MbFS - Benchmark FS.PutObject() for object size of 50MB.
func BenchmarkPutObject50MbFS ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObject ( b , "FS" , 50 * humanize . MiByte )
2016-07-09 09:45:49 +02:00
}
// BenchmarkPutObject50MbXL - Benchmark XL.PutObject() for object size of 50MB.
func BenchmarkPutObject50MbXL ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObject ( b , "XL" , 50 * humanize . MiByte )
2016-07-09 09:45:49 +02:00
}
2016-07-21 20:17:28 +02:00
// parallel benchmarks for ObjectLayer.PutObject() .
2016-07-10 20:08:45 +02:00
2016-07-21 20:17:28 +02:00
// BenchmarkParallelPutObjectVerySmallFS - BenchmarkParallel FS.PutObject() for object size of 10 bytes.
2016-07-10 20:08:45 +02:00
func BenchmarkParallelPutObjectVerySmallFS ( b * testing . B ) {
2016-07-21 20:17:28 +02:00
benchmarkPutObjectParallel ( b , "FS" , 10 )
2016-07-10 20:08:45 +02:00
}
// BenchmarkParallelPutObjectVerySmallXL - BenchmarkParallel XL.PutObject() for object size of 10 bytes.
func BenchmarkParallelPutObjectVerySmallXL ( b * testing . B ) {
2016-07-21 20:17:28 +02:00
benchmarkPutObjectParallel ( b , "XL" , 10 )
2016-07-10 20:08:45 +02:00
}
// BenchmarkParallelPutObject10KbFS - BenchmarkParallel FS.PutObject() for object size of 10KB.
func BenchmarkParallelPutObject10KbFS ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObjectParallel ( b , "FS" , 10 * humanize . KiByte )
2016-07-10 20:08:45 +02:00
}
// BenchmarkParallelPutObject10KbXL - BenchmarkParallel XL.PutObject() for object size of 10KB.
func BenchmarkParallelPutObject10KbXL ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObjectParallel ( b , "XL" , 10 * humanize . KiByte )
2016-07-10 20:08:45 +02:00
}
// BenchmarkParallelPutObject100KbFS - BenchmarkParallel FS.PutObject() for object size of 100KB.
func BenchmarkParallelPutObject100KbFS ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObjectParallel ( b , "FS" , 100 * humanize . KiByte )
2016-07-10 20:08:45 +02:00
}
// BenchmarkParallelPutObject100KbXL - BenchmarkParallel XL.PutObject() for object size of 100KB.
func BenchmarkParallelPutObject100KbXL ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObjectParallel ( b , "XL" , 100 * humanize . KiByte )
2016-07-10 20:08:45 +02:00
}
// BenchmarkParallelPutObject1MbFS - BenchmarkParallel FS.PutObject() for object size of 1MB.
func BenchmarkParallelPutObject1MbFS ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObjectParallel ( b , "FS" , 1 * humanize . MiByte )
2016-07-10 20:08:45 +02:00
}
// BenchmarkParallelPutObject1MbXL - BenchmarkParallel XL.PutObject() for object size of 1MB.
func BenchmarkParallelPutObject1MbXL ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObjectParallel ( b , "XL" , 1 * humanize . MiByte )
2016-07-10 20:08:45 +02:00
}
// BenchmarkParallelPutObject5MbFS - BenchmarkParallel FS.PutObject() for object size of 5MB.
func BenchmarkParallelPutObject5MbFS ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObjectParallel ( b , "FS" , 5 * humanize . MiByte )
2016-07-10 20:08:45 +02:00
}
// BenchmarkParallelPutObject5MbXL - BenchmarkParallel XL.PutObject() for object size of 5MB.
func BenchmarkParallelPutObject5MbXL ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObjectParallel ( b , "XL" , 5 * humanize . MiByte )
2016-07-10 20:08:45 +02:00
}
// BenchmarkParallelPutObject10MbFS - BenchmarkParallel FS.PutObject() for object size of 10MB.
func BenchmarkParallelPutObject10MbFS ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObjectParallel ( b , "FS" , 10 * humanize . MiByte )
2016-07-10 20:08:45 +02:00
}
// BenchmarkParallelPutObject10MbXL - BenchmarkParallel XL.PutObject() for object size of 10MB.
func BenchmarkParallelPutObject10MbXL ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObjectParallel ( b , "XL" , 10 * humanize . MiByte )
2016-07-10 20:08:45 +02:00
}
// BenchmarkParallelPutObject25MbFS - BenchmarkParallel FS.PutObject() for object size of 25MB.
func BenchmarkParallelPutObject25MbFS ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObjectParallel ( b , "FS" , 25 * humanize . MiByte )
2016-07-10 20:08:45 +02:00
}
// BenchmarkParallelPutObject25MbXL - BenchmarkParallel XL.PutObject() for object size of 25MB.
func BenchmarkParallelPutObject25MbXL ( b * testing . B ) {
2016-11-23 03:18:22 +01:00
benchmarkPutObjectParallel ( b , "XL" , 25 * humanize . MiByte )
2016-07-10 20:08:45 +02:00
}