2016-09-16 22:06:49 +02:00
/ *
2019-04-09 20:39:42 +02:00
* MinIO Cloud Storage , ( C ) 2016 , 2017 MinIO , Inc .
2016-09-16 22:06:49 +02:00
*
* Licensed under the Apache License , Version 2.0 ( the "License" ) ;
* you may not use this file except in compliance with the License .
* You may obtain a copy of the License at
*
* http : //www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing , software
* distributed under the License is distributed on an "AS IS" BASIS ,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
* See the License for the specific language governing permissions and
* limitations under the License .
* /
package cmd
import (
"bytes"
2018-03-15 21:27:16 +01:00
"context"
2017-08-13 04:25:43 +02:00
"os"
2016-09-16 22:06:49 +02:00
"path/filepath"
2020-03-19 00:19:29 +01:00
"sync"
2016-09-16 22:06:49 +02:00
"testing"
2017-08-10 23:11:57 +02:00
"time"
2016-09-16 22:06:49 +02:00
)
2017-12-01 03:11:42 +01:00
// Tests cleanup multipart uploads for filesystem backend.
2017-08-10 23:11:57 +02:00
func TestFSCleanupMultipartUploadsInRoutine ( t * testing . T ) {
// Prepare for tests
disk := filepath . Join ( globalTestTmpDir , "minio-" + nextSuffix ( ) )
2017-08-13 04:25:43 +02:00
defer os . RemoveAll ( disk )
2017-08-10 23:11:57 +02:00
obj := initFSObjects ( disk , t )
2018-02-20 21:21:12 +01:00
fs := obj . ( * FSObjects )
2017-08-10 23:11:57 +02:00
bucketName := "bucket"
objectName := "object"
2020-03-19 00:19:29 +01:00
// Create a context we can cancel.
2020-04-09 18:30:02 +02:00
ctx , cancel := context . WithCancel ( GlobalContext )
2020-05-08 22:44:44 +02:00
obj . MakeBucketWithLocation ( ctx , bucketName , "" , false )
2020-03-19 00:19:29 +01:00
uploadID , err := obj . NewMultipartUpload ( ctx , bucketName , objectName , ObjectOptions { } )
2017-08-10 23:11:57 +02:00
if err != nil {
t . Fatal ( "Unexpected err: " , err )
}
2020-03-19 00:19:29 +01:00
var cleanupWg sync . WaitGroup
cleanupWg . Add ( 1 )
go func ( ) {
defer cleanupWg . Done ( )
2020-04-16 19:56:18 +02:00
fs . cleanupStaleMultipartUploads ( ctx , time . Millisecond , 0 )
2020-03-19 00:19:29 +01:00
} ( )
2017-08-10 23:11:57 +02:00
2020-03-19 00:19:29 +01:00
// Wait for 100ms such that - we have given enough time for
// cleanup routine to kick in. Flaky on slow systems...
time . Sleep ( 100 * time . Millisecond )
cancel ( )
cleanupWg . Wait ( )
2017-08-10 23:11:57 +02:00
// Check if upload id was already purged.
2020-04-09 18:30:02 +02:00
if err = obj . AbortMultipartUpload ( GlobalContext , bucketName , objectName , uploadID ) ; err != nil {
2017-08-10 23:11:57 +02:00
if _ , ok := err . ( InvalidUploadID ) ; ! ok {
t . Fatal ( "Unexpected err: " , err )
}
2020-03-19 00:19:29 +01:00
} else {
t . Error ( "Item was not cleaned up." )
2017-08-10 23:11:57 +02:00
}
}
2016-09-16 22:06:49 +02:00
// TestNewMultipartUploadFaultyDisk - test NewMultipartUpload with faulty disks
func TestNewMultipartUploadFaultyDisk ( t * testing . T ) {
// Prepare for tests
2016-12-16 07:25:05 +01:00
disk := filepath . Join ( globalTestTmpDir , "minio-" + nextSuffix ( ) )
2017-08-13 04:25:43 +02:00
defer os . RemoveAll ( disk )
2016-10-05 21:48:07 +02:00
obj := initFSObjects ( disk , t )
2016-09-16 22:06:49 +02:00
2018-02-20 21:21:12 +01:00
fs := obj . ( * FSObjects )
2016-09-16 22:06:49 +02:00
bucketName := "bucket"
objectName := "object"
2020-05-08 22:44:44 +02:00
if err := obj . MakeBucketWithLocation ( GlobalContext , bucketName , "" , false ) ; err != nil {
2016-09-16 22:06:49 +02:00
t . Fatal ( "Cannot create bucket, err: " , err )
}
2017-01-17 02:05:00 +01:00
// Test with disk removed.
2018-05-23 12:11:29 +02:00
os . RemoveAll ( disk )
2020-04-09 18:30:02 +02:00
if _ , err := fs . NewMultipartUpload ( GlobalContext , bucketName , objectName , ObjectOptions { UserDefined : map [ string ] string { "X-Amz-Meta-xid" : "3f" } } ) ; err != nil {
2018-04-10 18:36:37 +02:00
if ! isSameType ( err , BucketNotFound { } ) {
2017-01-17 02:05:00 +01:00
t . Fatal ( "Unexpected error " , err )
2016-09-16 22:06:49 +02:00
}
}
}
// TestPutObjectPartFaultyDisk - test PutObjectPart with faulty disks
func TestPutObjectPartFaultyDisk ( t * testing . T ) {
// Prepare for tests
2016-12-16 07:25:05 +01:00
disk := filepath . Join ( globalTestTmpDir , "minio-" + nextSuffix ( ) )
2017-08-13 04:25:43 +02:00
defer os . RemoveAll ( disk )
2016-10-05 21:48:07 +02:00
obj := initFSObjects ( disk , t )
2018-08-15 06:41:47 +02:00
2016-09-16 22:06:49 +02:00
bucketName := "bucket"
objectName := "object"
data := [ ] byte ( "12345" )
dataLen := int64 ( len ( data ) )
2020-05-08 22:44:44 +02:00
if err := obj . MakeBucketWithLocation ( GlobalContext , bucketName , "" , false ) ; err != nil {
2016-09-16 22:06:49 +02:00
t . Fatal ( "Cannot create bucket, err: " , err )
}
2020-04-09 18:30:02 +02:00
uploadID , err := obj . NewMultipartUpload ( GlobalContext , bucketName , objectName , ObjectOptions { UserDefined : map [ string ] string { "X-Amz-Meta-xid" : "3f" } } )
2016-09-16 22:06:49 +02:00
if err != nil {
t . Fatal ( "Unexpected error " , err )
}
2016-11-21 22:51:05 +01:00
md5Hex := getMD5Hash ( data )
2016-10-03 00:51:49 +02:00
sha256sum := ""
2016-09-16 22:06:49 +02:00
2019-09-22 19:45:33 +02:00
newDisk := filepath . Join ( globalTestTmpDir , "minio-" + nextSuffix ( ) )
defer os . RemoveAll ( newDisk )
obj = initFSObjects ( newDisk , t )
2020-04-09 18:30:02 +02:00
if _ , err = obj . PutObjectPart ( GlobalContext , bucketName , objectName , uploadID , 1 , mustGetPutObjReader ( t , bytes . NewReader ( data ) , dataLen , md5Hex , sha256sum ) , ObjectOptions { } ) ; err != nil {
2019-09-22 19:45:33 +02:00
if ! isSameType ( err , BucketNotFound { } ) {
t . Fatal ( "Unexpected error " , err )
}
2016-09-16 22:06:49 +02:00
}
}
// TestCompleteMultipartUploadFaultyDisk - test CompleteMultipartUpload with faulty disks
func TestCompleteMultipartUploadFaultyDisk ( t * testing . T ) {
// Prepare for tests
2016-12-16 07:25:05 +01:00
disk := filepath . Join ( globalTestTmpDir , "minio-" + nextSuffix ( ) )
2017-08-13 04:25:43 +02:00
defer os . RemoveAll ( disk )
2016-10-05 21:48:07 +02:00
obj := initFSObjects ( disk , t )
2016-09-16 22:06:49 +02:00
bucketName := "bucket"
objectName := "object"
data := [ ] byte ( "12345" )
2020-05-08 22:44:44 +02:00
if err := obj . MakeBucketWithLocation ( GlobalContext , bucketName , "" , false ) ; err != nil {
2016-09-16 22:06:49 +02:00
t . Fatal ( "Cannot create bucket, err: " , err )
}
2020-04-09 18:30:02 +02:00
uploadID , err := obj . NewMultipartUpload ( GlobalContext , bucketName , objectName , ObjectOptions { UserDefined : map [ string ] string { "X-Amz-Meta-xid" : "3f" } } )
2016-09-16 22:06:49 +02:00
if err != nil {
t . Fatal ( "Unexpected error " , err )
}
2016-11-21 22:51:05 +01:00
md5Hex := getMD5Hash ( data )
2016-09-16 22:06:49 +02:00
2017-11-14 09:25:10 +01:00
parts := [ ] CompletePart { { PartNumber : 1 , ETag : md5Hex } }
2019-09-22 19:45:33 +02:00
newDisk := filepath . Join ( globalTestTmpDir , "minio-" + nextSuffix ( ) )
defer os . RemoveAll ( newDisk )
obj = initFSObjects ( newDisk , t )
2020-04-09 18:30:02 +02:00
if _ , err := obj . CompleteMultipartUpload ( GlobalContext , bucketName , objectName , uploadID , parts , ObjectOptions { } ) ; err != nil {
2018-04-10 18:36:37 +02:00
if ! isSameType ( err , BucketNotFound { } ) {
2017-01-17 02:05:00 +01:00
t . Fatal ( "Unexpected error " , err )
2016-09-16 22:06:49 +02:00
}
}
}
2018-01-31 22:17:24 +01:00
// TestCompleteMultipartUpload - test CompleteMultipartUpload
2017-10-18 23:26:20 +02:00
func TestCompleteMultipartUpload ( t * testing . T ) {
// Prepare for tests
disk := filepath . Join ( globalTestTmpDir , "minio-" + nextSuffix ( ) )
defer os . RemoveAll ( disk )
obj := initFSObjects ( disk , t )
bucketName := "bucket"
objectName := "object"
data := [ ] byte ( "12345" )
2020-05-08 22:44:44 +02:00
if err := obj . MakeBucketWithLocation ( GlobalContext , bucketName , "" , false ) ; err != nil {
2017-10-18 23:26:20 +02:00
t . Fatal ( "Cannot create bucket, err: " , err )
}
2020-04-09 18:30:02 +02:00
uploadID , err := obj . NewMultipartUpload ( GlobalContext , bucketName , objectName , ObjectOptions { UserDefined : map [ string ] string { "X-Amz-Meta-xid" : "3f" } } )
2017-10-18 23:26:20 +02:00
if err != nil {
t . Fatal ( "Unexpected error " , err )
}
md5Hex := getMD5Hash ( data )
2020-04-09 18:30:02 +02:00
if _ , err := obj . PutObjectPart ( GlobalContext , bucketName , objectName , uploadID , 1 , mustGetPutObjReader ( t , bytes . NewReader ( data ) , 5 , md5Hex , "" ) , ObjectOptions { } ) ; err != nil {
2017-10-18 23:26:20 +02:00
t . Fatal ( "Unexpected error " , err )
}
2017-11-14 09:25:10 +01:00
parts := [ ] CompletePart { { PartNumber : 1 , ETag : md5Hex } }
2020-04-09 18:30:02 +02:00
if _ , err := obj . CompleteMultipartUpload ( GlobalContext , bucketName , objectName , uploadID , parts , ObjectOptions { } ) ; err != nil {
2017-10-18 23:26:20 +02:00
t . Fatal ( "Unexpected error " , err )
}
}
2018-01-31 22:17:24 +01:00
// TestCompleteMultipartUpload - test CompleteMultipartUpload
2017-10-18 23:26:20 +02:00
func TestAbortMultipartUpload ( t * testing . T ) {
// Prepare for tests
disk := filepath . Join ( globalTestTmpDir , "minio-" + nextSuffix ( ) )
defer os . RemoveAll ( disk )
obj := initFSObjects ( disk , t )
bucketName := "bucket"
objectName := "object"
data := [ ] byte ( "12345" )
2020-05-08 22:44:44 +02:00
if err := obj . MakeBucketWithLocation ( GlobalContext , bucketName , "" , false ) ; err != nil {
2017-10-18 23:26:20 +02:00
t . Fatal ( "Cannot create bucket, err: " , err )
}
2020-04-09 18:30:02 +02:00
uploadID , err := obj . NewMultipartUpload ( GlobalContext , bucketName , objectName , ObjectOptions { UserDefined : map [ string ] string { "X-Amz-Meta-xid" : "3f" } } )
2017-10-18 23:26:20 +02:00
if err != nil {
t . Fatal ( "Unexpected error " , err )
}
md5Hex := getMD5Hash ( data )
2020-04-09 18:30:02 +02:00
if _ , err := obj . PutObjectPart ( GlobalContext , bucketName , objectName , uploadID , 1 , mustGetPutObjReader ( t , bytes . NewReader ( data ) , 5 , md5Hex , "" ) , ObjectOptions { } ) ; err != nil {
2017-10-18 23:26:20 +02:00
t . Fatal ( "Unexpected error " , err )
}
2018-01-31 22:17:24 +01:00
time . Sleep ( time . Second ) // Without Sleep on windows, the fs.AbortMultipartUpload() fails with "The process cannot access the file because it is being used by another process."
2020-04-09 18:30:02 +02:00
if err := obj . AbortMultipartUpload ( GlobalContext , bucketName , objectName , uploadID ) ; err != nil {
2017-10-18 23:26:20 +02:00
t . Fatal ( "Unexpected error " , err )
}
}
2016-09-16 22:06:49 +02:00
// TestListMultipartUploadsFaultyDisk - test ListMultipartUploads with faulty disks
func TestListMultipartUploadsFaultyDisk ( t * testing . T ) {
// Prepare for tests
2016-12-16 07:25:05 +01:00
disk := filepath . Join ( globalTestTmpDir , "minio-" + nextSuffix ( ) )
2017-08-13 04:25:43 +02:00
defer os . RemoveAll ( disk )
2017-01-17 02:05:00 +01:00
2016-10-05 21:48:07 +02:00
obj := initFSObjects ( disk , t )
2017-01-17 02:05:00 +01:00
2016-09-16 22:06:49 +02:00
bucketName := "bucket"
objectName := "object"
2020-05-08 22:44:44 +02:00
if err := obj . MakeBucketWithLocation ( GlobalContext , bucketName , "" , false ) ; err != nil {
2016-09-16 22:06:49 +02:00
t . Fatal ( "Cannot create bucket, err: " , err )
}
2020-04-09 18:30:02 +02:00
_ , err := obj . NewMultipartUpload ( GlobalContext , bucketName , objectName , ObjectOptions { UserDefined : map [ string ] string { "X-Amz-Meta-xid" : "3f" } } )
2016-09-16 22:06:49 +02:00
if err != nil {
t . Fatal ( "Unexpected error " , err )
}
2019-09-22 19:45:33 +02:00
newDisk := filepath . Join ( globalTestTmpDir , "minio-" + nextSuffix ( ) )
defer os . RemoveAll ( newDisk )
obj = initFSObjects ( newDisk , t )
2020-04-09 18:30:02 +02:00
if _ , err := obj . ListMultipartUploads ( GlobalContext , bucketName , objectName , "" , "" , "" , 1000 ) ; err != nil {
2018-04-10 18:36:37 +02:00
if ! isSameType ( err , BucketNotFound { } ) {
2017-01-17 02:05:00 +01:00
t . Fatal ( "Unexpected error " , err )
2016-09-16 22:06:49 +02:00
}
}
}