/* * Minimalist Object Storage, (C) 2015, 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package fs import ( "bytes" "crypto/md5" "encoding/hex" "encoding/xml" "math/rand" "strconv" "gopkg.in/check.v1" ) // APITestSuite - collection of API tests func APITestSuite(c *check.C, create func() Filesystem) { testMakeBucket(c, create) testMultipleObjectCreation(c, create) testPaging(c, create) testObjectOverwriteWorks(c, create) testNonExistantBucketOperations(c, create) testBucketRecreateFails(c, create) testPutObjectInSubdir(c, create) testListBuckets(c, create) testListBucketsOrder(c, create) testListObjectsTestsForNonExistantBucket(c, create) testNonExistantObjectInBucket(c, create) testGetDirectoryReturnsObjectNotFound(c, create) testDefaultContentType(c, create) testMultipartObjectCreation(c, create) testMultipartObjectAbort(c, create) } func testMakeBucket(c *check.C, create func() Filesystem) { fs := create() err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) } func testMultipartObjectCreation(c *check.C, create func() Filesystem) { fs := create() err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) uploadID, err := fs.NewMultipartUpload("bucket", "key") c.Assert(err, check.IsNil) completedParts := CompleteMultipartUpload{} //completedParts.Part = make([]CompletePart, 10) for i := 1; i <= 10; i++ { randomPerm := rand.Perm(10) randomString := "" for _, num := range randomPerm { randomString = randomString + strconv.Itoa(num) } hasher := md5.New() hasher.Write([]byte(randomString)) expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil)) var calculatedMD5sum string calculatedMD5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), hasher.Sum(nil)) c.Assert(err, check.IsNil) c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex) completedParts.Part = append(completedParts.Part, CompletePart{PartNumber: i, ETag: calculatedMD5sum}) } completedPartsBytes, e := xml.Marshal(completedParts) c.Assert(e, check.IsNil) objectInfo, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, completedPartsBytes) c.Assert(err, check.IsNil) c.Assert(objectInfo.MD5Sum, check.Equals, "9b7d6f13ba00e24d0b02de92e814891b-10") } func testMultipartObjectAbort(c *check.C, create func() Filesystem) { fs := create() err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) uploadID, err := fs.NewMultipartUpload("bucket", "key") c.Assert(err, check.IsNil) parts := make(map[int]string) for i := 1; i <= 10; i++ { randomPerm := rand.Perm(10) randomString := "" for _, num := range randomPerm { randomString = randomString + strconv.Itoa(num) } hasher := md5.New() hasher.Write([]byte(randomString)) expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil)) var calculatedMD5sum string calculatedMD5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), hasher.Sum(nil)) c.Assert(err, check.IsNil) c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex) parts[i] = expectedMD5Sumhex } err = fs.AbortMultipartUpload("bucket", "key", uploadID) c.Assert(err, check.IsNil) } func testMultipleObjectCreation(c *check.C, create func() Filesystem) { objects := make(map[string][]byte) fs := create() err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) for i := 0; i < 10; i++ { randomPerm := rand.Perm(10) randomString := "" for _, num := range randomPerm { randomString = randomString + strconv.Itoa(num) } hasher := md5.New() hasher.Write([]byte(randomString)) expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil)) key := "obj" + strconv.Itoa(i) objects[key] = []byte(randomString) var objectInfo ObjectInfo objectInfo, err = fs.CreateObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), hasher.Sum(nil)) c.Assert(err, check.IsNil) c.Assert(objectInfo.MD5Sum, check.Equals, expectedMD5Sumhex) } for key, value := range objects { var byteBuffer bytes.Buffer _, err := fs.GetObject(&byteBuffer, "bucket", key, 0, 0) c.Assert(err, check.IsNil) c.Assert(byteBuffer.Bytes(), check.DeepEquals, value) metadata, err := fs.GetObjectInfo("bucket", key) c.Assert(err, check.IsNil) c.Assert(metadata.Size, check.Equals, int64(len(value))) } } func testPaging(c *check.C, create func() Filesystem) { fs := create() fs.MakeBucket("bucket") result, err := fs.ListObjects("bucket", "", "", "", 0) c.Assert(err, check.IsNil) c.Assert(len(result.Objects), check.Equals, 0) c.Assert(result.IsTruncated, check.Equals, false) // check before paging occurs for i := 0; i < 5; i++ { key := "obj" + strconv.Itoa(i) _, err = fs.CreateObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) c.Assert(err, check.IsNil) result, err = fs.ListObjects("bucket", "", "", "", 5) c.Assert(err, check.IsNil) c.Assert(len(result.Objects), check.Equals, i+1) c.Assert(result.IsTruncated, check.Equals, false) } // check after paging occurs pages work for i := 6; i <= 10; i++ { key := "obj" + strconv.Itoa(i) _, err = fs.CreateObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) c.Assert(err, check.IsNil) result, err = fs.ListObjects("bucket", "obj", "", "", 5) c.Assert(err, check.IsNil) c.Assert(len(result.Objects), check.Equals, 5) c.Assert(result.IsTruncated, check.Equals, true) } // check paging with prefix at end returns less objects { _, err = fs.CreateObject("bucket", "newPrefix", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil) c.Assert(err, check.IsNil) _, err = fs.CreateObject("bucket", "newPrefix2", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) c.Assert(err, check.IsNil) result, err = fs.ListObjects("bucket", "new", "", "", 5) c.Assert(err, check.IsNil) c.Assert(len(result.Objects), check.Equals, 2) } // check ordering of pages { result, err = fs.ListObjects("bucket", "", "", "", 1000) c.Assert(err, check.IsNil) c.Assert(result.Objects[0].Name, check.Equals, "newPrefix") c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2") c.Assert(result.Objects[2].Name, check.Equals, "obj0") c.Assert(result.Objects[3].Name, check.Equals, "obj1") c.Assert(result.Objects[4].Name, check.Equals, "obj10") } // check delimited results with delimiter and prefix { _, err = fs.CreateObject("bucket", "this/is/delimited", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil) c.Assert(err, check.IsNil) _, err = fs.CreateObject("bucket", "this/is/also/a/delimited/file", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) c.Assert(err, check.IsNil) result, err = fs.ListObjects("bucket", "this/is/", "", "/", 10) c.Assert(err, check.IsNil) c.Assert(len(result.Objects), check.Equals, 1) c.Assert(result.Prefixes[0], check.Equals, "this/is/also/") } // check delimited results with delimiter without prefix { result, err = fs.ListObjects("bucket", "", "", "/", 1000) c.Assert(err, check.IsNil) c.Assert(result.Objects[0].Name, check.Equals, "newPrefix") c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2") c.Assert(result.Objects[2].Name, check.Equals, "obj0") c.Assert(result.Objects[3].Name, check.Equals, "obj1") c.Assert(result.Objects[4].Name, check.Equals, "obj10") c.Assert(result.Prefixes[0], check.Equals, "this/") } // check results with Marker { result, err = fs.ListObjects("bucket", "", "newPrefix", "", 3) c.Assert(err, check.IsNil) c.Assert(result.Objects[0].Name, check.Equals, "newPrefix2") c.Assert(result.Objects[1].Name, check.Equals, "obj0") c.Assert(result.Objects[2].Name, check.Equals, "obj1") } // check ordering of results with prefix { result, err = fs.ListObjects("bucket", "obj", "", "", 1000) c.Assert(err, check.IsNil) c.Assert(result.Objects[0].Name, check.Equals, "obj0") c.Assert(result.Objects[1].Name, check.Equals, "obj1") c.Assert(result.Objects[2].Name, check.Equals, "obj10") c.Assert(result.Objects[3].Name, check.Equals, "obj2") c.Assert(result.Objects[4].Name, check.Equals, "obj3") } // check ordering of results with prefix and no paging { result, err = fs.ListObjects("bucket", "new", "", "", 5) c.Assert(err, check.IsNil) c.Assert(result.Objects[0].Name, check.Equals, "newPrefix") c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2") } } func testObjectOverwriteWorks(c *check.C, create func() Filesystem) { fs := create() err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) _, err = fs.CreateObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil) c.Assert(err, check.IsNil) // c.Assert(md5Sum1hex, check.Equals, objectInfo.MD5Sum) _, err = fs.CreateObject("bucket", "object", int64(len("three")), bytes.NewBufferString("three"), nil) c.Assert(err, check.IsNil) var bytesBuffer bytes.Buffer length, err := fs.GetObject(&bytesBuffer, "bucket", "object", 0, 0) c.Assert(err, check.IsNil) c.Assert(length, check.Equals, int64(len("three"))) c.Assert(string(bytesBuffer.Bytes()), check.Equals, "three") } func testNonExistantBucketOperations(c *check.C, create func() Filesystem) { fs := create() _, err := fs.CreateObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil) c.Assert(err, check.Not(check.IsNil)) } func testBucketRecreateFails(c *check.C, create func() Filesystem) { fs := create() err := fs.MakeBucket("string") c.Assert(err, check.IsNil) err = fs.MakeBucket("string") c.Assert(err, check.Not(check.IsNil)) } func testPutObjectInSubdir(c *check.C, create func() Filesystem) { fs := create() err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) _, err = fs.CreateObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil) c.Assert(err, check.IsNil) var bytesBuffer bytes.Buffer length, err := fs.GetObject(&bytesBuffer, "bucket", "dir1/dir2/object", 0, 0) c.Assert(err, check.IsNil) c.Assert(len(bytesBuffer.Bytes()), check.Equals, len("hello world")) c.Assert(int64(len(bytesBuffer.Bytes())), check.Equals, length) } func testListBuckets(c *check.C, create func() Filesystem) { fs := create() // test empty list buckets, err := fs.ListBuckets() c.Assert(err, check.IsNil) c.Assert(len(buckets), check.Equals, 0) // add one and test exists err = fs.MakeBucket("bucket1") c.Assert(err, check.IsNil) buckets, err = fs.ListBuckets() c.Assert(len(buckets), check.Equals, 1) c.Assert(err, check.IsNil) // add two and test exists err = fs.MakeBucket("bucket2") c.Assert(err, check.IsNil) buckets, err = fs.ListBuckets() c.Assert(len(buckets), check.Equals, 2) c.Assert(err, check.IsNil) // add three and test exists + prefix err = fs.MakeBucket("bucket22") buckets, err = fs.ListBuckets() c.Assert(len(buckets), check.Equals, 3) c.Assert(err, check.IsNil) } func testListBucketsOrder(c *check.C, create func() Filesystem) { // if implementation contains a map, order of map keys will vary. // this ensures they return in the same order each time for i := 0; i < 10; i++ { fs := create() // add one and test exists err := fs.MakeBucket("bucket1") c.Assert(err, check.IsNil) err = fs.MakeBucket("bucket2") c.Assert(err, check.IsNil) buckets, err := fs.ListBuckets() c.Assert(err, check.IsNil) c.Assert(len(buckets), check.Equals, 2) c.Assert(buckets[0].Name, check.Equals, "bucket1") c.Assert(buckets[1].Name, check.Equals, "bucket2") } } func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Filesystem) { fs := create() result, err := fs.ListObjects("bucket", "", "", "", 1000) c.Assert(err, check.Not(check.IsNil)) c.Assert(result.IsTruncated, check.Equals, false) c.Assert(len(result.Objects), check.Equals, 0) } func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) { fs := create() err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) var byteBuffer bytes.Buffer length, err := fs.GetObject(&byteBuffer, "bucket", "dir1", 0, 0) c.Assert(length, check.Equals, int64(0)) c.Assert(err, check.Not(check.IsNil)) c.Assert(len(byteBuffer.Bytes()), check.Equals, 0) switch err := err.ToGoError().(type) { case ObjectNotFound: c.Assert(err, check.ErrorMatches, "Object not found: bucket#dir1") default: c.Assert(err, check.Equals, "fails") } } func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem) { fs := create() err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) _, err = fs.CreateObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil) c.Assert(err, check.IsNil) var byteBuffer bytes.Buffer length, err := fs.GetObject(&byteBuffer, "bucket", "dir1", 0, 0) c.Assert(length, check.Equals, int64(0)) switch err := err.ToGoError().(type) { case ObjectNotFound: c.Assert(err.Bucket, check.Equals, "bucket") c.Assert(err.Object, check.Equals, "dir1") default: // force a failure with a line number c.Assert(err, check.Equals, "ObjectNotFound") } c.Assert(len(byteBuffer.Bytes()), check.Equals, 0) var byteBuffer2 bytes.Buffer length, err = fs.GetObject(&byteBuffer, "bucket", "dir1/", 0, 0) c.Assert(length, check.Equals, int64(0)) switch err := err.ToGoError().(type) { case ObjectNotFound: c.Assert(err.Bucket, check.Equals, "bucket") c.Assert(err.Object, check.Equals, "dir1/") default: // force a failure with a line number c.Assert(err, check.Equals, "ObjectNotFound") } c.Assert(len(byteBuffer2.Bytes()), check.Equals, 0) } func testDefaultContentType(c *check.C, create func() Filesystem) { fs := create() err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) // Test empty _, err = fs.CreateObject("bucket", "one", int64(len("one")), bytes.NewBufferString("one"), nil) metadata, err := fs.GetObjectInfo("bucket", "one") c.Assert(err, check.IsNil) c.Assert(metadata.ContentType, check.Equals, "application/octet-stream") }