diff --git a/pkg/storage/donut/cache-multipart.go b/pkg/storage/donut/cache-multipart.go index 52eb4a035..e89b57ee6 100644 --- a/pkg/storage/donut/cache-multipart.go +++ b/pkg/storage/donut/cache-multipart.go @@ -217,23 +217,23 @@ func (cache Cache) cleanupMultiparts(bucket, key, uploadID string) { } // CompleteMultipartUpload - -func (cache Cache) CompleteMultipartUpload(bucket, key, uploadID string, parts map[int]string) (string, error) { +func (cache Cache) CompleteMultipartUpload(bucket, key, uploadID string, parts map[int]string) (ObjectMetadata, error) { if !IsValidBucket(bucket) { - return "", iodine.New(BucketNameInvalid{Bucket: bucket}, nil) + return ObjectMetadata{}, iodine.New(BucketNameInvalid{Bucket: bucket}, nil) } if !IsValidObjectName(key) { - return "", iodine.New(ObjectNameInvalid{Object: key}, nil) + return ObjectMetadata{}, iodine.New(ObjectNameInvalid{Object: key}, nil) } // Verify upload id cache.lock.RLock() if _, ok := cache.storedBuckets[bucket]; ok == false { cache.lock.RUnlock() - return "", iodine.New(BucketNotFound{Bucket: bucket}, nil) + return ObjectMetadata{}, iodine.New(BucketNotFound{Bucket: bucket}, nil) } storedBucket := cache.storedBuckets[bucket] if storedBucket.multiPartSession[key].uploadID != uploadID { cache.lock.RUnlock() - return "", iodine.New(InvalidUploadID{UploadID: uploadID}, nil) + return ObjectMetadata{}, iodine.New(InvalidUploadID{UploadID: uploadID}, nil) } cache.lock.RUnlock() @@ -245,21 +245,21 @@ func (cache Cache) CompleteMultipartUpload(bucket, key, uploadID string, parts m object, ok := cache.multiPartObjects.Get(bucket + "/" + getMultipartKey(key, uploadID, i)) if ok == false { cache.lock.Unlock() - return "", iodine.New(errors.New("missing part: "+strconv.Itoa(i)), nil) + return ObjectMetadata{}, iodine.New(errors.New("missing part: "+strconv.Itoa(i)), nil) } size += int64(len(object)) calcMD5Bytes := md5.Sum(object) // complete multi part request header md5sum per part is hex encoded recvMD5Bytes, err := hex.DecodeString(strings.Trim(recvMD5, "\"")) if err != nil { - return "", iodine.New(InvalidDigest{Md5: recvMD5}, nil) + return ObjectMetadata{}, iodine.New(InvalidDigest{Md5: recvMD5}, nil) } if !bytes.Equal(recvMD5Bytes, calcMD5Bytes[:]) { - return "", iodine.New(BadDigest{}, nil) + return ObjectMetadata{}, iodine.New(BadDigest{}, nil) } _, err = io.Copy(&fullObject, bytes.NewBuffer(object)) if err != nil { - return "", iodine.New(err, nil) + return ObjectMetadata{}, iodine.New(err, nil) } object = nil go debug.FreeOSMemory() @@ -269,16 +269,16 @@ func (cache Cache) CompleteMultipartUpload(bucket, key, uploadID string, parts m md5sumSlice := md5.Sum(fullObject.Bytes()) // this is needed for final verification inside CreateObject, do not convert this to hex md5sum := base64.StdEncoding.EncodeToString(md5sumSlice[:]) - etag, err := cache.CreateObject(bucket, key, "", md5sum, size, &fullObject) + objectMetadata, err := cache.CreateObject(bucket, key, "", md5sum, size, &fullObject) if err != nil { // No need to call internal cleanup functions here, caller will call AbortMultipartUpload() // which would in-turn cleanup properly in accordance with S3 Spec - return "", iodine.New(err, nil) + return ObjectMetadata{}, iodine.New(err, nil) } fullObject.Reset() cache.cleanupMultiparts(bucket, key, uploadID) cache.cleanupMultipartSession(bucket, key, uploadID) - return etag, nil + return objectMetadata, nil } // byKey is a sortable interface for UploadMetadata slice diff --git a/pkg/storage/donut/cache.go b/pkg/storage/donut/cache.go index 3c089cd5b..7e307c7bb 100644 --- a/pkg/storage/donut/cache.go +++ b/pkg/storage/donut/cache.go @@ -93,6 +93,9 @@ func NewCache(maxSize uint64, expiration time.Duration, donutName string, nodeDi c.multiPartObjects = trove.NewCache(0, time.Duration(0)) c.objects.OnExpired = c.expiredObject c.multiPartObjects.OnExpired = c.expiredPart + c.lock = new(sync.RWMutex) + c.maxSize = maxSize + c.expiration = expiration // set up cache expiration c.objects.ExpireObjects(time.Second * 5) @@ -262,42 +265,42 @@ func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) error { } // CreateObject - -func (cache Cache) CreateObject(bucket, key, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) { +func (cache Cache) CreateObject(bucket, key, contentType, expectedMD5Sum string, size int64, data io.Reader) (ObjectMetadata, error) { if size > int64(cache.maxSize) { generic := GenericObjectError{Bucket: bucket, Object: key} - return "", iodine.New(EntityTooLarge{ + return ObjectMetadata{}, iodine.New(EntityTooLarge{ GenericObjectError: generic, Size: strconv.FormatInt(size, 10), MaxSize: strconv.FormatUint(cache.maxSize, 10), }, nil) } - md5sum, err := cache.createObject(bucket, key, contentType, expectedMD5Sum, size, data) + objectMetadata, err := cache.createObject(bucket, key, contentType, expectedMD5Sum, size, data) // free debug.FreeOSMemory() - return md5sum, iodine.New(err, nil) + return objectMetadata, iodine.New(err, nil) } // createObject - PUT object to cache buffer -func (cache Cache) createObject(bucket, key, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) { +func (cache Cache) createObject(bucket, key, contentType, expectedMD5Sum string, size int64, data io.Reader) (ObjectMetadata, error) { cache.lock.RLock() if !IsValidBucket(bucket) { cache.lock.RUnlock() - return "", iodine.New(BucketNameInvalid{Bucket: bucket}, nil) + return ObjectMetadata{}, iodine.New(BucketNameInvalid{Bucket: bucket}, nil) } if !IsValidObjectName(key) { cache.lock.RUnlock() - return "", iodine.New(ObjectNameInvalid{Object: key}, nil) + return ObjectMetadata{}, iodine.New(ObjectNameInvalid{Object: key}, nil) } if _, ok := cache.storedBuckets[bucket]; ok == false { cache.lock.RUnlock() - return "", iodine.New(BucketNotFound{Bucket: bucket}, nil) + return ObjectMetadata{}, iodine.New(BucketNotFound{Bucket: bucket}, nil) } storedBucket := cache.storedBuckets[bucket] // get object key objectKey := bucket + "/" + key if _, ok := storedBucket.objectMetadata[objectKey]; ok == true { cache.lock.RUnlock() - return "", iodine.New(ObjectExists{Object: key}, nil) + return ObjectMetadata{}, iodine.New(ObjectExists{Object: key}, nil) } cache.lock.RUnlock() @@ -309,7 +312,7 @@ func (cache Cache) createObject(bucket, key, contentType, expectedMD5Sum string, expectedMD5SumBytes, err := base64.StdEncoding.DecodeString(strings.TrimSpace(expectedMD5Sum)) if err != nil { // pro-actively close the connection - return "", iodine.New(InvalidDigest{Md5: expectedMD5Sum}, nil) + return ObjectMetadata{}, iodine.New(InvalidDigest{Md5: expectedMD5Sum}, nil) } expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes) } @@ -332,7 +335,7 @@ func (cache Cache) createObject(bucket, key, contentType, expectedMD5Sum string, readBytes = append(readBytes, byteBuffer[0:length]...) } if err != io.EOF { - return "", iodine.New(err, nil) + return ObjectMetadata{}, iodine.New(err, nil) } md5SumBytes := hash.Sum(nil) totalLength := len(readBytes) @@ -344,14 +347,14 @@ func (cache Cache) createObject(bucket, key, contentType, expectedMD5Sum string, go debug.FreeOSMemory() cache.lock.Unlock() if !ok { - return "", iodine.New(InternalError{}, nil) + return ObjectMetadata{}, iodine.New(InternalError{}, nil) } md5Sum := hex.EncodeToString(md5SumBytes) // Verify if the written object is equal to what is expected, only if it is requested as such if strings.TrimSpace(expectedMD5Sum) != "" { if err := isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), md5Sum); err != nil { - return "", iodine.New(BadDigest{}, nil) + return ObjectMetadata{}, iodine.New(BadDigest{}, nil) } } @@ -371,11 +374,11 @@ func (cache Cache) createObject(bucket, key, contentType, expectedMD5Sum string, storedBucket.objectMetadata[objectKey] = newObject cache.storedBuckets[bucket] = storedBucket cache.lock.Unlock() - return newObject.MD5Sum, nil + return newObject, nil } -// CreateBucket - create bucket in cache -func (cache Cache) CreateBucket(bucketName, acl string) error { +// MakeBucket - create bucket in cache +func (cache Cache) MakeBucket(bucketName, acl string) error { cache.lock.RLock() if len(cache.storedBuckets) == totalBuckets { cache.lock.RUnlock() @@ -418,22 +421,21 @@ func (cache Cache) CreateBucket(bucketName, acl string) error { return nil } -func (cache Cache) filterDelimiterPrefix(keys []string, key, prefix, delim string) ([]string, []string) { - var commonPrefixes []string +func (cache Cache) filterDelimiterPrefix(keys []string, commonPrefixes []string, key, prefix, delim string) ([]string, []string) { switch true { case key == prefix: keys = append(keys, key) // delim - requires r.Prefix as it was trimmed off earlier case key == prefix+delim: keys = append(keys, key) + fallthrough case delim != "": commonPrefixes = append(commonPrefixes, prefix+delim) } - return RemoveDuplicates(keys), RemoveDuplicates(commonPrefixes) + return keys, commonPrefixes } -func (cache Cache) listObjects(keys []string, key string, r BucketResourcesMetadata) ([]string, []string) { - var commonPrefixes []string +func (cache Cache) listObjects(keys []string, commonPrefixes []string, key string, r BucketResourcesMetadata) ([]string, []string) { switch true { // Prefix absent, delimit object key based on delimiter case r.IsDelimiterSet(): @@ -449,7 +451,7 @@ func (cache Cache) listObjects(keys []string, key string, r BucketResourcesMetad if strings.HasPrefix(key, r.Prefix) { trimmedName := strings.TrimPrefix(key, r.Prefix) delim := Delimiter(trimmedName, r.Delimiter) - keys, commonPrefixes = cache.filterDelimiterPrefix(keys, key, r.Prefix, delim) + keys, commonPrefixes = cache.filterDelimiterPrefix(keys, commonPrefixes, key, r.Prefix, delim) } // Prefix present, nothing to delimit case r.IsPrefixSet(): @@ -458,7 +460,7 @@ func (cache Cache) listObjects(keys []string, key string, r BucketResourcesMetad case r.IsDefault(): keys = append(keys, key) } - return RemoveDuplicates(keys), RemoveDuplicates(commonPrefixes) + return keys, commonPrefixes } // ListObjects - list objects from cache @@ -468,7 +470,7 @@ func (cache Cache) ListObjects(bucket string, resources BucketResourcesMetadata) if !IsValidBucket(bucket) { return nil, BucketResourcesMetadata{IsTruncated: false}, iodine.New(BucketNameInvalid{Bucket: bucket}, nil) } - if !IsValidObjectName(resources.Prefix) { + if !IsValidPrefix(resources.Prefix) { return nil, BucketResourcesMetadata{IsTruncated: false}, iodine.New(ObjectNameInvalid{Object: resources.Prefix}, nil) } if _, ok := cache.storedBuckets[bucket]; ok == false { @@ -476,12 +478,11 @@ func (cache Cache) ListObjects(bucket string, resources BucketResourcesMetadata) } var results []ObjectMetadata var keys []string - var commonPrefixes []string storedBucket := cache.storedBuckets[bucket] for key := range storedBucket.objectMetadata { if strings.HasPrefix(key, bucket+"/") { key = key[len(bucket)+1:] - keys, commonPrefixes = cache.listObjects(keys, key, resources) + keys, resources.CommonPrefixes = cache.listObjects(keys, resources.CommonPrefixes, key, resources) } } var newKeys []string @@ -508,7 +509,7 @@ func (cache Cache) ListObjects(bucket string, resources BucketResourcesMetadata) object := storedBucket.objectMetadata[bucket+"/"+key] results = append(results, object) } - resources.CommonPrefixes = commonPrefixes + resources.CommonPrefixes = RemoveDuplicates(resources.CommonPrefixes) return results, resources, nil } diff --git a/pkg/storage/donut/donut_test.go b/pkg/storage/donut/donut_test.go index fad0a26fa..6b7c949bd 100644 --- a/pkg/storage/donut/donut_test.go +++ b/pkg/storage/donut/donut_test.go @@ -19,20 +19,23 @@ package donut import ( "bytes" "crypto/md5" + "encoding/base64" "encoding/hex" - "io" "io/ioutil" "os" "path/filepath" "strconv" "testing" + "time" . "github.com/minio/check" ) func Test(t *testing.T) { TestingT(t) } -type MySuite struct{} +type MySuite struct { + root string +} var _ = Suite(&MySuite{}) @@ -52,293 +55,224 @@ func createTestNodeDiskMap(p string) map[string][]string { return nodes } -// test empty donut -func (s *MySuite) TestEmptyDonut(c *C) { +var d Cache + +func (s *MySuite) SetUpSuite(c *C) { root, err := ioutil.TempDir(os.TempDir(), "donut-") c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut("test", createTestNodeDiskMap(root)) + s.root = root + d = NewCache(100000, time.Duration(1*time.Hour), "test", createTestNodeDiskMap(root)) + buckets, err := d.ListBuckets() c.Assert(err, IsNil) + c.Assert(len(buckets), Equals, 0) +} - // check donut is empty - metadata, err := donut.ListBuckets() - c.Assert(err, IsNil) - c.Assert(len(metadata), Equals, 0) +func (s *MySuite) TearDownSuite(c *C) { + os.RemoveAll(s.root) } // test make bucket without name func (s *MySuite) TestBucketWithoutNameFails(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut("test", createTestNodeDiskMap(root)) - c.Assert(err, IsNil) // fail to create new bucket without a name - err = donut.MakeBucket("", "private") + err := d.MakeBucket("", "private") c.Assert(err, Not(IsNil)) - err = donut.MakeBucket(" ", "private") + err = d.MakeBucket(" ", "private") c.Assert(err, Not(IsNil)) } // test empty bucket func (s *MySuite) TestEmptyBucket(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut("test", createTestNodeDiskMap(root)) - c.Assert(err, IsNil) - - c.Assert(donut.MakeBucket("foo", BucketACL("private")), IsNil) + c.Assert(d.MakeBucket("foo1", "private"), IsNil) // check if bucket is empty - listObjects, err := donut.ListObjects("foo", "", "", "", 1) + var resources BucketResourcesMetadata + resources.Maxkeys = 1 + objectsMetadata, resources, err := d.ListObjects("foo1", resources) c.Assert(err, IsNil) - c.Assert(len(listObjects.Objects), Equals, 0) - c.Assert(listObjects.CommonPrefixes, DeepEquals, []string{}) - c.Assert(listObjects.IsTruncated, Equals, false) + c.Assert(len(objectsMetadata), Equals, 0) + c.Assert(resources.CommonPrefixes, DeepEquals, []string{}) + c.Assert(resources.IsTruncated, Equals, false) } // test bucket list func (s *MySuite) TestMakeBucketAndList(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut("test", createTestNodeDiskMap(root)) - c.Assert(err, IsNil) // create bucket - err = donut.MakeBucket("foo", BucketACL("private")) + err := d.MakeBucket("foo2", "private") c.Assert(err, IsNil) // check bucket exists - buckets, err := donut.ListBuckets() + buckets, err := d.ListBuckets() c.Assert(err, IsNil) - c.Assert(len(buckets), Equals, 1) - c.Assert(buckets["foo"].ACL, Equals, BucketACL("private")) + c.Assert(len(buckets), Equals, 5) + c.Assert(buckets[0].ACL, Equals, BucketACL("private")) } // test re-create bucket func (s *MySuite) TestMakeBucketWithSameNameFails(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut("test", createTestNodeDiskMap(root)) - c.Assert(err, IsNil) - err = donut.MakeBucket("foo", BucketACL("private")) + err := d.MakeBucket("foo3", "private") c.Assert(err, IsNil) - err = donut.MakeBucket("foo", BucketACL("private")) + err = d.MakeBucket("foo3", "private") c.Assert(err, Not(IsNil)) } // test make multiple buckets func (s *MySuite) TestCreateMultipleBucketsAndList(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut("test", createTestNodeDiskMap(root)) - c.Assert(err, IsNil) // add a second bucket - err = donut.MakeBucket("foo", BucketACL("private")) + err := d.MakeBucket("foo4", "private") c.Assert(err, IsNil) - err = donut.MakeBucket("bar", BucketACL("private")) + err = d.MakeBucket("bar1", "private") c.Assert(err, IsNil) - buckets, err := donut.ListBuckets() + buckets, err := d.ListBuckets() c.Assert(err, IsNil) - _, ok := buckets["foo"] - c.Assert(ok, Equals, true) - _, ok = buckets["bar"] - c.Assert(ok, Equals, true) + c.Assert(len(buckets), Equals, 2) + c.Assert(buckets[0].Name, Equals, "bar1") + c.Assert(buckets[1].Name, Equals, "foo4") - err = donut.MakeBucket("foobar", BucketACL("private")) + err = d.MakeBucket("foobar1", "private") c.Assert(err, IsNil) - buckets, err = donut.ListBuckets() + buckets, err = d.ListBuckets() c.Assert(err, IsNil) - _, ok = buckets["foobar"] - c.Assert(ok, Equals, true) + + c.Assert(len(buckets), Equals, 3) + c.Assert(buckets[2].Name, Equals, "foobar1") } // test object create without bucket func (s *MySuite) TestNewObjectFailsWithoutBucket(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut("test", createTestNodeDiskMap(root)) - c.Assert(err, IsNil) - _, err = donut.PutObject("foo", "obj", "", nil, nil) + _, err := d.CreateObject("unknown", "obj", "", "", 0, nil) c.Assert(err, Not(IsNil)) } // test create object metadata func (s *MySuite) TestNewObjectMetadata(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut("test", createTestNodeDiskMap(root)) - c.Assert(err, IsNil) - - metadata := make(map[string]string) - metadata["contentType"] = "application/json" - metadata["foo"] = "value1" - metadata["hello"] = "world" - data := "Hello World" hasher := md5.New() hasher.Write([]byte(data)) - expectedMd5Sum := hex.EncodeToString(hasher.Sum(nil)) + expectedMd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) reader := ioutil.NopCloser(bytes.NewReader([]byte(data))) - metadata["contentLength"] = strconv.Itoa(len(data)) - err = donut.MakeBucket("foo", "private") + err := d.MakeBucket("foo6", "private") c.Assert(err, IsNil) - objectMetadata, err := donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata) + objectMetadata, err := d.CreateObject("foo6", "obj", "application/json", expectedMd5Sum, int64(len(data)), reader) c.Assert(err, IsNil) - c.Assert(objectMetadata.MD5Sum, Equals, expectedMd5Sum) - c.Assert(objectMetadata.Metadata["contentType"], Equals, metadata["contentType"]) - c.Assert(objectMetadata.Metadata["foo"], Equals, metadata["foo"]) - c.Assert(objectMetadata.Metadata["hello"], Equals, metadata["hello"]) + c.Assert(objectMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil))) + c.Assert(objectMetadata.Metadata["contentType"], Equals, "application/json") } // test create object fails without name func (s *MySuite) TestNewObjectFailsWithEmptyName(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut("test", createTestNodeDiskMap(root)) - c.Assert(err, IsNil) - - _, err = donut.PutObject("foo", "", "", nil, nil) + _, err := d.CreateObject("foo", "", "", "", 0, nil) c.Assert(err, Not(IsNil)) } // test create object func (s *MySuite) TestNewObjectCanBeWritten(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut("test", createTestNodeDiskMap(root)) + err := d.MakeBucket("foo", "private") c.Assert(err, IsNil) - err = donut.MakeBucket("foo", "private") - c.Assert(err, IsNil) - - metadata := make(map[string]string) - metadata["contentType"] = "application/octet-stream" data := "Hello World" hasher := md5.New() hasher.Write([]byte(data)) - expectedMd5Sum := hex.EncodeToString(hasher.Sum(nil)) + expectedMd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) reader := ioutil.NopCloser(bytes.NewReader([]byte(data))) - metadata["contentLength"] = strconv.Itoa(len(data)) - actualMetadata, err := donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata) + actualMetadata, err := d.CreateObject("foo", "obj", "application/octet-stream", expectedMd5Sum, int64(len(data)), reader) c.Assert(err, IsNil) - c.Assert(actualMetadata.MD5Sum, Equals, expectedMd5Sum) + c.Assert(actualMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil))) - reader, size, err := donut.GetObject("foo", "obj") + var buffer bytes.Buffer + size, err := d.GetObject(&buffer, "foo", "obj") c.Assert(err, IsNil) c.Assert(size, Equals, int64(len(data))) + c.Assert(buffer.Bytes(), DeepEquals, []byte(data)) - var actualData bytes.Buffer - _, err = io.Copy(&actualData, reader) + actualMetadata, err = d.GetObjectMetadata("foo", "obj") c.Assert(err, IsNil) - c.Assert(actualData.Bytes(), DeepEquals, []byte(data)) - - actualMetadata, err = donut.GetObjectMetadata("foo", "obj") - c.Assert(err, IsNil) - c.Assert(expectedMd5Sum, Equals, actualMetadata.MD5Sum) + c.Assert(hex.EncodeToString(hasher.Sum(nil)), Equals, actualMetadata.MD5Sum) c.Assert(int64(len(data)), Equals, actualMetadata.Size) - c.Assert("1.0.0", Equals, actualMetadata.Version) } // test list objects func (s *MySuite) TestMultipleNewObjects(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut("test", createTestNodeDiskMap(root)) - c.Assert(err, IsNil) - - c.Assert(donut.MakeBucket("foo", BucketACL("private")), IsNil) + c.Assert(d.MakeBucket("foo5", "private"), IsNil) one := ioutil.NopCloser(bytes.NewReader([]byte("one"))) - metadata := make(map[string]string) - metadata["contentLength"] = strconv.Itoa(len("one")) - _, err = donut.PutObject("foo", "obj1", "", one, metadata) + _, err := d.CreateObject("foo5", "obj1", "", "", int64(len("one")), one) c.Assert(err, IsNil) two := ioutil.NopCloser(bytes.NewReader([]byte("two"))) - - metadata["contentLength"] = strconv.Itoa(len("two")) - _, err = donut.PutObject("foo", "obj2", "", two, metadata) + _, err = d.CreateObject("foo5", "obj2", "", "", int64(len("two")), two) c.Assert(err, IsNil) - obj1, size, err := donut.GetObject("foo", "obj1") + var buffer1 bytes.Buffer + size, err := d.GetObject(&buffer1, "foo5", "obj1") c.Assert(err, IsNil) c.Assert(size, Equals, int64(len([]byte("one")))) + c.Assert(buffer1.Bytes(), DeepEquals, []byte("one")) - var readerBuffer1 bytes.Buffer - _, err = io.CopyN(&readerBuffer1, obj1, size) - c.Assert(err, IsNil) - c.Assert(readerBuffer1.Bytes(), DeepEquals, []byte("one")) - - obj2, size, err := donut.GetObject("foo", "obj2") + var buffer2 bytes.Buffer + size, err = d.GetObject(&buffer2, "foo5", "obj2") c.Assert(err, IsNil) c.Assert(size, Equals, int64(len([]byte("two")))) - var readerBuffer2 bytes.Buffer - _, err = io.CopyN(&readerBuffer2, obj2, size) - c.Assert(err, IsNil) - c.Assert(readerBuffer2.Bytes(), DeepEquals, []byte("two")) + c.Assert(buffer2.Bytes(), DeepEquals, []byte("two")) /// test list of objects // test list objects with prefix and delimiter - listObjects, err := donut.ListObjects("foo", "o", "", "1", 10) + var resources BucketResourcesMetadata + resources.Prefix = "o" + resources.Delimiter = "1" + resources.Maxkeys = 10 + objectsMetadata, resources, err := d.ListObjects("foo5", resources) c.Assert(err, IsNil) - c.Assert(listObjects.IsTruncated, Equals, false) - c.Assert(listObjects.CommonPrefixes[0], Equals, "obj1") + c.Assert(resources.IsTruncated, Equals, false) + c.Assert(resources.CommonPrefixes[0], Equals, "obj1") // test list objects with only delimiter - listObjects, err = donut.ListObjects("foo", "", "", "1", 10) + resources.Prefix = "" + resources.Delimiter = "1" + resources.Maxkeys = 10 + objectsMetadata, resources, err = d.ListObjects("foo5", resources) c.Assert(err, IsNil) - _, ok := listObjects.Objects["obj2"] - c.Assert(ok, Equals, true) - c.Assert(listObjects.IsTruncated, Equals, false) - c.Assert(listObjects.CommonPrefixes[0], Equals, "obj1") + c.Assert(objectsMetadata[0].Object, Equals, "obj1") + c.Assert(resources.IsTruncated, Equals, false) + c.Assert(resources.CommonPrefixes[0], Equals, "obj1") // test list objects with only prefix - listObjects, err = donut.ListObjects("foo", "o", "", "", 10) + resources.Prefix = "o" + resources.Delimiter = "" + resources.Maxkeys = 10 + objectsMetadata, resources, err = d.ListObjects("foo5", resources) c.Assert(err, IsNil) - c.Assert(listObjects.IsTruncated, Equals, false) - _, ok1 := listObjects.Objects["obj1"] - _, ok2 := listObjects.Objects["obj2"] - c.Assert(ok1, Equals, true) - c.Assert(ok2, Equals, true) + c.Assert(resources.IsTruncated, Equals, false) + c.Assert(objectsMetadata[0].Object, Equals, "obj1") + c.Assert(objectsMetadata[1].Object, Equals, "obj2") three := ioutil.NopCloser(bytes.NewReader([]byte("three"))) - metadata["contentLength"] = strconv.Itoa(len("three")) - _, err = donut.PutObject("foo", "obj3", "", three, metadata) + _, err = d.CreateObject("foo5", "obj3", "", "", int64(len("three")), three) c.Assert(err, IsNil) - obj3, size, err := donut.GetObject("foo", "obj3") + var buffer bytes.Buffer + size, err = d.GetObject(&buffer, "foo5", "obj3") c.Assert(err, IsNil) c.Assert(size, Equals, int64(len([]byte("three")))) - - var readerBuffer3 bytes.Buffer - _, err = io.CopyN(&readerBuffer3, obj3, size) - c.Assert(err, IsNil) - c.Assert(readerBuffer3.Bytes(), DeepEquals, []byte("three")) + c.Assert(buffer.Bytes(), DeepEquals, []byte("three")) // test list objects with maxkeys - listObjects, err = donut.ListObjects("foo", "o", "", "", 2) + resources.Prefix = "o" + resources.Delimiter = "" + resources.Maxkeys = 2 + objectsMetadata, resources, err = d.ListObjects("foo5", resources) c.Assert(err, IsNil) - c.Assert(listObjects.IsTruncated, Equals, true) - c.Assert(len(listObjects.Objects), Equals, 2) + c.Assert(resources.IsTruncated, Equals, true) + c.Assert(len(objectsMetadata), Equals, 2) } diff --git a/pkg/storage/donut/utils.go b/pkg/storage/donut/utils.go index ed2ce6604..5966b1bba 100644 --- a/pkg/storage/donut/utils.go +++ b/pkg/storage/donut/utils.go @@ -181,7 +181,7 @@ func IsValidBucket(bucket string) bool { // - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html func IsValidObjectName(object string) bool { if strings.TrimSpace(object) == "" { - return true + return false } if len(object) > 1024 || len(object) == 0 { return false @@ -191,3 +191,11 @@ func IsValidObjectName(object string) bool { } return true } + +// IsValidPrefix - verify prefix name is correct, an empty prefix is valid +func IsValidPrefix(prefix string) bool { + if strings.TrimSpace(prefix) == "" { + return true + } + return IsValidObjectName(prefix) +}