maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Changes to CreateObject() now returns back md5 along with any error

Browse source 
- This change is necessary to avoid the racy calls to GetObjectMetadata()
- This change is also necessary since one has to reply back md5sum with
  PUT object response header
This commit is contained in:
Harshavardhana 2015-04-30 03:38:11 -07:00
parent 13cae94191
commit d815e6adfd
12 changed files with 90 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
pkg/api/api_object_handlers.go
View file

@ -171,29 +171,12 @@ func (server *minioAPI) putObjectHandler(w http.ResponseWriter, req *http.Reques
writeErrorResponse(w, req, EntityTooSmall, acceptsContentType, req.URL.Path)
return
}
err := server.driver.CreateObject(bucket, object, "", md5, req.Body)
calculatedMD5, err := server.driver.CreateObject(bucket, object, "", md5, req.Body)
switch err := iodine.ToError(err).(type) {
case nil:
{
metadata, err := server.driver.GetObjectMetadata(bucket, object, "")
switch err := iodine.ToError(err).(type) {
case nil:
w.Header().Set("ETag", metadata.Md5)
writeSuccessResponse(w, acceptsContentType)
case drivers.ObjectNotFound:
{
writeErrorResponse(w, req, NoSuchKey, acceptsContentType, req.URL.Path)
}
case drivers.ObjectNameInvalid:
{
writeErrorResponse(w, req, NoSuchKey, acceptsContentType, req.URL.Path)
}
default:
{
log.Error.Println(iodine.New(err, nil))
writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path)
}
}
w.Header().Set("ETag", calculatedMD5)
writeSuccessResponse(w, acceptsContentType)
}
case drivers.ObjectExists:

37
pkg/api/api_test.go
View file

@ -167,8 +167,7 @@ func (s *MySuite) TestEmptyObject(c *C) {
Size: 0,
}
typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once()
typedDriver.On("CreateObject", "bucket", "object", "", "", mock.Anything).Return(nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "object", "").Return(metadata, nil).Once()
typedDriver.On("CreateObject", "bucket", "object", "", "", mock.Anything).Return(metadata.Md5, nil).Once()
typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Twice()
typedDriver.On("GetObjectMetadata", "bucket", "object", "").Return(metadata, nil).Once()
typedDriver.On("GetObject", mock.Anything, "bucket", "object").Return(int64(0), nil).Once()
@ -180,7 +179,6 @@ func (s *MySuite) TestEmptyObject(c *C) {
buffer := bytes.NewBufferString("")
driver.CreateBucket("bucket", "private")
driver.CreateObject("bucket", "object", "", "", buffer)
driver.GetObjectMetadata("bucket", "object", "")
request, err := http.NewRequest("GET", testServer.URL+"/bucket/object", nil)
c.Assert(err, IsNil)
@ -251,8 +249,7 @@ func (s *MySuite) TestObject(c *C) {
Size: 11,
}
typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once()
typedDriver.On("CreateObject", "bucket", "object", "", "", mock.Anything).Return(nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "object", "").Return(metadata, nil).Twice()
typedDriver.On("CreateObject", "bucket", "object", "", "", mock.Anything).Return(metadata.Md5, nil).Once()
typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Twice()
typedDriver.On("GetObjectMetadata", "bucket", "object", "").Return(metadata, nil).Twice()
typedDriver.SetGetObjectWriter("bucket", "object", []byte("hello world"))
@ -265,7 +262,6 @@ func (s *MySuite) TestObject(c *C) {
buffer := bytes.NewBufferString("hello world")
driver.CreateBucket("bucket", "private")
driver.CreateObject("bucket", "object", "", "", buffer)
driver.GetObjectMetadata("bucket", "object", "")
request, err := http.NewRequest("GET", testServer.URL+"/bucket/object", nil)
c.Assert(err, IsNil)
@ -328,18 +324,12 @@ func (s *MySuite) TestMultipleObjects(c *C) {
typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once()
driver.CreateBucket("bucket", "private")
typedDriver.On("CreateObject", "bucket", "object1", "", "", mock.Anything).Return(nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "object1", "").Return(metadata1, nil).Once()
typedDriver.On("CreateObject", "bucket", "object1", "", "", mock.Anything).Return(metadata1.Md5, nil).Once()
driver.CreateObject("bucket", "object1", "", "", buffer1)
driver.GetObjectMetadata("bucket", "object1", "")
typedDriver.On("CreateObject", "bucket", "object2", "", "", mock.Anything).Return(nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "object2", "").Return(metadata2, nil).Once()
typedDriver.On("CreateObject", "bucket", "object2", "", "", mock.Anything).Return(metadata2.Md5, nil).Once()
driver.CreateObject("bucket", "object2", "", "", buffer2)
driver.GetObjectMetadata("bucket", "object2", "")
typedDriver.On("CreateObject", "bucket", "object3", "", "", mock.Anything).Return(nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "object3", "").Return(metadata3, nil).Once()
typedDriver.On("CreateObject", "bucket", "object3", "", "", mock.Anything).Return(metadata3.Md5, nil).Once()
driver.CreateObject("bucket", "object3", "", "", buffer3)
driver.GetObjectMetadata("bucket", "object3", "")
// test non-existant object
typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Once()
@ -515,10 +505,8 @@ func (s *MySuite) TestHeader(c *C) {
buffer := bytes.NewBufferString("hello world")
typedDriver.On("GetBucketMetadata", "foo").Return(bucketMetadata, nil).Once()
typedDriver.On("CreateObject", "bucket", "object", "", "", mock.Anything).Return(nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "object", "").Return(objectMetadata, nil).Once()
typedDriver.On("CreateObject", "bucket", "object", "", "", mock.Anything).Return(objectMetadata.Md5, nil).Once()
driver.CreateObject("bucket", "object", "", "", buffer)
driver.GetObjectMetadata("bucket", "object", "")
typedDriver.On("GetBucketMetadata", "bucket").Return(bucketMetadata, nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "object", "").Return(objectMetadata, nil).Once()
@ -629,8 +617,7 @@ func (s *MySuite) TestPutObject(c *C) {
Size: 11,
}
typedDriver.On("CreateObject", "bucket", "two", "", "", mock.Anything).Return(nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "two", "").Return(twoMetadata, nil).Once()
typedDriver.On("CreateObject", "bucket", "two", "", "", mock.Anything).Return(twoMetadata.Md5, nil).Once()
request, err = http.NewRequest("PUT", testServer.URL+"/bucket/two", bytes.NewBufferString("hello world"))
c.Assert(err, IsNil)
setAuthHeader(request)
@ -917,8 +904,7 @@ func (s *MySuite) TestContentTypePersists(c *C) {
}
typedDriver.On("GetBucketMetadata", "bucket").Return(metadata, nil).Once()
typedDriver.On("CreateObject", "bucket", "one", "", "", mock.Anything).Return(nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "one", "").Return(oneMetadata, nil).Once()
typedDriver.On("CreateObject", "bucket", "one", "", "", mock.Anything).Return(oneMetadata.Md5, nil).Once()
request, err := http.NewRequest("PUT", testServer.URL+"/bucket/one", bytes.NewBufferString("hello world"))
delete(request.Header, "Content-Type")
c.Assert(err, IsNil)
@ -965,8 +951,7 @@ func (s *MySuite) TestContentTypePersists(c *C) {
}
typedDriver.On("GetBucketMetadata", "bucket").Return(metadata, nil).Once()
typedDriver.On("CreateObject", "bucket", "two", "", "", mock.Anything).Return(nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "two", "").Return(twoMetadata, nil).Once()
typedDriver.On("CreateObject", "bucket", "two", "", "", mock.Anything).Return(twoMetadata.Md5, nil).Once()
request, err = http.NewRequest("PUT", testServer.URL+"/bucket/two", bytes.NewBufferString("hello world"))
delete(request.Header, "Content-Type")
request.Header.Add("Content-Type", "application/json")
@ -1024,13 +1009,11 @@ func (s *MySuite) TestPartialContent(c *C) {
}
typedDriver.On("CreateBucket", "foo", "private").Return(nil).Once()
typedDriver.On("CreateObject", "foo", "bar", "", "", mock.Anything).Return(nil).Once()
typedDriver.On("GetObjectMetadata", "foo", "bar", "").Return(metadata, nil).Once()
typedDriver.On("CreateObject", "foo", "bar", "", "", mock.Anything).Return(metadata.Md5, nil).Once()
err := driver.CreateBucket("foo", "private")
c.Assert(err, IsNil)
driver.CreateObject("foo", "bar", "", "", bytes.NewBufferString("hello world"))
driver.GetObjectMetadata("foo", "bar", "")
// prepare for GET on range request
typedDriver.SetGetObjectWriter("foo", "bar", []byte("hello world"))

20
pkg/storage/donut/donut_bucket.go
View file

@ -137,13 +137,13 @@ func (b bucket) GetObject(objectName string) (reader io.ReadCloser, size int64,
}
// PutObject - put a new object
func (b bucket) PutObject(objectName string, objectData io.Reader, expectedMD5Sum string, metadata map[string]string) error {
func (b bucket) PutObject(objectName string, objectData io.Reader, expectedMD5Sum string, metadata map[string]string) (string, error) {
if objectName == "" || objectData == nil {
return iodine.New(errors.New("invalid argument"), nil)
return "", iodine.New(errors.New("invalid argument"), nil)
}
writers, err := b.getDiskWriters(b.normalizeObjectName(objectName), "data")
if err != nil {
return iodine.New(err, nil)
return "", iodine.New(err, nil)
}
summer := md5.New()
objectMetadata := make(map[string]string)
@ -156,7 +156,7 @@ func (b bucket) PutObject(objectName string, objectData io.Reader, expectedMD5Su
mw := io.MultiWriter(writers[0], summer)
totalLength, err := io.Copy(mw, objectData)
if err != nil {
return iodine.New(err, nil)
return "", iodine.New(err, nil)
}
donutObjectMetadata["sys.size"] = strconv.FormatInt(totalLength, 10)
objectMetadata["size"] = strconv.FormatInt(totalLength, 10)
@ -164,12 +164,12 @@ func (b bucket) PutObject(objectName string, objectData io.Reader, expectedMD5Su
// calculate data and parity dictated by total number of writers
k, m, err := b.getDataAndParity(len(writers))
if err != nil {
return iodine.New(err, nil)
return "", iodine.New(err, nil)
}
// encoded data with k, m and write
chunkCount, totalLength, err := b.writeEncodedData(k, m, writers, objectData, summer)
if err != nil {
return iodine.New(err, nil)
return "", iodine.New(err, nil)
}
/// donutMetadata section
donutObjectMetadata["sys.blockSize"] = strconv.Itoa(10 * 1024 * 1024)
@ -198,20 +198,20 @@ func (b bucket) PutObject(objectName string, objectData io.Reader, expectedMD5Su
// Verify if the written object is equal to what is expected, only if it is requested as such
if strings.TrimSpace(expectedMD5Sum) != "" {
if err := b.isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), objectMetadata["md5"]); err != nil {
return iodine.New(err, nil)
return "", iodine.New(err, nil)
}
}
// write donut specific metadata
if err := b.writeDonutObjectMetadata(b.normalizeObjectName(objectName), donutObjectMetadata); err != nil {
return iodine.New(err, nil)
return "", iodine.New(err, nil)
}
// write object specific metadata
if err := b.writeObjectMetadata(b.normalizeObjectName(objectName), objectMetadata); err != nil {
return iodine.New(err, nil)
return "", iodine.New(err, nil)
}
// close all writers, when control flow reaches here
for _, writer := range writers {
writer.Close()
}
return nil
return objectMetadata["md5"], nil
}

2
pkg/storage/donut/donut_internal_interfaces.go
View file

@ -33,7 +33,7 @@ type Bucket interface {
ListObjects() (map[string]Object, error)
GetObject(object string) (io.ReadCloser, int64, error)
PutObject(object string, contents io.Reader, expectedMD5Sum string, metadata map[string]string) error
PutObject(object string, contents io.Reader, expectedMD5Sum string, metadata map[string]string) (string, error)
}
// Object interface

2
pkg/storage/donut/donut_public_interfaces.go
View file

@ -40,7 +40,7 @@ type ObjectStorage interface {
// Object Operations
GetObject(bucket, object string) (io.ReadCloser, int64, error)
GetObjectMetadata(bucket, object string) (map[string]string, error)
PutObject(bucket, object, expectedMD5Sum string, reader io.ReadCloser, metadata map[string]string) error
PutObject(bucket, object, expectedMD5Sum string, reader io.ReadCloser, metadata map[string]string) (string, error)
}
// Management is a donut management system interface

18
pkg/storage/donut/donut_test.go
View file

@ -162,7 +162,7 @@ func (s *MySuite) TestNewObjectFailsWithoutBucket(c *C) {
defer os.RemoveAll(root)
donut, err := NewDonut("test", createTestNodeDiskMap(root))
c.Assert(err, IsNil)
err = donut.PutObject("foo", "obj", "", nil, nil)
_, err = donut.PutObject("foo", "obj", "", nil, nil)
c.Assert(err, Not(IsNil))
}
@ -188,8 +188,9 @@ func (s *MySuite) TestNewObjectMetadata(c *C) {
err = donut.MakeBucket("foo", "private")
c.Assert(err, IsNil)
err = donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata)
calculatedMd5Sum, err := donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata)
c.Assert(err, IsNil)
c.Assert(calculatedMd5Sum, Equals, expectedMd5Sum)
objectMetadata, err := donut.GetObjectMetadata("foo", "obj")
c.Assert(err, IsNil)
@ -207,10 +208,10 @@ func (s *MySuite) TestNewObjectFailsWithEmptyName(c *C) {
donut, err := NewDonut("test", createTestNodeDiskMap(root))
c.Assert(err, IsNil)
err = donut.PutObject("foo", "", "", nil, nil)
_, err = donut.PutObject("foo", "", "", nil, nil)
c.Assert(err, Not(IsNil))
err = donut.PutObject("foo", " ", "", nil, nil)
_, err = donut.PutObject("foo", " ", "", nil, nil)
c.Assert(err, Not(IsNil))
}
@ -234,8 +235,9 @@ func (s *MySuite) TestNewObjectCanBeWritten(c *C) {
expectedMd5Sum := hex.EncodeToString(hasher.Sum(nil))
reader := ioutil.NopCloser(bytes.NewReader([]byte(data)))
err = donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata)
calculatedMd5Sum, err := donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata)
c.Assert(err, IsNil)
c.Assert(calculatedMd5Sum, Equals, expectedMd5Sum)
reader, size, err := donut.GetObject("foo", "obj")
c.Assert(err, IsNil)
@ -266,11 +268,11 @@ func (s *MySuite) TestMultipleNewObjects(c *C) {
c.Assert(donut.MakeBucket("foo", "private"), IsNil)
one := ioutil.NopCloser(bytes.NewReader([]byte("one")))
err = donut.PutObject("foo", "obj1", "", one, nil)
_, err = donut.PutObject("foo", "obj1", "", one, nil)
c.Assert(err, IsNil)
two := ioutil.NopCloser(bytes.NewReader([]byte("two")))
err = donut.PutObject("foo", "obj2", "", two, nil)
_, err = donut.PutObject("foo", "obj2", "", two, nil)
c.Assert(err, IsNil)
obj1, size, err := donut.GetObject("foo", "obj1")
@ -313,7 +315,7 @@ func (s *MySuite) TestMultipleNewObjects(c *C) {
c.Assert(listObjects, DeepEquals, []string{"obj1", "obj2"})
three := ioutil.NopCloser(bytes.NewReader([]byte("three")))
err = donut.PutObject("foo", "obj3", "", three, nil)
_, err = donut.PutObject("foo", "obj3", "", three, nil)
c.Assert(err, IsNil)
obj3, size, err := donut.GetObject("foo", "obj3")

20
pkg/storage/donut/objectstorage.go
View file

@ -153,38 +153,38 @@ func (d donut) ListObjects(bucket, prefix, marker, delimiter string, maxkeys int
}
// PutObject - put object
func (d donut) PutObject(bucket, object, expectedMD5Sum string, reader io.ReadCloser, metadata map[string]string) error {
func (d donut) PutObject(bucket, object, expectedMD5Sum string, reader io.ReadCloser, metadata map[string]string) (string, error) {
errParams := map[string]string{
"bucket": bucket,
"object": object,
}
if bucket == "" || strings.TrimSpace(bucket) == "" {
return iodine.New(errors.New("invalid argument"), errParams)
return "", iodine.New(errors.New("invalid argument"), errParams)
}
if object == "" || strings.TrimSpace(object) == "" {
return iodine.New(errors.New("invalid argument"), errParams)
return "", iodine.New(errors.New("invalid argument"), errParams)
}
err := d.getDonutBuckets()
if err != nil {
return iodine.New(err, errParams)
return "", iodine.New(err, errParams)
}
if _, ok := d.buckets[bucket]; !ok {
return iodine.New(errors.New("bucket does not exist"), nil)
return "", iodine.New(errors.New("bucket does not exist"), nil)
}
objectList, err := d.buckets[bucket].ListObjects()
if err != nil {
return iodine.New(err, nil)
return "", iodine.New(err, nil)
}
for objectName := range objectList {
if objectName == object {
return iodine.New(errors.New("object exists"), nil)
return "", iodine.New(errors.New("object exists"), nil)
}
}
err = d.buckets[bucket].PutObject(object, reader, expectedMD5Sum, metadata)
md5sum, err := d.buckets[bucket].PutObject(object, reader, expectedMD5Sum, metadata)
if err != nil {
return iodine.New(err, errParams)
return "", iodine.New(err, errParams)
}
return nil
return md5sum, nil
}
// GetObject - get object

36
pkg/storage/drivers/api_testsuite.go
View file

@ -20,6 +20,7 @@ import (
"bytes"
"crypto/md5"
"encoding/base64"
"encoding/hex"
"math/rand"
"strconv"
@ -67,12 +68,14 @@ func testMultipleObjectCreation(c *check.C, create func() Driver) {
hasher := md5.New()
hasher.Write([]byte(randomString))
md5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
key := "obj" + strconv.Itoa(i)
objects[key] = []byte(randomString)
err := drivers.CreateObject("bucket", key, "", md5Sum, bytes.NewBufferString(randomString))
calculatedmd5sum, err := drivers.CreateObject("bucket", key, "", expectedmd5Sum, bytes.NewBufferString(randomString))
c.Assert(err, check.IsNil)
c.Assert(calculatedmd5sum, check.Equals, expectedmd5Sumhex)
}
// ensure no duplicate etags
@ -206,15 +209,17 @@ func testObjectOverwriteFails(c *check.C, create func() Driver) {
hasher1 := md5.New()
hasher1.Write([]byte("one"))
md5Sum1 := base64.StdEncoding.EncodeToString(hasher1.Sum(nil))
err := drivers.CreateObject("bucket", "object", "", md5Sum1, bytes.NewBufferString("one"))
md5Sum1hex := hex.EncodeToString(hasher1.Sum(nil))
md5Sum11, err := drivers.CreateObject("bucket", "object", "", md5Sum1, bytes.NewBufferString("one"))
c.Assert(err, check.IsNil)
c.Assert(md5Sum1hex, check.Equals, md5Sum11)
hasher2 := md5.New()
hasher2.Write([]byte("three"))
md5Sum2 := base64.StdEncoding.EncodeToString(hasher2.Sum(nil))
err = drivers.CreateObject("bucket", "object", "", md5Sum2, bytes.NewBufferString("three"))
_, err = drivers.CreateObject("bucket", "object", "", md5Sum2, bytes.NewBufferString("three"))
c.Assert(err, check.Not(check.IsNil))
var bytesBuffer bytes.Buffer
length, err := drivers.GetObject(&bytesBuffer, "bucket", "object")
c.Assert(length, check.Equals, int64(len("one")))
@ -224,7 +229,7 @@ func testObjectOverwriteFails(c *check.C, create func() Driver) {
func testNonExistantBucketOperations(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateObject("bucket", "object", "", "", bytes.NewBufferString("one"))
_, err := drivers.CreateObject("bucket", "object", "", "", bytes.NewBufferString("one"))
c.Assert(err, check.Not(check.IsNil))
}
@ -253,9 +258,11 @@ func testPutObjectInSubdir(c *check.C, create func() Driver) {
hasher := md5.New()
hasher.Write([]byte("hello world"))
md5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
err = drivers.CreateObject("bucket", "dir1/dir2/object", "", md5Sum, bytes.NewBufferString("hello world"))
md5Sum1 := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
md5Sum1hex := hex.EncodeToString(hasher.Sum(nil))
md5Sum11, err := drivers.CreateObject("bucket", "dir1/dir2/object", "", md5Sum1, bytes.NewBufferString("hello world"))
c.Assert(err, check.IsNil)
c.Assert(md5Sum11, check.Equals, md5Sum1hex)
var bytesBuffer bytes.Buffer
length, err := drivers.GetObject(&bytesBuffer, "bucket", "dir1/dir2/object")
@ -349,7 +356,7 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Driver) {
err := drivers.CreateBucket("bucket", "")
c.Assert(err, check.IsNil)
err = drivers.CreateObject("bucket", "dir1/dir2/object", "", "", bytes.NewBufferString("hello world"))
_, err = drivers.CreateObject("bucket", "dir1/dir2/object", "", "", bytes.NewBufferString("hello world"))
c.Assert(err, check.IsNil)
var byteBuffer bytes.Buffer
@ -393,7 +400,7 @@ func testDefaultContentType(c *check.C, create func() Driver) {
c.Assert(err, check.IsNil)
// test empty
err = drivers.CreateObject("bucket", "one", "", "", bytes.NewBufferString("one"))
_, err = drivers.CreateObject("bucket", "one", "", "", bytes.NewBufferString("one"))
metadata, err := drivers.GetObjectMetadata("bucket", "one", "")
c.Assert(err, check.IsNil)
c.Assert(metadata.ContentType, check.Equals, "application/octet-stream")
@ -417,8 +424,13 @@ func testContentMd5Set(c *check.C, create func() Driver) {
c.Assert(err, check.IsNil)
// test md5 invalid
err = drivers.CreateObject("bucket", "one", "", "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA", bytes.NewBufferString("one"))
badmd5Sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA"
calculatedmd5sum, err := drivers.CreateObject("bucket", "one", "", badmd5Sum, bytes.NewBufferString("one"))
c.Assert(err, check.Not(check.IsNil))
err = drivers.CreateObject("bucket", "two", "", "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA=", bytes.NewBufferString("one"))
c.Assert(calculatedmd5sum, check.Not(check.Equals), badmd5Sum)
goodmd5sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA="
calculatedmd5sum, err = drivers.CreateObject("bucket", "two", "", goodmd5sum, bytes.NewBufferString("one"))
c.Assert(err, check.IsNil)
c.Assert(calculatedmd5sum, check.Equals, goodmd5sum)
}

14
pkg/storage/drivers/donut/donut.go
View file

@ -366,17 +366,17 @@ func (d donutDriver) ListObjects(bucketName string, resources drivers.BucketReso
}
// CreateObject creates a new object
func (d donutDriver) CreateObject(bucketName, objectName, contentType, expectedMD5Sum string, reader io.Reader) error {
func (d donutDriver) CreateObject(bucketName, objectName, contentType, expectedMD5Sum string, reader io.Reader) (string, error) {
errParams := map[string]string{
"bucketName": bucketName,
"objectName": objectName,
"contentType": contentType,
}
if !drivers.IsValidBucket(bucketName) || strings.Contains(bucketName, ".") {
return iodine.New(drivers.BucketNameInvalid{Bucket: bucketName}, nil)
return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucketName}, nil)
}
if !drivers.IsValidObject(objectName) || strings.TrimSpace(objectName) == "" {
return iodine.New(drivers.ObjectNameInvalid{Object: objectName}, nil)
return "", iodine.New(drivers.ObjectNameInvalid{Object: objectName}, nil)
}
if strings.TrimSpace(contentType) == "" {
contentType = "application/octet-stream"
@ -387,13 +387,13 @@ func (d donutDriver) CreateObject(bucketName, objectName, contentType, expectedM
if strings.TrimSpace(expectedMD5Sum) != "" {
expectedMD5SumBytes, err := base64.StdEncoding.DecodeString(strings.TrimSpace(expectedMD5Sum))
if err != nil {
return iodine.New(err, nil)
return "", iodine.New(err, nil)
}
expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
}
err := d.donut.PutObject(bucketName, objectName, expectedMD5Sum, ioutil.NopCloser(reader), metadata)
calculatedMD5Sum, err := d.donut.PutObject(bucketName, objectName, expectedMD5Sum, ioutil.NopCloser(reader), metadata)
if err != nil {
return iodine.New(err, errParams)
return "", iodine.New(err, errParams)
}
return nil
return calculatedMD5Sum, nil
}

2
pkg/storage/drivers/driver.go
View file

@ -37,7 +37,7 @@ type Driver interface {
GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error)
GetObjectMetadata(bucket string, object string, prefix string) (ObjectMetadata, error)
ListObjects(bucket string, resources BucketResourcesMetadata) ([]ObjectMetadata, BucketResourcesMetadata, error)
CreateObject(bucket string, key string, contentType string, md5sum string, data io.Reader) error
CreateObject(bucket string, key string, contentType string, md5sum string, data io.Reader) (string, error)
}
// BucketACL - bucket level access control

18
pkg/storage/drivers/memory/memory.go
View file

@ -199,26 +199,26 @@ func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) error {
}
// CreateObject - PUT object to memory buffer
func (memory *memoryDriver) CreateObject(bucket, key, contentType, expectedMD5Sum string, data io.Reader) error {
func (memory *memoryDriver) CreateObject(bucket, key, contentType, expectedMD5Sum string, data io.Reader) (string, error) {
memory.lock.RLock()
if !drivers.IsValidBucket(bucket) {
memory.lock.RUnlock()
return iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
}
if !drivers.IsValidObject(key) {
memory.lock.RUnlock()
return iodine.New(drivers.ObjectNameInvalid{Object: key}, nil)
return "", iodine.New(drivers.ObjectNameInvalid{Object: key}, nil)
}
if _, ok := memory.storedBuckets[bucket]; ok == false {
memory.lock.RUnlock()
return iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
return "", iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
}
storedBucket := memory.storedBuckets[bucket]
// get object key
objectKey := bucket + "/" + key
if _, ok := storedBucket.objectMetadata[objectKey]; ok == true {
memory.lock.RUnlock()
return iodine.New(drivers.ObjectExists{Bucket: bucket, Object: key}, nil)
return "", iodine.New(drivers.ObjectExists{Bucket: bucket, Object: key}, nil)
}
memory.lock.RUnlock()
@ -230,7 +230,7 @@ func (memory *memoryDriver) CreateObject(bucket, key, contentType, expectedMD5Su
expectedMD5SumBytes, err := base64.StdEncoding.DecodeString(strings.TrimSpace(expectedMD5Sum))
if err != nil {
// pro-actively close the connection
return iodine.New(drivers.InvalidDigest{Md5: expectedMD5Sum}, nil)
return "", iodine.New(drivers.InvalidDigest{Md5: expectedMD5Sum}, nil)
}
expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
}
@ -248,7 +248,7 @@ func (memory *memoryDriver) CreateObject(bucket, key, contentType, expectedMD5Su
if err != nil {
err := iodine.New(err, nil)
log.Println(err)
return err
return "", err
}
if uint64(totalLength)+memory.totalSize > memory.maxSize {
memory.objects.RemoveOldest()
@ -263,7 +263,7 @@ func (memory *memoryDriver) CreateObject(bucket, key, contentType, expectedMD5Su
memory.lock.Lock()
defer memory.lock.Unlock()
memory.objects.RemoveOldest()
return iodine.New(drivers.BadDigest{Md5: expectedMD5Sum, Bucket: bucket, Key: key}, nil)
return "", iodine.New(drivers.BadDigest{Md5: expectedMD5Sum, Bucket: bucket, Key: key}, nil)
}
}
newObject := drivers.ObjectMetadata{
@ -292,7 +292,7 @@ func (memory *memoryDriver) CreateObject(bucket, key, contentType, expectedMD5Su
memory.lock.Unlock()
// free memory if possible for kernel to reclaim
debug.FreeOSMemory()
return nil
return newObject.Md5, nil
}
// CreateBucket - create bucket in memory

7
pkg/storage/drivers/mocks/Driver.go
View file

@ -116,10 +116,11 @@ func (m *Driver) ListObjects(bucket string, resources drivers.BucketResourcesMet
}
// CreateObject is a mock
func (m *Driver) CreateObject(bucket string, key string, contentType string, md5sum string, data io.Reader) error {
func (m *Driver) CreateObject(bucket string, key string, contentType string, md5sum string, data io.Reader) (string, error) {
ret := m.Called(bucket, key, contentType, md5sum, data)
r0 := ret.Error(0)
r0 := ret.Get(0).(string)
r1 := ret.Error(1)
return r0
return r0, r1
}