Changes to CreateObject() now returns back md5 along with any error

- This change is necessary to avoid the racy calls to GetObjectMetadata()
- This change is also necessary since one has to reply back md5sum with
  PUT object response header
This commit is contained in:
Harshavardhana 2015-04-30 03:38:11 -07:00
parent 13cae94191
commit d815e6adfd
12 changed files with 90 additions and 109 deletions

View file

@ -171,29 +171,12 @@ func (server *minioAPI) putObjectHandler(w http.ResponseWriter, req *http.Reques
writeErrorResponse(w, req, EntityTooSmall, acceptsContentType, req.URL.Path) writeErrorResponse(w, req, EntityTooSmall, acceptsContentType, req.URL.Path)
return return
} }
err := server.driver.CreateObject(bucket, object, "", md5, req.Body) calculatedMD5, err := server.driver.CreateObject(bucket, object, "", md5, req.Body)
switch err := iodine.ToError(err).(type) { switch err := iodine.ToError(err).(type) {
case nil: case nil:
{ {
metadata, err := server.driver.GetObjectMetadata(bucket, object, "") w.Header().Set("ETag", calculatedMD5)
switch err := iodine.ToError(err).(type) {
case nil:
w.Header().Set("ETag", metadata.Md5)
writeSuccessResponse(w, acceptsContentType) writeSuccessResponse(w, acceptsContentType)
case drivers.ObjectNotFound:
{
writeErrorResponse(w, req, NoSuchKey, acceptsContentType, req.URL.Path)
}
case drivers.ObjectNameInvalid:
{
writeErrorResponse(w, req, NoSuchKey, acceptsContentType, req.URL.Path)
}
default:
{
log.Error.Println(iodine.New(err, nil))
writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path)
}
}
} }
case drivers.ObjectExists: case drivers.ObjectExists:

View file

@ -167,8 +167,7 @@ func (s *MySuite) TestEmptyObject(c *C) {
Size: 0, Size: 0,
} }
typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once() typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once()
typedDriver.On("CreateObject", "bucket", "object", "", "", mock.Anything).Return(nil).Once() typedDriver.On("CreateObject", "bucket", "object", "", "", mock.Anything).Return(metadata.Md5, nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "object", "").Return(metadata, nil).Once()
typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Twice() typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Twice()
typedDriver.On("GetObjectMetadata", "bucket", "object", "").Return(metadata, nil).Once() typedDriver.On("GetObjectMetadata", "bucket", "object", "").Return(metadata, nil).Once()
typedDriver.On("GetObject", mock.Anything, "bucket", "object").Return(int64(0), nil).Once() typedDriver.On("GetObject", mock.Anything, "bucket", "object").Return(int64(0), nil).Once()
@ -180,7 +179,6 @@ func (s *MySuite) TestEmptyObject(c *C) {
buffer := bytes.NewBufferString("") buffer := bytes.NewBufferString("")
driver.CreateBucket("bucket", "private") driver.CreateBucket("bucket", "private")
driver.CreateObject("bucket", "object", "", "", buffer) driver.CreateObject("bucket", "object", "", "", buffer)
driver.GetObjectMetadata("bucket", "object", "")
request, err := http.NewRequest("GET", testServer.URL+"/bucket/object", nil) request, err := http.NewRequest("GET", testServer.URL+"/bucket/object", nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -251,8 +249,7 @@ func (s *MySuite) TestObject(c *C) {
Size: 11, Size: 11,
} }
typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once() typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once()
typedDriver.On("CreateObject", "bucket", "object", "", "", mock.Anything).Return(nil).Once() typedDriver.On("CreateObject", "bucket", "object", "", "", mock.Anything).Return(metadata.Md5, nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "object", "").Return(metadata, nil).Twice()
typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Twice() typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Twice()
typedDriver.On("GetObjectMetadata", "bucket", "object", "").Return(metadata, nil).Twice() typedDriver.On("GetObjectMetadata", "bucket", "object", "").Return(metadata, nil).Twice()
typedDriver.SetGetObjectWriter("bucket", "object", []byte("hello world")) typedDriver.SetGetObjectWriter("bucket", "object", []byte("hello world"))
@ -265,7 +262,6 @@ func (s *MySuite) TestObject(c *C) {
buffer := bytes.NewBufferString("hello world") buffer := bytes.NewBufferString("hello world")
driver.CreateBucket("bucket", "private") driver.CreateBucket("bucket", "private")
driver.CreateObject("bucket", "object", "", "", buffer) driver.CreateObject("bucket", "object", "", "", buffer)
driver.GetObjectMetadata("bucket", "object", "")
request, err := http.NewRequest("GET", testServer.URL+"/bucket/object", nil) request, err := http.NewRequest("GET", testServer.URL+"/bucket/object", nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -328,18 +324,12 @@ func (s *MySuite) TestMultipleObjects(c *C) {
typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once() typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once()
driver.CreateBucket("bucket", "private") driver.CreateBucket("bucket", "private")
typedDriver.On("CreateObject", "bucket", "object1", "", "", mock.Anything).Return(nil).Once() typedDriver.On("CreateObject", "bucket", "object1", "", "", mock.Anything).Return(metadata1.Md5, nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "object1", "").Return(metadata1, nil).Once()
driver.CreateObject("bucket", "object1", "", "", buffer1) driver.CreateObject("bucket", "object1", "", "", buffer1)
driver.GetObjectMetadata("bucket", "object1", "") typedDriver.On("CreateObject", "bucket", "object2", "", "", mock.Anything).Return(metadata2.Md5, nil).Once()
typedDriver.On("CreateObject", "bucket", "object2", "", "", mock.Anything).Return(nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "object2", "").Return(metadata2, nil).Once()
driver.CreateObject("bucket", "object2", "", "", buffer2) driver.CreateObject("bucket", "object2", "", "", buffer2)
driver.GetObjectMetadata("bucket", "object2", "") typedDriver.On("CreateObject", "bucket", "object3", "", "", mock.Anything).Return(metadata3.Md5, nil).Once()
typedDriver.On("CreateObject", "bucket", "object3", "", "", mock.Anything).Return(nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "object3", "").Return(metadata3, nil).Once()
driver.CreateObject("bucket", "object3", "", "", buffer3) driver.CreateObject("bucket", "object3", "", "", buffer3)
driver.GetObjectMetadata("bucket", "object3", "")
// test non-existant object // test non-existant object
typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Once() typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Once()
@ -515,10 +505,8 @@ func (s *MySuite) TestHeader(c *C) {
buffer := bytes.NewBufferString("hello world") buffer := bytes.NewBufferString("hello world")
typedDriver.On("GetBucketMetadata", "foo").Return(bucketMetadata, nil).Once() typedDriver.On("GetBucketMetadata", "foo").Return(bucketMetadata, nil).Once()
typedDriver.On("CreateObject", "bucket", "object", "", "", mock.Anything).Return(nil).Once() typedDriver.On("CreateObject", "bucket", "object", "", "", mock.Anything).Return(objectMetadata.Md5, nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "object", "").Return(objectMetadata, nil).Once()
driver.CreateObject("bucket", "object", "", "", buffer) driver.CreateObject("bucket", "object", "", "", buffer)
driver.GetObjectMetadata("bucket", "object", "")
typedDriver.On("GetBucketMetadata", "bucket").Return(bucketMetadata, nil).Once() typedDriver.On("GetBucketMetadata", "bucket").Return(bucketMetadata, nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "object", "").Return(objectMetadata, nil).Once() typedDriver.On("GetObjectMetadata", "bucket", "object", "").Return(objectMetadata, nil).Once()
@ -629,8 +617,7 @@ func (s *MySuite) TestPutObject(c *C) {
Size: 11, Size: 11,
} }
typedDriver.On("CreateObject", "bucket", "two", "", "", mock.Anything).Return(nil).Once() typedDriver.On("CreateObject", "bucket", "two", "", "", mock.Anything).Return(twoMetadata.Md5, nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "two", "").Return(twoMetadata, nil).Once()
request, err = http.NewRequest("PUT", testServer.URL+"/bucket/two", bytes.NewBufferString("hello world")) request, err = http.NewRequest("PUT", testServer.URL+"/bucket/two", bytes.NewBufferString("hello world"))
c.Assert(err, IsNil) c.Assert(err, IsNil)
setAuthHeader(request) setAuthHeader(request)
@ -917,8 +904,7 @@ func (s *MySuite) TestContentTypePersists(c *C) {
} }
typedDriver.On("GetBucketMetadata", "bucket").Return(metadata, nil).Once() typedDriver.On("GetBucketMetadata", "bucket").Return(metadata, nil).Once()
typedDriver.On("CreateObject", "bucket", "one", "", "", mock.Anything).Return(nil).Once() typedDriver.On("CreateObject", "bucket", "one", "", "", mock.Anything).Return(oneMetadata.Md5, nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "one", "").Return(oneMetadata, nil).Once()
request, err := http.NewRequest("PUT", testServer.URL+"/bucket/one", bytes.NewBufferString("hello world")) request, err := http.NewRequest("PUT", testServer.URL+"/bucket/one", bytes.NewBufferString("hello world"))
delete(request.Header, "Content-Type") delete(request.Header, "Content-Type")
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -965,8 +951,7 @@ func (s *MySuite) TestContentTypePersists(c *C) {
} }
typedDriver.On("GetBucketMetadata", "bucket").Return(metadata, nil).Once() typedDriver.On("GetBucketMetadata", "bucket").Return(metadata, nil).Once()
typedDriver.On("CreateObject", "bucket", "two", "", "", mock.Anything).Return(nil).Once() typedDriver.On("CreateObject", "bucket", "two", "", "", mock.Anything).Return(twoMetadata.Md5, nil).Once()
typedDriver.On("GetObjectMetadata", "bucket", "two", "").Return(twoMetadata, nil).Once()
request, err = http.NewRequest("PUT", testServer.URL+"/bucket/two", bytes.NewBufferString("hello world")) request, err = http.NewRequest("PUT", testServer.URL+"/bucket/two", bytes.NewBufferString("hello world"))
delete(request.Header, "Content-Type") delete(request.Header, "Content-Type")
request.Header.Add("Content-Type", "application/json") request.Header.Add("Content-Type", "application/json")
@ -1024,13 +1009,11 @@ func (s *MySuite) TestPartialContent(c *C) {
} }
typedDriver.On("CreateBucket", "foo", "private").Return(nil).Once() typedDriver.On("CreateBucket", "foo", "private").Return(nil).Once()
typedDriver.On("CreateObject", "foo", "bar", "", "", mock.Anything).Return(nil).Once() typedDriver.On("CreateObject", "foo", "bar", "", "", mock.Anything).Return(metadata.Md5, nil).Once()
typedDriver.On("GetObjectMetadata", "foo", "bar", "").Return(metadata, nil).Once()
err := driver.CreateBucket("foo", "private") err := driver.CreateBucket("foo", "private")
c.Assert(err, IsNil) c.Assert(err, IsNil)
driver.CreateObject("foo", "bar", "", "", bytes.NewBufferString("hello world")) driver.CreateObject("foo", "bar", "", "", bytes.NewBufferString("hello world"))
driver.GetObjectMetadata("foo", "bar", "")
// prepare for GET on range request // prepare for GET on range request
typedDriver.SetGetObjectWriter("foo", "bar", []byte("hello world")) typedDriver.SetGetObjectWriter("foo", "bar", []byte("hello world"))

View file

@ -137,13 +137,13 @@ func (b bucket) GetObject(objectName string) (reader io.ReadCloser, size int64,
} }
// PutObject - put a new object // PutObject - put a new object
func (b bucket) PutObject(objectName string, objectData io.Reader, expectedMD5Sum string, metadata map[string]string) error { func (b bucket) PutObject(objectName string, objectData io.Reader, expectedMD5Sum string, metadata map[string]string) (string, error) {
if objectName == "" || objectData == nil { if objectName == "" || objectData == nil {
return iodine.New(errors.New("invalid argument"), nil) return "", iodine.New(errors.New("invalid argument"), nil)
} }
writers, err := b.getDiskWriters(b.normalizeObjectName(objectName), "data") writers, err := b.getDiskWriters(b.normalizeObjectName(objectName), "data")
if err != nil { if err != nil {
return iodine.New(err, nil) return "", iodine.New(err, nil)
} }
summer := md5.New() summer := md5.New()
objectMetadata := make(map[string]string) objectMetadata := make(map[string]string)
@ -156,7 +156,7 @@ func (b bucket) PutObject(objectName string, objectData io.Reader, expectedMD5Su
mw := io.MultiWriter(writers[0], summer) mw := io.MultiWriter(writers[0], summer)
totalLength, err := io.Copy(mw, objectData) totalLength, err := io.Copy(mw, objectData)
if err != nil { if err != nil {
return iodine.New(err, nil) return "", iodine.New(err, nil)
} }
donutObjectMetadata["sys.size"] = strconv.FormatInt(totalLength, 10) donutObjectMetadata["sys.size"] = strconv.FormatInt(totalLength, 10)
objectMetadata["size"] = strconv.FormatInt(totalLength, 10) objectMetadata["size"] = strconv.FormatInt(totalLength, 10)
@ -164,12 +164,12 @@ func (b bucket) PutObject(objectName string, objectData io.Reader, expectedMD5Su
// calculate data and parity dictated by total number of writers // calculate data and parity dictated by total number of writers
k, m, err := b.getDataAndParity(len(writers)) k, m, err := b.getDataAndParity(len(writers))
if err != nil { if err != nil {
return iodine.New(err, nil) return "", iodine.New(err, nil)
} }
// encoded data with k, m and write // encoded data with k, m and write
chunkCount, totalLength, err := b.writeEncodedData(k, m, writers, objectData, summer) chunkCount, totalLength, err := b.writeEncodedData(k, m, writers, objectData, summer)
if err != nil { if err != nil {
return iodine.New(err, nil) return "", iodine.New(err, nil)
} }
/// donutMetadata section /// donutMetadata section
donutObjectMetadata["sys.blockSize"] = strconv.Itoa(10 * 1024 * 1024) donutObjectMetadata["sys.blockSize"] = strconv.Itoa(10 * 1024 * 1024)
@ -198,20 +198,20 @@ func (b bucket) PutObject(objectName string, objectData io.Reader, expectedMD5Su
// Verify if the written object is equal to what is expected, only if it is requested as such // Verify if the written object is equal to what is expected, only if it is requested as such
if strings.TrimSpace(expectedMD5Sum) != "" { if strings.TrimSpace(expectedMD5Sum) != "" {
if err := b.isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), objectMetadata["md5"]); err != nil { if err := b.isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), objectMetadata["md5"]); err != nil {
return iodine.New(err, nil) return "", iodine.New(err, nil)
} }
} }
// write donut specific metadata // write donut specific metadata
if err := b.writeDonutObjectMetadata(b.normalizeObjectName(objectName), donutObjectMetadata); err != nil { if err := b.writeDonutObjectMetadata(b.normalizeObjectName(objectName), donutObjectMetadata); err != nil {
return iodine.New(err, nil) return "", iodine.New(err, nil)
} }
// write object specific metadata // write object specific metadata
if err := b.writeObjectMetadata(b.normalizeObjectName(objectName), objectMetadata); err != nil { if err := b.writeObjectMetadata(b.normalizeObjectName(objectName), objectMetadata); err != nil {
return iodine.New(err, nil) return "", iodine.New(err, nil)
} }
// close all writers, when control flow reaches here // close all writers, when control flow reaches here
for _, writer := range writers { for _, writer := range writers {
writer.Close() writer.Close()
} }
return nil return objectMetadata["md5"], nil
} }

View file

@ -33,7 +33,7 @@ type Bucket interface {
ListObjects() (map[string]Object, error) ListObjects() (map[string]Object, error)
GetObject(object string) (io.ReadCloser, int64, error) GetObject(object string) (io.ReadCloser, int64, error)
PutObject(object string, contents io.Reader, expectedMD5Sum string, metadata map[string]string) error PutObject(object string, contents io.Reader, expectedMD5Sum string, metadata map[string]string) (string, error)
} }
// Object interface // Object interface

View file

@ -40,7 +40,7 @@ type ObjectStorage interface {
// Object Operations // Object Operations
GetObject(bucket, object string) (io.ReadCloser, int64, error) GetObject(bucket, object string) (io.ReadCloser, int64, error)
GetObjectMetadata(bucket, object string) (map[string]string, error) GetObjectMetadata(bucket, object string) (map[string]string, error)
PutObject(bucket, object, expectedMD5Sum string, reader io.ReadCloser, metadata map[string]string) error PutObject(bucket, object, expectedMD5Sum string, reader io.ReadCloser, metadata map[string]string) (string, error)
} }
// Management is a donut management system interface // Management is a donut management system interface

View file

@ -162,7 +162,7 @@ func (s *MySuite) TestNewObjectFailsWithoutBucket(c *C) {
defer os.RemoveAll(root) defer os.RemoveAll(root)
donut, err := NewDonut("test", createTestNodeDiskMap(root)) donut, err := NewDonut("test", createTestNodeDiskMap(root))
c.Assert(err, IsNil) c.Assert(err, IsNil)
err = donut.PutObject("foo", "obj", "", nil, nil) _, err = donut.PutObject("foo", "obj", "", nil, nil)
c.Assert(err, Not(IsNil)) c.Assert(err, Not(IsNil))
} }
@ -188,8 +188,9 @@ func (s *MySuite) TestNewObjectMetadata(c *C) {
err = donut.MakeBucket("foo", "private") err = donut.MakeBucket("foo", "private")
c.Assert(err, IsNil) c.Assert(err, IsNil)
err = donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata) calculatedMd5Sum, err := donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata)
c.Assert(err, IsNil) c.Assert(err, IsNil)
c.Assert(calculatedMd5Sum, Equals, expectedMd5Sum)
objectMetadata, err := donut.GetObjectMetadata("foo", "obj") objectMetadata, err := donut.GetObjectMetadata("foo", "obj")
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -207,10 +208,10 @@ func (s *MySuite) TestNewObjectFailsWithEmptyName(c *C) {
donut, err := NewDonut("test", createTestNodeDiskMap(root)) donut, err := NewDonut("test", createTestNodeDiskMap(root))
c.Assert(err, IsNil) c.Assert(err, IsNil)
err = donut.PutObject("foo", "", "", nil, nil) _, err = donut.PutObject("foo", "", "", nil, nil)
c.Assert(err, Not(IsNil)) c.Assert(err, Not(IsNil))
err = donut.PutObject("foo", " ", "", nil, nil) _, err = donut.PutObject("foo", " ", "", nil, nil)
c.Assert(err, Not(IsNil)) c.Assert(err, Not(IsNil))
} }
@ -234,8 +235,9 @@ func (s *MySuite) TestNewObjectCanBeWritten(c *C) {
expectedMd5Sum := hex.EncodeToString(hasher.Sum(nil)) expectedMd5Sum := hex.EncodeToString(hasher.Sum(nil))
reader := ioutil.NopCloser(bytes.NewReader([]byte(data))) reader := ioutil.NopCloser(bytes.NewReader([]byte(data)))
err = donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata) calculatedMd5Sum, err := donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata)
c.Assert(err, IsNil) c.Assert(err, IsNil)
c.Assert(calculatedMd5Sum, Equals, expectedMd5Sum)
reader, size, err := donut.GetObject("foo", "obj") reader, size, err := donut.GetObject("foo", "obj")
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -266,11 +268,11 @@ func (s *MySuite) TestMultipleNewObjects(c *C) {
c.Assert(donut.MakeBucket("foo", "private"), IsNil) c.Assert(donut.MakeBucket("foo", "private"), IsNil)
one := ioutil.NopCloser(bytes.NewReader([]byte("one"))) one := ioutil.NopCloser(bytes.NewReader([]byte("one")))
err = donut.PutObject("foo", "obj1", "", one, nil) _, err = donut.PutObject("foo", "obj1", "", one, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
two := ioutil.NopCloser(bytes.NewReader([]byte("two"))) two := ioutil.NopCloser(bytes.NewReader([]byte("two")))
err = donut.PutObject("foo", "obj2", "", two, nil) _, err = donut.PutObject("foo", "obj2", "", two, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
obj1, size, err := donut.GetObject("foo", "obj1") obj1, size, err := donut.GetObject("foo", "obj1")
@ -313,7 +315,7 @@ func (s *MySuite) TestMultipleNewObjects(c *C) {
c.Assert(listObjects, DeepEquals, []string{"obj1", "obj2"}) c.Assert(listObjects, DeepEquals, []string{"obj1", "obj2"})
three := ioutil.NopCloser(bytes.NewReader([]byte("three"))) three := ioutil.NopCloser(bytes.NewReader([]byte("three")))
err = donut.PutObject("foo", "obj3", "", three, nil) _, err = donut.PutObject("foo", "obj3", "", three, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
obj3, size, err := donut.GetObject("foo", "obj3") obj3, size, err := donut.GetObject("foo", "obj3")

View file

@ -153,38 +153,38 @@ func (d donut) ListObjects(bucket, prefix, marker, delimiter string, maxkeys int
} }
// PutObject - put object // PutObject - put object
func (d donut) PutObject(bucket, object, expectedMD5Sum string, reader io.ReadCloser, metadata map[string]string) error { func (d donut) PutObject(bucket, object, expectedMD5Sum string, reader io.ReadCloser, metadata map[string]string) (string, error) {
errParams := map[string]string{ errParams := map[string]string{
"bucket": bucket, "bucket": bucket,
"object": object, "object": object,
} }
if bucket == "" || strings.TrimSpace(bucket) == "" { if bucket == "" || strings.TrimSpace(bucket) == "" {
return iodine.New(errors.New("invalid argument"), errParams) return "", iodine.New(errors.New("invalid argument"), errParams)
} }
if object == "" || strings.TrimSpace(object) == "" { if object == "" || strings.TrimSpace(object) == "" {
return iodine.New(errors.New("invalid argument"), errParams) return "", iodine.New(errors.New("invalid argument"), errParams)
} }
err := d.getDonutBuckets() err := d.getDonutBuckets()
if err != nil { if err != nil {
return iodine.New(err, errParams) return "", iodine.New(err, errParams)
} }
if _, ok := d.buckets[bucket]; !ok { if _, ok := d.buckets[bucket]; !ok {
return iodine.New(errors.New("bucket does not exist"), nil) return "", iodine.New(errors.New("bucket does not exist"), nil)
} }
objectList, err := d.buckets[bucket].ListObjects() objectList, err := d.buckets[bucket].ListObjects()
if err != nil { if err != nil {
return iodine.New(err, nil) return "", iodine.New(err, nil)
} }
for objectName := range objectList { for objectName := range objectList {
if objectName == object { if objectName == object {
return iodine.New(errors.New("object exists"), nil) return "", iodine.New(errors.New("object exists"), nil)
} }
} }
err = d.buckets[bucket].PutObject(object, reader, expectedMD5Sum, metadata) md5sum, err := d.buckets[bucket].PutObject(object, reader, expectedMD5Sum, metadata)
if err != nil { if err != nil {
return iodine.New(err, errParams) return "", iodine.New(err, errParams)
} }
return nil return md5sum, nil
} }
// GetObject - get object // GetObject - get object

View file

@ -20,6 +20,7 @@ import (
"bytes" "bytes"
"crypto/md5" "crypto/md5"
"encoding/base64" "encoding/base64"
"encoding/hex"
"math/rand" "math/rand"
"strconv" "strconv"
@ -67,12 +68,14 @@ func testMultipleObjectCreation(c *check.C, create func() Driver) {
hasher := md5.New() hasher := md5.New()
hasher.Write([]byte(randomString)) hasher.Write([]byte(randomString))
md5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
key := "obj" + strconv.Itoa(i) key := "obj" + strconv.Itoa(i)
objects[key] = []byte(randomString) objects[key] = []byte(randomString)
err := drivers.CreateObject("bucket", key, "", md5Sum, bytes.NewBufferString(randomString)) calculatedmd5sum, err := drivers.CreateObject("bucket", key, "", expectedmd5Sum, bytes.NewBufferString(randomString))
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
c.Assert(calculatedmd5sum, check.Equals, expectedmd5Sumhex)
} }
// ensure no duplicate etags // ensure no duplicate etags
@ -206,15 +209,17 @@ func testObjectOverwriteFails(c *check.C, create func() Driver) {
hasher1 := md5.New() hasher1 := md5.New()
hasher1.Write([]byte("one")) hasher1.Write([]byte("one"))
md5Sum1 := base64.StdEncoding.EncodeToString(hasher1.Sum(nil)) md5Sum1 := base64.StdEncoding.EncodeToString(hasher1.Sum(nil))
err := drivers.CreateObject("bucket", "object", "", md5Sum1, bytes.NewBufferString("one")) md5Sum1hex := hex.EncodeToString(hasher1.Sum(nil))
md5Sum11, err := drivers.CreateObject("bucket", "object", "", md5Sum1, bytes.NewBufferString("one"))
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
c.Assert(md5Sum1hex, check.Equals, md5Sum11)
hasher2 := md5.New() hasher2 := md5.New()
hasher2.Write([]byte("three")) hasher2.Write([]byte("three"))
md5Sum2 := base64.StdEncoding.EncodeToString(hasher2.Sum(nil)) md5Sum2 := base64.StdEncoding.EncodeToString(hasher2.Sum(nil))
err = drivers.CreateObject("bucket", "object", "", md5Sum2, bytes.NewBufferString("three")) _, err = drivers.CreateObject("bucket", "object", "", md5Sum2, bytes.NewBufferString("three"))
c.Assert(err, check.Not(check.IsNil)) c.Assert(err, check.Not(check.IsNil))
var bytesBuffer bytes.Buffer var bytesBuffer bytes.Buffer
length, err := drivers.GetObject(&bytesBuffer, "bucket", "object") length, err := drivers.GetObject(&bytesBuffer, "bucket", "object")
c.Assert(length, check.Equals, int64(len("one"))) c.Assert(length, check.Equals, int64(len("one")))
@ -224,7 +229,7 @@ func testObjectOverwriteFails(c *check.C, create func() Driver) {
func testNonExistantBucketOperations(c *check.C, create func() Driver) { func testNonExistantBucketOperations(c *check.C, create func() Driver) {
drivers := create() drivers := create()
err := drivers.CreateObject("bucket", "object", "", "", bytes.NewBufferString("one")) _, err := drivers.CreateObject("bucket", "object", "", "", bytes.NewBufferString("one"))
c.Assert(err, check.Not(check.IsNil)) c.Assert(err, check.Not(check.IsNil))
} }
@ -253,9 +258,11 @@ func testPutObjectInSubdir(c *check.C, create func() Driver) {
hasher := md5.New() hasher := md5.New()
hasher.Write([]byte("hello world")) hasher.Write([]byte("hello world"))
md5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) md5Sum1 := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
err = drivers.CreateObject("bucket", "dir1/dir2/object", "", md5Sum, bytes.NewBufferString("hello world")) md5Sum1hex := hex.EncodeToString(hasher.Sum(nil))
md5Sum11, err := drivers.CreateObject("bucket", "dir1/dir2/object", "", md5Sum1, bytes.NewBufferString("hello world"))
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
c.Assert(md5Sum11, check.Equals, md5Sum1hex)
var bytesBuffer bytes.Buffer var bytesBuffer bytes.Buffer
length, err := drivers.GetObject(&bytesBuffer, "bucket", "dir1/dir2/object") length, err := drivers.GetObject(&bytesBuffer, "bucket", "dir1/dir2/object")
@ -349,7 +356,7 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Driver) {
err := drivers.CreateBucket("bucket", "") err := drivers.CreateBucket("bucket", "")
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
err = drivers.CreateObject("bucket", "dir1/dir2/object", "", "", bytes.NewBufferString("hello world")) _, err = drivers.CreateObject("bucket", "dir1/dir2/object", "", "", bytes.NewBufferString("hello world"))
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
var byteBuffer bytes.Buffer var byteBuffer bytes.Buffer
@ -393,7 +400,7 @@ func testDefaultContentType(c *check.C, create func() Driver) {
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
// test empty // test empty
err = drivers.CreateObject("bucket", "one", "", "", bytes.NewBufferString("one")) _, err = drivers.CreateObject("bucket", "one", "", "", bytes.NewBufferString("one"))
metadata, err := drivers.GetObjectMetadata("bucket", "one", "") metadata, err := drivers.GetObjectMetadata("bucket", "one", "")
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
c.Assert(metadata.ContentType, check.Equals, "application/octet-stream") c.Assert(metadata.ContentType, check.Equals, "application/octet-stream")
@ -417,8 +424,13 @@ func testContentMd5Set(c *check.C, create func() Driver) {
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
// test md5 invalid // test md5 invalid
err = drivers.CreateObject("bucket", "one", "", "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA", bytes.NewBufferString("one")) badmd5Sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA"
calculatedmd5sum, err := drivers.CreateObject("bucket", "one", "", badmd5Sum, bytes.NewBufferString("one"))
c.Assert(err, check.Not(check.IsNil)) c.Assert(err, check.Not(check.IsNil))
err = drivers.CreateObject("bucket", "two", "", "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA=", bytes.NewBufferString("one")) c.Assert(calculatedmd5sum, check.Not(check.Equals), badmd5Sum)
goodmd5sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA="
calculatedmd5sum, err = drivers.CreateObject("bucket", "two", "", goodmd5sum, bytes.NewBufferString("one"))
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
c.Assert(calculatedmd5sum, check.Equals, goodmd5sum)
} }

View file

@ -366,17 +366,17 @@ func (d donutDriver) ListObjects(bucketName string, resources drivers.BucketReso
} }
// CreateObject creates a new object // CreateObject creates a new object
func (d donutDriver) CreateObject(bucketName, objectName, contentType, expectedMD5Sum string, reader io.Reader) error { func (d donutDriver) CreateObject(bucketName, objectName, contentType, expectedMD5Sum string, reader io.Reader) (string, error) {
errParams := map[string]string{ errParams := map[string]string{
"bucketName": bucketName, "bucketName": bucketName,
"objectName": objectName, "objectName": objectName,
"contentType": contentType, "contentType": contentType,
} }
if !drivers.IsValidBucket(bucketName) || strings.Contains(bucketName, ".") { if !drivers.IsValidBucket(bucketName) || strings.Contains(bucketName, ".") {
return iodine.New(drivers.BucketNameInvalid{Bucket: bucketName}, nil) return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucketName}, nil)
} }
if !drivers.IsValidObject(objectName) || strings.TrimSpace(objectName) == "" { if !drivers.IsValidObject(objectName) || strings.TrimSpace(objectName) == "" {
return iodine.New(drivers.ObjectNameInvalid{Object: objectName}, nil) return "", iodine.New(drivers.ObjectNameInvalid{Object: objectName}, nil)
} }
if strings.TrimSpace(contentType) == "" { if strings.TrimSpace(contentType) == "" {
contentType = "application/octet-stream" contentType = "application/octet-stream"
@ -387,13 +387,13 @@ func (d donutDriver) CreateObject(bucketName, objectName, contentType, expectedM
if strings.TrimSpace(expectedMD5Sum) != "" { if strings.TrimSpace(expectedMD5Sum) != "" {
expectedMD5SumBytes, err := base64.StdEncoding.DecodeString(strings.TrimSpace(expectedMD5Sum)) expectedMD5SumBytes, err := base64.StdEncoding.DecodeString(strings.TrimSpace(expectedMD5Sum))
if err != nil { if err != nil {
return iodine.New(err, nil) return "", iodine.New(err, nil)
} }
expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes) expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
} }
err := d.donut.PutObject(bucketName, objectName, expectedMD5Sum, ioutil.NopCloser(reader), metadata) calculatedMD5Sum, err := d.donut.PutObject(bucketName, objectName, expectedMD5Sum, ioutil.NopCloser(reader), metadata)
if err != nil { if err != nil {
return iodine.New(err, errParams) return "", iodine.New(err, errParams)
} }
return nil return calculatedMD5Sum, nil
} }

View file

@ -37,7 +37,7 @@ type Driver interface {
GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error)
GetObjectMetadata(bucket string, object string, prefix string) (ObjectMetadata, error) GetObjectMetadata(bucket string, object string, prefix string) (ObjectMetadata, error)
ListObjects(bucket string, resources BucketResourcesMetadata) ([]ObjectMetadata, BucketResourcesMetadata, error) ListObjects(bucket string, resources BucketResourcesMetadata) ([]ObjectMetadata, BucketResourcesMetadata, error)
CreateObject(bucket string, key string, contentType string, md5sum string, data io.Reader) error CreateObject(bucket string, key string, contentType string, md5sum string, data io.Reader) (string, error)
} }
// BucketACL - bucket level access control // BucketACL - bucket level access control

View file

@ -199,26 +199,26 @@ func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) error {
} }
// CreateObject - PUT object to memory buffer // CreateObject - PUT object to memory buffer
func (memory *memoryDriver) CreateObject(bucket, key, contentType, expectedMD5Sum string, data io.Reader) error { func (memory *memoryDriver) CreateObject(bucket, key, contentType, expectedMD5Sum string, data io.Reader) (string, error) {
memory.lock.RLock() memory.lock.RLock()
if !drivers.IsValidBucket(bucket) { if !drivers.IsValidBucket(bucket) {
memory.lock.RUnlock() memory.lock.RUnlock()
return iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil) return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
} }
if !drivers.IsValidObject(key) { if !drivers.IsValidObject(key) {
memory.lock.RUnlock() memory.lock.RUnlock()
return iodine.New(drivers.ObjectNameInvalid{Object: key}, nil) return "", iodine.New(drivers.ObjectNameInvalid{Object: key}, nil)
} }
if _, ok := memory.storedBuckets[bucket]; ok == false { if _, ok := memory.storedBuckets[bucket]; ok == false {
memory.lock.RUnlock() memory.lock.RUnlock()
return iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) return "", iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
} }
storedBucket := memory.storedBuckets[bucket] storedBucket := memory.storedBuckets[bucket]
// get object key // get object key
objectKey := bucket + "/" + key objectKey := bucket + "/" + key
if _, ok := storedBucket.objectMetadata[objectKey]; ok == true { if _, ok := storedBucket.objectMetadata[objectKey]; ok == true {
memory.lock.RUnlock() memory.lock.RUnlock()
return iodine.New(drivers.ObjectExists{Bucket: bucket, Object: key}, nil) return "", iodine.New(drivers.ObjectExists{Bucket: bucket, Object: key}, nil)
} }
memory.lock.RUnlock() memory.lock.RUnlock()
@ -230,7 +230,7 @@ func (memory *memoryDriver) CreateObject(bucket, key, contentType, expectedMD5Su
expectedMD5SumBytes, err := base64.StdEncoding.DecodeString(strings.TrimSpace(expectedMD5Sum)) expectedMD5SumBytes, err := base64.StdEncoding.DecodeString(strings.TrimSpace(expectedMD5Sum))
if err != nil { if err != nil {
// pro-actively close the connection // pro-actively close the connection
return iodine.New(drivers.InvalidDigest{Md5: expectedMD5Sum}, nil) return "", iodine.New(drivers.InvalidDigest{Md5: expectedMD5Sum}, nil)
} }
expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes) expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
} }
@ -248,7 +248,7 @@ func (memory *memoryDriver) CreateObject(bucket, key, contentType, expectedMD5Su
if err != nil { if err != nil {
err := iodine.New(err, nil) err := iodine.New(err, nil)
log.Println(err) log.Println(err)
return err return "", err
} }
if uint64(totalLength)+memory.totalSize > memory.maxSize { if uint64(totalLength)+memory.totalSize > memory.maxSize {
memory.objects.RemoveOldest() memory.objects.RemoveOldest()
@ -263,7 +263,7 @@ func (memory *memoryDriver) CreateObject(bucket, key, contentType, expectedMD5Su
memory.lock.Lock() memory.lock.Lock()
defer memory.lock.Unlock() defer memory.lock.Unlock()
memory.objects.RemoveOldest() memory.objects.RemoveOldest()
return iodine.New(drivers.BadDigest{Md5: expectedMD5Sum, Bucket: bucket, Key: key}, nil) return "", iodine.New(drivers.BadDigest{Md5: expectedMD5Sum, Bucket: bucket, Key: key}, nil)
} }
} }
newObject := drivers.ObjectMetadata{ newObject := drivers.ObjectMetadata{
@ -292,7 +292,7 @@ func (memory *memoryDriver) CreateObject(bucket, key, contentType, expectedMD5Su
memory.lock.Unlock() memory.lock.Unlock()
// free memory if possible for kernel to reclaim // free memory if possible for kernel to reclaim
debug.FreeOSMemory() debug.FreeOSMemory()
return nil return newObject.Md5, nil
} }
// CreateBucket - create bucket in memory // CreateBucket - create bucket in memory

View file

@ -116,10 +116,11 @@ func (m *Driver) ListObjects(bucket string, resources drivers.BucketResourcesMet
} }
// CreateObject is a mock // CreateObject is a mock
func (m *Driver) CreateObject(bucket string, key string, contentType string, md5sum string, data io.Reader) error { func (m *Driver) CreateObject(bucket string, key string, contentType string, md5sum string, data io.Reader) (string, error) {
ret := m.Called(bucket, key, contentType, md5sum, data) ret := m.Called(bucket, key, contentType, md5sum, data)
r0 := ret.Error(0) r0 := ret.Get(0).(string)
r1 := ret.Error(1)
return r0 return r0, r1
} }