maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Put object on successful write returns full metadata, to avoid subsequent GetObjectMetadata() calls in driver

Browse source
This commit is contained in:
Harshavardhana 2015-06-29 11:10:52 -07:00
parent 6921328b93
commit 10c807f233
5 changed files with 40 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
pkg/storage/donut/bucket.go
View file

@ -217,19 +217,19 @@ func (b bucket) ReadObject(objectName string) (reader io.ReadCloser, size int64,
}
// WriteObject - write a new object into bucket
func (b bucket) WriteObject(objectName string, objectData io.Reader, expectedMD5Sum string, metadata map[string]string) (string, error) {
func (b bucket) WriteObject(objectName string, objectData io.Reader, expectedMD5Sum string, metadata map[string]string) (ObjectMetadata, error) {
b.lock.Lock()
defer b.lock.Unlock()
if objectName == "" || objectData == nil {
return "", iodine.New(InvalidArgument{}, nil)
return ObjectMetadata{}, iodine.New(InvalidArgument{}, nil)
}
writers, err := b.getDiskWriters(normalizeObjectName(objectName), "data")
if err != nil {
return "", iodine.New(err, nil)
return ObjectMetadata{}, iodine.New(err, nil)
}
sumMD5 := md5.New()
sum512 := sha512.New()
objMetadata := new(ObjectMetadata)
objMetadata := ObjectMetadata{}
objMetadata.Version = objectMetadataVersion
objMetadata.Created = time.Now().UTC()
// if total writers are only '1' do not compute erasure
@ -238,19 +238,19 @@ func (b bucket) WriteObject(objectName string, objectData io.Reader, expectedMD5
mw := io.MultiWriter(writers[0], sumMD5, sum512)
totalLength, err := io.Copy(mw, objectData)
if err != nil {
return "", iodine.New(err, nil)
return ObjectMetadata{}, iodine.New(err, nil)
}
objMetadata.Size = totalLength
case false:
// calculate data and parity dictated by total number of writers
k, m, err := b.getDataAndParity(len(writers))
if err != nil {
return "", iodine.New(err, nil)
return ObjectMetadata{}, iodine.New(err, nil)
}
// encoded data with k, m and write
chunkCount, totalLength, err := b.writeEncodedData(k, m, writers, objectData, sumMD5, sum512)
if err != nil {
return "", iodine.New(err, nil)
return ObjectMetadata{}, iodine.New(err, nil)
}
/// donutMetadata section
objMetadata.BlockSize = blockSize
@ -271,20 +271,19 @@ func (b bucket) WriteObject(objectName string, objectData io.Reader, expectedMD5
// Verify if the written object is equal to what is expected, only if it is requested as such
if strings.TrimSpace(expectedMD5Sum) != "" {
if err := b.isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), objMetadata.MD5Sum); err != nil {
return "", iodine.New(err, nil)
return ObjectMetadata{}, iodine.New(err, nil)
}
}
objMetadata.Metadata = metadata
// write object specific metadata
if err := b.writeObjectMetadata(normalizeObjectName(objectName), objMetadata); err != nil {
return "", iodine.New(err, nil)
return ObjectMetadata{}, iodine.New(err, nil)
}
// close all writers, when control flow reaches here
for _, writer := range writers {
writer.Close()
}
return objMetadata.MD5Sum, nil
return objMetadata, nil
}
// isMD5SumEqual - returns error if md5sum mismatches, other its `nil`
@ -307,8 +306,8 @@ func (b bucket) isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) error {
}
// writeObjectMetadata - write additional object metadata
func (b bucket) writeObjectMetadata(objectName string, objMetadata *ObjectMetadata) error {
if objMetadata == nil {
func (b bucket) writeObjectMetadata(objectName string, objMetadata ObjectMetadata) error {
if objMetadata.Object == "" {
return iodine.New(InvalidArgument{}, nil)
}
objMetadataWriters, err := b.getDiskWriters(objectName, objectMetadataConfig)
@ -320,7 +319,7 @@ func (b bucket) writeObjectMetadata(objectName string, objMetadata *ObjectMetada
}
for _, objMetadataWriter := range objMetadataWriters {
jenc := json.NewEncoder(objMetadataWriter)
if err := jenc.Encode(objMetadata); err != nil {
if err := jenc.Encode(&objMetadata); err != nil {
return iodine.New(err, nil)
}
}

22
pkg/storage/donut/donut.go
View file

@ -175,7 +175,7 @@ func (dt donut) ListObjects(bucket, prefix, marker, delimiter string, maxkeys in
}
// PutObject - put object
func (dt donut) PutObject(bucket, object, expectedMD5Sum string, reader io.Reader, metadata map[string]string) (string, error) {
func (dt donut) PutObject(bucket, object, expectedMD5Sum string, reader io.Reader, metadata map[string]string) (ObjectMetadata, error) {
dt.lock.Lock()
defer dt.lock.Unlock()
errParams := map[string]string{
@ -183,33 +183,33 @@ func (dt donut) PutObject(bucket, object, expectedMD5Sum string, reader io.Reade
"object": object,
}
if bucket == "" || strings.TrimSpace(bucket) == "" {
return "", iodine.New(InvalidArgument{}, errParams)
return ObjectMetadata{}, iodine.New(InvalidArgument{}, errParams)
}
if object == "" || strings.TrimSpace(object) == "" {
return "", iodine.New(InvalidArgument{}, errParams)
return ObjectMetadata{}, iodine.New(InvalidArgument{}, errParams)
}
if err := dt.listDonutBuckets(); err != nil {
return "", iodine.New(err, errParams)
return ObjectMetadata{}, iodine.New(err, errParams)
}
if _, ok := dt.buckets[bucket]; !ok {
return "", iodine.New(BucketNotFound{Bucket: bucket}, nil)
return ObjectMetadata{}, iodine.New(BucketNotFound{Bucket: bucket}, nil)
}
bucketMeta, err := dt.getDonutBucketMetadata()
if err != nil {
return "", iodine.New(err, errParams)
return ObjectMetadata{}, iodine.New(err, errParams)
}
if _, ok := bucketMeta.Buckets[bucket].BucketObjects[object]; ok {
return "", iodine.New(ObjectExists{Object: object}, errParams)
return ObjectMetadata{}, iodine.New(ObjectExists{Object: object}, errParams)
}
md5sum, err := dt.buckets[bucket].WriteObject(object, reader, expectedMD5Sum, metadata)
objMetadata, err := dt.buckets[bucket].WriteObject(object, reader, expectedMD5Sum, metadata)
if err != nil {
return "", iodine.New(err, errParams)
return ObjectMetadata{}, iodine.New(err, errParams)
}
bucketMeta.Buckets[bucket].BucketObjects[object] = 1
if err := dt.setDonutBucketMetadata(bucketMeta); err != nil {
return "", iodine.New(err, errParams)
return ObjectMetadata{}, iodine.New(err, errParams)
}
return md5sum, nil
return objMetadata, nil
}
// GetObject - get object

14
pkg/storage/donut/donut_test.go
View file

@ -195,13 +195,9 @@ func (s *MySuite) TestNewObjectMetadata(c *C) {
err = donut.MakeBucket("foo", "private")
c.Assert(err, IsNil)
calculatedMd5Sum, err := donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata)
objectMetadata, err := donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata)
c.Assert(err, IsNil)
c.Assert(calculatedMd5Sum, Equals, expectedMd5Sum)
objectMetadata, err := donut.GetObjectMetadata("foo", "obj")
c.Assert(err, IsNil)
c.Assert(objectMetadata.MD5Sum, Equals, expectedMd5Sum)
c.Assert(objectMetadata.Metadata["contentType"], Equals, metadata["contentType"])
c.Assert(objectMetadata.Metadata["foo"], Equals, metadata["foo"])
c.Assert(objectMetadata.Metadata["hello"], Equals, metadata["hello"])
@ -240,9 +236,9 @@ func (s *MySuite) TestNewObjectCanBeWritten(c *C) {
reader := ioutil.NopCloser(bytes.NewReader([]byte(data)))
metadata["contentLength"] = strconv.Itoa(len(data))
calculatedMd5Sum, err := donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata)
actualMetadata, err := donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata)
c.Assert(err, IsNil)
c.Assert(calculatedMd5Sum, Equals, expectedMd5Sum)
c.Assert(actualMetadata.MD5Sum, Equals, expectedMd5Sum)
reader, size, err := donut.GetObject("foo", "obj")
c.Assert(err, IsNil)
@ -253,7 +249,7 @@ func (s *MySuite) TestNewObjectCanBeWritten(c *C) {
c.Assert(err, IsNil)
c.Assert(actualData.Bytes(), DeepEquals, []byte(data))
actualMetadata, err := donut.GetObjectMetadata("foo", "obj")
actualMetadata, err = donut.GetObjectMetadata("foo", "obj")
c.Assert(err, IsNil)
c.Assert(expectedMd5Sum, Equals, actualMetadata.MD5Sum)
c.Assert(int64(len(data)), Equals, actualMetadata.Size)

2
pkg/storage/donut/interfaces.go
View file

@ -40,7 +40,7 @@ type ObjectStorage interface {
// Object operations
GetObject(bucket, object string) (io.ReadCloser, int64, error)
GetObjectMetadata(bucket, object string) (ObjectMetadata, error)
PutObject(bucket, object, expectedMD5Sum string, reader io.Reader, metadata map[string]string) (string, error)
PutObject(bucket, object, expectedMD5Sum string, reader io.Reader, metadata map[string]string) (ObjectMetadata, error)
}
// Management is a donut management system interface

23
pkg/storage/drivers/donut/donut.go
View file

@ -528,6 +528,10 @@ func (d donutDriver) CreateObject(bucketName, objectName, contentType, expectedM
"objectName": objectName,
"contentType": contentType,
}
if d.donut == nil {
return "", iodine.New(drivers.InternalError{}, errParams)
}
// TODO - Should be able to write bigger than cache
if size > int64(d.maxSize) {
generic := drivers.GenericObjectError{Bucket: bucketName, Object: objectName}
return "", iodine.New(drivers.EntityTooLarge{
@ -536,9 +540,6 @@ func (d donutDriver) CreateObject(bucketName, objectName, contentType, expectedM
MaxSize: strconv.FormatUint(d.maxSize, 10),
}, nil)
}
if d.donut == nil {
return "", iodine.New(drivers.InternalError{}, errParams)
}
if !drivers.IsValidBucket(bucketName) {
return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucketName}, nil)
}
@ -565,7 +566,7 @@ func (d donutDriver) CreateObject(bucketName, objectName, contentType, expectedM
expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
}
newReader := newProxyReader(reader)
calculatedMD5Sum, err := d.donut.PutObject(bucketName, objectName, expectedMD5Sum, newReader, metadata)
objMetadata, err := d.donut.PutObject(bucketName, objectName, expectedMD5Sum, newReader, metadata)
if err != nil {
switch iodine.ToError(err).(type) {
case donut.BadDigest:
@ -577,20 +578,16 @@ func (d donutDriver) CreateObject(bucketName, objectName, contentType, expectedM
// free up
newReader.readBytes = nil
go debug.FreeOSMemory()
objectMetadata, err := d.donut.GetObjectMetadata(bucketName, objectName)
if err != nil {
return "", iodine.New(err, nil)
}
newObject := drivers.ObjectMetadata{
Bucket: bucketName,
Key: objectName,
ContentType: objectMetadata.Metadata["contentType"],
Created: objectMetadata.Created,
Md5: calculatedMD5Sum,
Size: objectMetadata.Size,
ContentType: objMetadata.Metadata["contentType"],
Created: objMetadata.Created,
Md5: objMetadata.MD5Sum,
Size: objMetadata.Size,
}
storedBucket.objectMetadata[objectKey] = newObject
d.storedBuckets[bucketName] = storedBucket
return calculatedMD5Sum, nil
return newObject.Md5, nil
}