Merge pull request #677 from harshavardhana/pr_out_moving_to_more_typed_responses_this_removes_all_the_necessity_for_strconv
Donut moves to typed metadata, removing the necessity for strconv, parsing and string converstions
This commit is contained in:
commit
aab4937084
|
@ -49,20 +49,18 @@ type bucket struct {
|
|||
}
|
||||
|
||||
// newBucket - instantiate a new bucket
|
||||
func newBucket(bucketName, aclType, donutName string, nodes map[string]node) (bucket, map[string]string, error) {
|
||||
func newBucket(bucketName, aclType, donutName string, nodes map[string]node) (bucket, BucketMetadata, error) {
|
||||
errParams := map[string]string{
|
||||
"bucketName": bucketName,
|
||||
"donutName": donutName,
|
||||
"aclType": aclType,
|
||||
}
|
||||
if strings.TrimSpace(bucketName) == "" || strings.TrimSpace(donutName) == "" {
|
||||
return bucket{}, nil, iodine.New(InvalidArgument{}, errParams)
|
||||
return bucket{}, BucketMetadata{}, iodine.New(InvalidArgument{}, errParams)
|
||||
}
|
||||
bucketMetadata := make(map[string]string)
|
||||
bucketMetadata["acl"] = aclType
|
||||
t := time.Now().UTC()
|
||||
bucketMetadata["created"] = t.Format(time.RFC3339Nano)
|
||||
|
||||
b := bucket{}
|
||||
t := time.Now().UTC()
|
||||
b.name = bucketName
|
||||
b.acl = aclType
|
||||
b.time = t
|
||||
|
@ -70,7 +68,13 @@ func newBucket(bucketName, aclType, donutName string, nodes map[string]node) (bu
|
|||
b.nodes = nodes
|
||||
b.objects = make(map[string]object)
|
||||
b.lock = new(sync.RWMutex)
|
||||
return b, bucketMetadata, nil
|
||||
|
||||
metadata := BucketMetadata{}
|
||||
metadata.Name = bucketName
|
||||
metadata.ACL = aclType
|
||||
metadata.Created = t
|
||||
|
||||
return b, metadata, nil
|
||||
}
|
||||
|
||||
func (b bucket) getBucketName() string {
|
||||
|
@ -93,7 +97,7 @@ func (b bucket) getObjectName(fileName, diskPath, bucketPath string) (string, er
|
|||
return newObjectMetadata.Object, nil
|
||||
}
|
||||
|
||||
func (b bucket) GetObjectMetadata(objectName string) (*objectMetadata, error) {
|
||||
func (b bucket) GetObjectMetadata(objectName string) (ObjectMetadata, error) {
|
||||
return b.objects[objectName].GetObjectMetadata()
|
||||
}
|
||||
|
||||
|
@ -183,14 +187,14 @@ func (b bucket) ReadObject(objectName string) (reader io.ReadCloser, size int64,
|
|||
if !ok {
|
||||
return nil, 0, iodine.New(ObjectNotFound{Object: objectName}, nil)
|
||||
}
|
||||
// verify if donutObjectMetadata is readable, before we server the request
|
||||
donutObjMetadata, err := object.GetDonutObjectMetadata()
|
||||
// verify if sysObjectMetadata is readable, before we server the request
|
||||
sysObjMetadata, err := object.GetSystemObjectMetadata()
|
||||
if err != nil {
|
||||
return nil, 0, iodine.New(err, nil)
|
||||
}
|
||||
// read and reply back to GetObject() request in a go-routine
|
||||
go b.readEncodedData(b.normalizeObjectName(objectName), writer, donutObjMetadata)
|
||||
return reader, donutObjMetadata.Size, nil
|
||||
go b.readEncodedData(b.normalizeObjectName(objectName), writer, sysObjMetadata)
|
||||
return reader, sysObjMetadata.Size, nil
|
||||
}
|
||||
|
||||
// WriteObject - write a new object into bucket
|
||||
|
@ -207,10 +211,10 @@ func (b bucket) WriteObject(objectName string, objectData io.Reader, expectedMD5
|
|||
sumMD5 := md5.New()
|
||||
sum512 := sha512.New()
|
||||
|
||||
objectMetadata := new(objectMetadata)
|
||||
donutObjectMetadata := new(donutObjectMetadata)
|
||||
objectMetadata.Version = objectMetadataVersion
|
||||
donutObjectMetadata.Version = donutObjectMetadataVersion
|
||||
objMetadata := new(ObjectMetadata)
|
||||
sysObjMetadata := new(SystemObjectMetadata)
|
||||
objMetadata.Version = objectMetadataVersion
|
||||
sysObjMetadata.Version = systemObjectMetadataVersion
|
||||
size := metadata["contentLength"]
|
||||
sizeInt, err := strconv.ParseInt(size, 10, 64)
|
||||
if err != nil {
|
||||
|
@ -225,8 +229,8 @@ func (b bucket) WriteObject(objectName string, objectData io.Reader, expectedMD5
|
|||
if err != nil {
|
||||
return "", iodine.New(err, nil)
|
||||
}
|
||||
donutObjectMetadata.Size = totalLength
|
||||
objectMetadata.Size = totalLength
|
||||
sysObjMetadata.Size = totalLength
|
||||
objMetadata.Size = totalLength
|
||||
case false:
|
||||
// calculate data and parity dictated by total number of writers
|
||||
k, m, err := b.getDataAndParity(len(writers))
|
||||
|
@ -239,50 +243,50 @@ func (b bucket) WriteObject(objectName string, objectData io.Reader, expectedMD5
|
|||
return "", iodine.New(err, nil)
|
||||
}
|
||||
/// donutMetadata section
|
||||
donutObjectMetadata.BlockSize = 10 * 1024 * 1024
|
||||
donutObjectMetadata.ChunkCount = chunkCount
|
||||
donutObjectMetadata.DataDisks = k
|
||||
donutObjectMetadata.ParityDisks = m
|
||||
donutObjectMetadata.ErasureTechnique = "Cauchy"
|
||||
donutObjectMetadata.Size = int64(totalLength)
|
||||
// keep size inside objectMetadata as well for Object API requests
|
||||
objectMetadata.Size = int64(totalLength)
|
||||
sysObjMetadata.BlockSize = 10 * 1024 * 1024
|
||||
sysObjMetadata.ChunkCount = chunkCount
|
||||
sysObjMetadata.DataDisks = k
|
||||
sysObjMetadata.ParityDisks = m
|
||||
sysObjMetadata.ErasureTechnique = "Cauchy"
|
||||
sysObjMetadata.Size = int64(totalLength)
|
||||
// keep size inside ObjectMetadata as well for Object API requests
|
||||
objMetadata.Size = int64(totalLength)
|
||||
}
|
||||
objectMetadata.Bucket = b.getBucketName()
|
||||
objectMetadata.Object = objectName
|
||||
objectMetadata.Metadata = metadata
|
||||
objMetadata.Bucket = b.getBucketName()
|
||||
objMetadata.Object = objectName
|
||||
objMetadata.Metadata = metadata
|
||||
dataMD5sum := sumMD5.Sum(nil)
|
||||
dataSHA512sum := sum512.Sum(nil)
|
||||
objectMetadata.Created = time.Now().UTC()
|
||||
objMetadata.Created = time.Now().UTC()
|
||||
|
||||
// keeping md5sum for the object in two different places
|
||||
// one for object storage and another is for internal use
|
||||
hexMD5Sum := hex.EncodeToString(dataMD5sum)
|
||||
hex512Sum := hex.EncodeToString(dataSHA512sum)
|
||||
objectMetadata.MD5Sum = hexMD5Sum
|
||||
objectMetadata.SHA512Sum = hex512Sum
|
||||
donutObjectMetadata.MD5Sum = hexMD5Sum
|
||||
donutObjectMetadata.SHA512Sum = hex512Sum
|
||||
objMetadata.MD5Sum = hexMD5Sum
|
||||
objMetadata.SHA512Sum = hex512Sum
|
||||
sysObjMetadata.MD5Sum = hexMD5Sum
|
||||
sysObjMetadata.SHA512Sum = hex512Sum
|
||||
|
||||
// Verify if the written object is equal to what is expected, only if it is requested as such
|
||||
if strings.TrimSpace(expectedMD5Sum) != "" {
|
||||
if err := b.isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), objectMetadata.MD5Sum); err != nil {
|
||||
if err := b.isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), objMetadata.MD5Sum); err != nil {
|
||||
return "", iodine.New(err, nil)
|
||||
}
|
||||
}
|
||||
// write donut specific metadata
|
||||
if err := b.writeDonutObjectMetadata(b.normalizeObjectName(objectName), donutObjectMetadata); err != nil {
|
||||
if err := b.writeSystemObjectMetadata(b.normalizeObjectName(objectName), sysObjMetadata); err != nil {
|
||||
return "", iodine.New(err, nil)
|
||||
}
|
||||
// write object specific metadata
|
||||
if err := b.writeObjectMetadata(b.normalizeObjectName(objectName), objectMetadata); err != nil {
|
||||
if err := b.writeObjectMetadata(b.normalizeObjectName(objectName), objMetadata); err != nil {
|
||||
return "", iodine.New(err, nil)
|
||||
}
|
||||
// close all writers, when control flow reaches here
|
||||
for _, writer := range writers {
|
||||
writer.Close()
|
||||
}
|
||||
return objectMetadata.MD5Sum, nil
|
||||
return objMetadata.MD5Sum, nil
|
||||
}
|
||||
|
||||
// isMD5SumEqual - returns error if md5sum mismatches, other its `nil`
|
||||
|
@ -305,41 +309,41 @@ func (b bucket) isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) error {
|
|||
}
|
||||
|
||||
// writeObjectMetadata - write additional object metadata
|
||||
func (b bucket) writeObjectMetadata(objectName string, objectMetadata *objectMetadata) error {
|
||||
if objectMetadata == nil {
|
||||
func (b bucket) writeObjectMetadata(objectName string, objMetadata *ObjectMetadata) error {
|
||||
if objMetadata == nil {
|
||||
return iodine.New(InvalidArgument{}, nil)
|
||||
}
|
||||
objectMetadataWriters, err := b.getDiskWriters(objectName, objectMetadataConfig)
|
||||
objMetadataWriters, err := b.getDiskWriters(objectName, objectMetadataConfig)
|
||||
if err != nil {
|
||||
return iodine.New(err, nil)
|
||||
}
|
||||
for _, objectMetadataWriter := range objectMetadataWriters {
|
||||
defer objectMetadataWriter.Close()
|
||||
for _, objMetadataWriter := range objMetadataWriters {
|
||||
defer objMetadataWriter.Close()
|
||||
}
|
||||
for _, objectMetadataWriter := range objectMetadataWriters {
|
||||
jenc := json.NewEncoder(objectMetadataWriter)
|
||||
if err := jenc.Encode(objectMetadata); err != nil {
|
||||
for _, objMetadataWriter := range objMetadataWriters {
|
||||
jenc := json.NewEncoder(objMetadataWriter)
|
||||
if err := jenc.Encode(objMetadata); err != nil {
|
||||
return iodine.New(err, nil)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeDonutObjectMetadata - write donut related object metadata
|
||||
func (b bucket) writeDonutObjectMetadata(objectName string, donutObjectMetadata *donutObjectMetadata) error {
|
||||
if donutObjectMetadata == nil {
|
||||
// writeSystemObjectMetadata - write donut related object metadata
|
||||
func (b bucket) writeSystemObjectMetadata(objectName string, sysObjMetadata *SystemObjectMetadata) error {
|
||||
if sysObjMetadata == nil {
|
||||
return iodine.New(InvalidArgument{}, nil)
|
||||
}
|
||||
donutObjectMetadataWriters, err := b.getDiskWriters(objectName, donutObjectMetadataConfig)
|
||||
sysObjMetadataWriters, err := b.getDiskWriters(objectName, sysObjectMetadataConfig)
|
||||
if err != nil {
|
||||
return iodine.New(err, nil)
|
||||
}
|
||||
for _, donutObjectMetadataWriter := range donutObjectMetadataWriters {
|
||||
defer donutObjectMetadataWriter.Close()
|
||||
for _, sysObjMetadataWriter := range sysObjMetadataWriters {
|
||||
defer sysObjMetadataWriter.Close()
|
||||
}
|
||||
for _, donutObjectMetadataWriter := range donutObjectMetadataWriters {
|
||||
jenc := json.NewEncoder(donutObjectMetadataWriter)
|
||||
if err := jenc.Encode(donutObjectMetadata); err != nil {
|
||||
for _, sysObjMetadataWriter := range sysObjMetadataWriters {
|
||||
jenc := json.NewEncoder(sysObjMetadataWriter)
|
||||
if err := jenc.Encode(sysObjMetadata); err != nil {
|
||||
return iodine.New(err, nil)
|
||||
}
|
||||
}
|
||||
|
@ -403,8 +407,8 @@ func (b bucket) writeEncodedData(k, m uint8, writers []io.WriteCloser, objectDat
|
|||
}
|
||||
|
||||
// readEncodedData -
|
||||
func (b bucket) readEncodedData(objectName string, writer *io.PipeWriter, donutObjMetadata *donutObjectMetadata) {
|
||||
expectedMd5sum, err := hex.DecodeString(donutObjMetadata.MD5Sum)
|
||||
func (b bucket) readEncodedData(objectName string, writer *io.PipeWriter, sysObjMetadata SystemObjectMetadata) {
|
||||
expectedMd5sum, err := hex.DecodeString(sysObjMetadata.MD5Sum)
|
||||
if err != nil {
|
||||
writer.CloseWithError(iodine.New(err, nil))
|
||||
return
|
||||
|
@ -421,18 +425,18 @@ func (b bucket) readEncodedData(objectName string, writer *io.PipeWriter, donutO
|
|||
mwriter := io.MultiWriter(writer, hasher)
|
||||
switch len(readers) == 1 {
|
||||
case false:
|
||||
if donutObjMetadata.ErasureTechnique == "" {
|
||||
if sysObjMetadata.ErasureTechnique == "" {
|
||||
writer.CloseWithError(iodine.New(MissingErasureTechnique{}, nil))
|
||||
return
|
||||
}
|
||||
encoder, err := newEncoder(donutObjMetadata.DataDisks, donutObjMetadata.ParityDisks, donutObjMetadata.ErasureTechnique)
|
||||
encoder, err := newEncoder(sysObjMetadata.DataDisks, sysObjMetadata.ParityDisks, sysObjMetadata.ErasureTechnique)
|
||||
if err != nil {
|
||||
writer.CloseWithError(iodine.New(err, nil))
|
||||
return
|
||||
}
|
||||
totalLeft := donutObjMetadata.Size
|
||||
for i := 0; i < donutObjMetadata.ChunkCount; i++ {
|
||||
decodedData, err := b.decodeEncodedData(totalLeft, int64(donutObjMetadata.BlockSize), readers, encoder, writer)
|
||||
totalLeft := sysObjMetadata.Size
|
||||
for i := 0; i < sysObjMetadata.ChunkCount; i++ {
|
||||
decodedData, err := b.decodeEncodedData(totalLeft, int64(sysObjMetadata.BlockSize), readers, encoder, writer)
|
||||
if err != nil {
|
||||
writer.CloseWithError(iodine.New(err, nil))
|
||||
return
|
||||
|
@ -442,7 +446,7 @@ func (b bucket) readEncodedData(objectName string, writer *io.PipeWriter, donutO
|
|||
writer.CloseWithError(iodine.New(err, nil))
|
||||
return
|
||||
}
|
||||
totalLeft = totalLeft - int64(donutObjMetadata.BlockSize)
|
||||
totalLeft = totalLeft - int64(sysObjMetadata.BlockSize)
|
||||
}
|
||||
case true:
|
||||
_, err := io.Copy(writer, readers[0])
|
||||
|
|
|
@ -18,8 +18,8 @@ package donut
|
|||
|
||||
import "time"
|
||||
|
||||
// objectMetadata object specific metadata per object
|
||||
type objectMetadata struct {
|
||||
// ObjectMetadata object specific metadata per object
|
||||
type ObjectMetadata struct {
|
||||
// version
|
||||
Version string `json:"version"`
|
||||
|
||||
|
@ -37,8 +37,8 @@ type objectMetadata struct {
|
|||
Metadata map[string]string `json:"metadata"`
|
||||
}
|
||||
|
||||
// donutObjectMetadata container for donut specific internal metadata per object
|
||||
type donutObjectMetadata struct {
|
||||
// SystemObjectMetadata container for donut system specific metadata per object
|
||||
type SystemObjectMetadata struct {
|
||||
// version
|
||||
Version string `json:"version"`
|
||||
|
||||
|
@ -57,14 +57,21 @@ type donutObjectMetadata struct {
|
|||
SHA512Sum string `json:"sys.sha512sum"`
|
||||
}
|
||||
|
||||
// donutMetadata container for donut level metadata
|
||||
type donutMetadata struct {
|
||||
// Metadata container for donut metadata
|
||||
type Metadata struct {
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
// bucketMetadata container for bucket level metadata
|
||||
type bucketMetadata struct {
|
||||
// AllBuckets container for all buckets
|
||||
type AllBuckets struct {
|
||||
Buckets map[string]BucketMetadata
|
||||
}
|
||||
|
||||
// BucketMetadata container for bucket level metadata
|
||||
type BucketMetadata struct {
|
||||
Version string `json:"version"`
|
||||
Name string `json:"name"`
|
||||
ACL string `json:"acl"`
|
||||
Created time.Time `json:"created"`
|
||||
Metadata map[string]string `json:"metadata"`
|
||||
}
|
||||
|
|
|
@ -25,7 +25,6 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/pkg/iodine"
|
||||
)
|
||||
|
@ -40,17 +39,18 @@ type donut struct {
|
|||
|
||||
// config files used inside Donut
|
||||
const (
|
||||
// donut object metadata and config
|
||||
donutObjectMetadataConfig = "donutObjectMetadata.json"
|
||||
donutConfig = "donutMetadata.json"
|
||||
// donut system object metadata
|
||||
sysObjectMetadataConfig = "sysObjectMetadata.json"
|
||||
// donut system config
|
||||
donutConfig = "donutConfig.json"
|
||||
|
||||
// bucket, object metadata
|
||||
bucketMetadataConfig = "bucketMetadata.json"
|
||||
objectMetadataConfig = "objectMetadata.json"
|
||||
|
||||
// versions
|
||||
objectMetadataVersion = "1.0"
|
||||
donutObjectMetadataVersion = "1.0"
|
||||
objectMetadataVersion = "1.0.0"
|
||||
systemObjectMetadataVersion = "1.0.0"
|
||||
)
|
||||
|
||||
// attachDonutNode - wrapper function to instantiate a new node for associatedt donut
|
||||
|
@ -98,24 +98,24 @@ func (dt donut) MakeBucket(bucket, acl string) error {
|
|||
}
|
||||
|
||||
// GetBucketMetadata - get bucket metadata
|
||||
func (dt donut) GetBucketMetadata(bucket string) (map[string]string, error) {
|
||||
func (dt donut) GetBucketMetadata(bucketName string) (BucketMetadata, error) {
|
||||
dt.lock.RLock()
|
||||
defer dt.lock.RUnlock()
|
||||
if err := dt.listDonutBuckets(); err != nil {
|
||||
return nil, iodine.New(err, nil)
|
||||
return BucketMetadata{}, iodine.New(err, nil)
|
||||
}
|
||||
if _, ok := dt.buckets[bucket]; !ok {
|
||||
return nil, iodine.New(BucketNotFound{Bucket: bucket}, nil)
|
||||
if _, ok := dt.buckets[bucketName]; !ok {
|
||||
return BucketMetadata{}, iodine.New(BucketNotFound{Bucket: bucketName}, nil)
|
||||
}
|
||||
metadata, err := dt.getDonutBucketMetadata()
|
||||
if err != nil {
|
||||
return nil, iodine.New(err, nil)
|
||||
return BucketMetadata{}, iodine.New(err, nil)
|
||||
}
|
||||
return metadata[bucket], nil
|
||||
return metadata.Buckets[bucketName], nil
|
||||
}
|
||||
|
||||
// SetBucketMetadata - set bucket metadata
|
||||
func (dt donut) SetBucketMetadata(bucket string, bucketMetadata map[string]string) error {
|
||||
func (dt donut) SetBucketMetadata(bucketName string, bucketMetadata map[string]string) error {
|
||||
dt.lock.Lock()
|
||||
defer dt.lock.Unlock()
|
||||
if err := dt.listDonutBuckets(); err != nil {
|
||||
|
@ -125,28 +125,31 @@ func (dt donut) SetBucketMetadata(bucket string, bucketMetadata map[string]strin
|
|||
if err != nil {
|
||||
return iodine.New(err, nil)
|
||||
}
|
||||
oldBucketMetadata := metadata[bucket]
|
||||
// TODO ignore rest of the keys for now, only mutable data is "acl"
|
||||
oldBucketMetadata["acl"] = bucketMetadata["acl"]
|
||||
metadata[bucket] = oldBucketMetadata
|
||||
oldBucketMetadata := metadata.Buckets[bucketName]
|
||||
acl, ok := bucketMetadata["acl"]
|
||||
if !ok {
|
||||
return iodine.New(InvalidArgument{}, nil)
|
||||
}
|
||||
oldBucketMetadata.ACL = acl
|
||||
metadata.Buckets[bucketName] = oldBucketMetadata
|
||||
return dt.setDonutBucketMetadata(metadata)
|
||||
}
|
||||
|
||||
// ListBuckets - return list of buckets
|
||||
func (dt donut) ListBuckets() (metadata map[string]map[string]string, err error) {
|
||||
func (dt donut) ListBuckets() (map[string]BucketMetadata, error) {
|
||||
dt.lock.RLock()
|
||||
defer dt.lock.RUnlock()
|
||||
if err := dt.listDonutBuckets(); err != nil {
|
||||
return nil, iodine.New(err, nil)
|
||||
}
|
||||
metadata, err = dt.getDonutBucketMetadata()
|
||||
metadata, err := dt.getDonutBucketMetadata()
|
||||
if err != nil {
|
||||
// intentionally left out the error when Donut is empty
|
||||
// but we need to revisit this area in future - since we need
|
||||
// to figure out between acceptable and unacceptable errors
|
||||
return make(map[string]map[string]string), nil
|
||||
return make(map[string]BucketMetadata), nil
|
||||
}
|
||||
return metadata, nil
|
||||
return metadata.Buckets, nil
|
||||
}
|
||||
|
||||
// ListObjects - return list of objects
|
||||
|
@ -233,7 +236,7 @@ func (dt donut) GetObject(bucket, object string) (reader io.ReadCloser, size int
|
|||
}
|
||||
|
||||
// GetObjectMetadata - get object metadata
|
||||
func (dt donut) GetObjectMetadata(bucket, object string) (map[string]string, error) {
|
||||
func (dt donut) GetObjectMetadata(bucket, object string) (ObjectMetadata, error) {
|
||||
dt.lock.RLock()
|
||||
defer dt.lock.RUnlock()
|
||||
errParams := map[string]string{
|
||||
|
@ -241,10 +244,10 @@ func (dt donut) GetObjectMetadata(bucket, object string) (map[string]string, err
|
|||
"object": object,
|
||||
}
|
||||
if err := dt.listDonutBuckets(); err != nil {
|
||||
return nil, iodine.New(err, errParams)
|
||||
return ObjectMetadata{}, iodine.New(err, errParams)
|
||||
}
|
||||
if _, ok := dt.buckets[bucket]; !ok {
|
||||
return nil, iodine.New(BucketNotFound{Bucket: bucket}, errParams)
|
||||
return ObjectMetadata{}, iodine.New(BucketNotFound{Bucket: bucket}, errParams)
|
||||
}
|
||||
//
|
||||
// there is a potential issue here, if the object comes after the truncated list
|
||||
|
@ -253,26 +256,14 @@ func (dt donut) GetObjectMetadata(bucket, object string) (map[string]string, err
|
|||
// will fix it when we bring in persistent json into Donut - TODO
|
||||
objectList, _, _, err := dt.buckets[bucket].ListObjects("", "", "", 1000)
|
||||
if err != nil {
|
||||
return nil, iodine.New(err, errParams)
|
||||
return ObjectMetadata{}, iodine.New(err, errParams)
|
||||
}
|
||||
for _, objectName := range objectList {
|
||||
if objectName == object {
|
||||
objectMetadataMap := make(map[string]string)
|
||||
objectMetadata, err := dt.buckets[bucket].GetObjectMetadata(object)
|
||||
if err != nil {
|
||||
return nil, iodine.New(err, nil)
|
||||
}
|
||||
objectMetadataMap["created"] = objectMetadata.Created.Format(time.RFC3339Nano)
|
||||
objectMetadataMap["size"] = strconv.FormatInt(objectMetadata.Size, 10)
|
||||
objectMetadataMap["md5"] = objectMetadata.MD5Sum
|
||||
objectMetadataMap["version"] = objectMetadata.Version
|
||||
for k, v := range objectMetadata.Metadata {
|
||||
objectMetadataMap[k] = v
|
||||
}
|
||||
return objectMetadataMap, nil
|
||||
return dt.buckets[bucket].GetObjectMetadata(object)
|
||||
}
|
||||
}
|
||||
return nil, iodine.New(ObjectNotFound{Object: object}, errParams)
|
||||
return ObjectMetadata{}, iodine.New(ObjectNotFound{Object: object}, errParams)
|
||||
}
|
||||
|
||||
// getDiskWriters -
|
||||
|
@ -315,7 +306,7 @@ func (dt donut) getBucketMetadataReaders() ([]io.ReadCloser, error) {
|
|||
}
|
||||
|
||||
//
|
||||
func (dt donut) setDonutBucketMetadata(metadata map[string]map[string]string) error {
|
||||
func (dt donut) setDonutBucketMetadata(metadata *AllBuckets) error {
|
||||
writers, err := dt.getBucketMetadataWriters()
|
||||
if err != nil {
|
||||
return iodine.New(err, nil)
|
||||
|
@ -332,8 +323,8 @@ func (dt donut) setDonutBucketMetadata(metadata map[string]map[string]string) er
|
|||
return nil
|
||||
}
|
||||
|
||||
func (dt donut) getDonutBucketMetadata() (map[string]map[string]string, error) {
|
||||
metadata := make(map[string]map[string]string)
|
||||
func (dt donut) getDonutBucketMetadata() (*AllBuckets, error) {
|
||||
metadata := new(AllBuckets)
|
||||
readers, err := dt.getBucketMetadataReaders()
|
||||
if err != nil {
|
||||
return nil, iodine.New(err, nil)
|
||||
|
@ -343,7 +334,7 @@ func (dt donut) getDonutBucketMetadata() (map[string]map[string]string, error) {
|
|||
}
|
||||
for _, reader := range readers {
|
||||
jenc := json.NewDecoder(reader)
|
||||
if err := jenc.Decode(&metadata); err != nil {
|
||||
if err := jenc.Decode(metadata); err != nil {
|
||||
return nil, iodine.New(err, nil)
|
||||
}
|
||||
}
|
||||
|
@ -380,8 +371,9 @@ func (dt donut) makeDonutBucket(bucketName, acl string) error {
|
|||
metadata, err := dt.getDonutBucketMetadata()
|
||||
if err != nil {
|
||||
if os.IsNotExist(iodine.ToError(err)) {
|
||||
metadata := make(map[string]map[string]string)
|
||||
metadata[bucketName] = bucketMetadata
|
||||
metadata := new(AllBuckets)
|
||||
metadata.Buckets = make(map[string]BucketMetadata)
|
||||
metadata.Buckets[bucketName] = bucketMetadata
|
||||
err = dt.setDonutBucketMetadata(metadata)
|
||||
if err != nil {
|
||||
return iodine.New(err, nil)
|
||||
|
@ -390,7 +382,7 @@ func (dt donut) makeDonutBucket(bucketName, acl string) error {
|
|||
}
|
||||
return iodine.New(err, nil)
|
||||
}
|
||||
metadata[bucketName] = bucketMetadata
|
||||
metadata.Buckets[bucketName] = bucketMetadata
|
||||
err = dt.setDonutBucketMetadata(metadata)
|
||||
if err != nil {
|
||||
return iodine.New(err, nil)
|
||||
|
|
|
@ -26,7 +26,6 @@ import (
|
|||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
. "github.com/minio/check"
|
||||
)
|
||||
|
@ -113,7 +112,7 @@ func (s *MySuite) TestMakeBucketAndList(c *C) {
|
|||
buckets, err := donut.ListBuckets()
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(len(buckets), Equals, 1)
|
||||
c.Assert(buckets["foo"]["acl"], Equals, "private")
|
||||
c.Assert(buckets["foo"].ACL, Equals, "private")
|
||||
}
|
||||
|
||||
// test re-create bucket
|
||||
|
@ -202,9 +201,9 @@ func (s *MySuite) TestNewObjectMetadata(c *C) {
|
|||
objectMetadata, err := donut.GetObjectMetadata("foo", "obj")
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
c.Assert(objectMetadata["contentType"], Equals, metadata["contentType"])
|
||||
c.Assert(objectMetadata["foo"], Equals, metadata["foo"])
|
||||
c.Assert(objectMetadata["hello"], Equals, metadata["hello"])
|
||||
c.Assert(objectMetadata.Metadata["contentType"], Equals, metadata["contentType"])
|
||||
c.Assert(objectMetadata.Metadata["foo"], Equals, metadata["foo"])
|
||||
c.Assert(objectMetadata.Metadata["hello"], Equals, metadata["hello"])
|
||||
}
|
||||
|
||||
// test create object fails without name
|
||||
|
@ -255,11 +254,9 @@ func (s *MySuite) TestNewObjectCanBeWritten(c *C) {
|
|||
|
||||
actualMetadata, err := donut.GetObjectMetadata("foo", "obj")
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(expectedMd5Sum, Equals, actualMetadata["md5"])
|
||||
c.Assert(strconv.Itoa(len(data)), Equals, actualMetadata["size"])
|
||||
c.Assert("1.0", Equals, actualMetadata["version"])
|
||||
_, err = time.Parse(time.RFC3339Nano, actualMetadata["created"])
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(expectedMd5Sum, Equals, actualMetadata.MD5Sum)
|
||||
c.Assert(int64(len(data)), Equals, actualMetadata.Size)
|
||||
c.Assert("1.0.0", Equals, actualMetadata.Version)
|
||||
}
|
||||
|
||||
// test list objects
|
||||
|
|
|
@ -29,9 +29,9 @@ type Donut interface {
|
|||
// ObjectStorage is a donut object storage interface
|
||||
type ObjectStorage interface {
|
||||
// Storage service operations
|
||||
GetBucketMetadata(bucket string) (map[string]string, error)
|
||||
GetBucketMetadata(bucket string) (BucketMetadata, error)
|
||||
SetBucketMetadata(bucket string, metadata map[string]string) error
|
||||
ListBuckets() (map[string]map[string]string, error)
|
||||
ListBuckets() (map[string]BucketMetadata, error)
|
||||
MakeBucket(bucket, acl string) error
|
||||
|
||||
// Bucket operations
|
||||
|
@ -39,7 +39,7 @@ type ObjectStorage interface {
|
|||
|
||||
// Object operations
|
||||
GetObject(bucket, object string) (io.ReadCloser, int64, error)
|
||||
GetObjectMetadata(bucket, object string) (map[string]string, error)
|
||||
GetObjectMetadata(bucket, object string) (ObjectMetadata, error)
|
||||
PutObject(bucket, object, expectedMD5Sum string, reader io.ReadCloser, metadata map[string]string) (string, error)
|
||||
}
|
||||
|
||||
|
|
|
@ -41,26 +41,26 @@ func newObject(objectName, p string) (object, error) {
|
|||
return o, nil
|
||||
}
|
||||
|
||||
func (o object) GetObjectMetadata() (*objectMetadata, error) {
|
||||
objectMetadata := new(objectMetadata)
|
||||
objectMetadataBytes, err := ioutil.ReadFile(filepath.Join(o.objectPath, objectMetadataConfig))
|
||||
func (o object) GetObjectMetadata() (ObjectMetadata, error) {
|
||||
objMetadata := ObjectMetadata{}
|
||||
objMetadataBytes, err := ioutil.ReadFile(filepath.Join(o.objectPath, objectMetadataConfig))
|
||||
if err != nil {
|
||||
return nil, iodine.New(ObjectNotFound{Object: o.name}, nil)
|
||||
return ObjectMetadata{}, iodine.New(ObjectNotFound{Object: o.name}, nil)
|
||||
}
|
||||
if err := json.Unmarshal(objectMetadataBytes, objectMetadata); err != nil {
|
||||
return nil, iodine.New(err, nil)
|
||||
if err := json.Unmarshal(objMetadataBytes, &objMetadata); err != nil {
|
||||
return ObjectMetadata{}, iodine.New(err, nil)
|
||||
}
|
||||
return objectMetadata, nil
|
||||
return objMetadata, nil
|
||||
}
|
||||
|
||||
func (o object) GetDonutObjectMetadata() (*donutObjectMetadata, error) {
|
||||
donutObjectMetadata := new(donutObjectMetadata)
|
||||
donutObjectMetadataBytes, err := ioutil.ReadFile(filepath.Join(o.objectPath, donutObjectMetadataConfig))
|
||||
func (o object) GetSystemObjectMetadata() (SystemObjectMetadata, error) {
|
||||
sysObjMetadata := SystemObjectMetadata{}
|
||||
sysObjMetadataBytes, err := ioutil.ReadFile(filepath.Join(o.objectPath, sysObjectMetadataConfig))
|
||||
if err != nil {
|
||||
return nil, iodine.New(ObjectNotFound{Object: o.name}, nil)
|
||||
return SystemObjectMetadata{}, iodine.New(ObjectNotFound{Object: o.name}, nil)
|
||||
}
|
||||
if err := json.Unmarshal(donutObjectMetadataBytes, donutObjectMetadata); err != nil {
|
||||
return nil, iodine.New(err, nil)
|
||||
if err := json.Unmarshal(sysObjMetadataBytes, &sysObjMetadata); err != nil {
|
||||
return SystemObjectMetadata{}, iodine.New(err, nil)
|
||||
}
|
||||
return donutObjectMetadata, nil
|
||||
return sysObjMetadata, nil
|
||||
}
|
||||
|
|
|
@ -26,7 +26,6 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"io/ioutil"
|
||||
|
||||
|
@ -141,13 +140,9 @@ func (d donutDriver) ListBuckets() (results []drivers.BucketMetadata, err error)
|
|||
return nil, err
|
||||
}
|
||||
for name, metadata := range buckets {
|
||||
created, err := time.Parse(time.RFC3339Nano, metadata["created"])
|
||||
if err != nil {
|
||||
return nil, iodine.New(err, nil)
|
||||
}
|
||||
result := drivers.BucketMetadata{
|
||||
Name: name,
|
||||
Created: created,
|
||||
Created: metadata.Created,
|
||||
}
|
||||
results = append(results, result)
|
||||
}
|
||||
|
@ -195,18 +190,10 @@ func (d donutDriver) GetBucketMetadata(bucketName string) (drivers.BucketMetadat
|
|||
if err != nil {
|
||||
return drivers.BucketMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucketName}, nil)
|
||||
}
|
||||
created, err := time.Parse(time.RFC3339Nano, metadata["created"])
|
||||
if err != nil {
|
||||
return drivers.BucketMetadata{}, iodine.New(err, nil)
|
||||
}
|
||||
acl, ok := metadata["acl"]
|
||||
if !ok {
|
||||
return drivers.BucketMetadata{}, iodine.New(drivers.BackendCorrupted{}, nil)
|
||||
}
|
||||
bucketMetadata := drivers.BucketMetadata{
|
||||
Name: bucketName,
|
||||
Created: created,
|
||||
ACL: drivers.BucketACL(acl),
|
||||
Created: metadata.Created,
|
||||
ACL: drivers.BucketACL(metadata.ACL),
|
||||
}
|
||||
return bucketMetadata, nil
|
||||
}
|
||||
|
@ -332,22 +319,14 @@ func (d donutDriver) GetObjectMetadata(bucketName, objectName string) (drivers.O
|
|||
Object: objectName,
|
||||
}, errParams)
|
||||
}
|
||||
created, err := time.Parse(time.RFC3339Nano, metadata["created"])
|
||||
if err != nil {
|
||||
return drivers.ObjectMetadata{}, iodine.New(err, errParams)
|
||||
}
|
||||
size, err := strconv.ParseInt(metadata["size"], 10, 64)
|
||||
if err != nil {
|
||||
return drivers.ObjectMetadata{}, iodine.New(err, errParams)
|
||||
}
|
||||
objectMetadata := drivers.ObjectMetadata{
|
||||
Bucket: bucketName,
|
||||
Key: objectName,
|
||||
|
||||
ContentType: metadata["contentType"],
|
||||
Created: created,
|
||||
Md5: metadata["md5"],
|
||||
Size: size,
|
||||
ContentType: metadata.Metadata["contentType"],
|
||||
Created: metadata.Created,
|
||||
Md5: metadata.MD5Sum,
|
||||
Size: metadata.Size,
|
||||
}
|
||||
return objectMetadata, nil
|
||||
}
|
||||
|
@ -390,18 +369,10 @@ func (d donutDriver) ListObjects(bucketName string, resources drivers.BucketReso
|
|||
if err != nil {
|
||||
return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, errParams)
|
||||
}
|
||||
t, err := time.Parse(time.RFC3339Nano, objectMetadata["created"])
|
||||
if err != nil {
|
||||
return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, nil)
|
||||
}
|
||||
size, err := strconv.ParseInt(objectMetadata["size"], 10, 64)
|
||||
if err != nil {
|
||||
return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, nil)
|
||||
}
|
||||
metadata := drivers.ObjectMetadata{
|
||||
Key: objectName,
|
||||
Created: t,
|
||||
Size: size,
|
||||
Created: objectMetadata.Created,
|
||||
Size: objectMetadata.Size,
|
||||
}
|
||||
results = append(results, metadata)
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue