maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Rename memory driver as cache

Browse source
This commit is contained in:
Harshavardhana 2015-06-29 16:02:17 -07:00
parent 2571342451
commit 12de98fb62
4 changed files with 190 additions and 195 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

150
pkg/storage/drivers/memory/memory_multipart.go → pkg/storage/drivers/cache/cache-multipart.go vendored
View file

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
package memory package cache
import ( import (
"bytes" "bytes"
@ -35,54 +35,54 @@ import (
"github.com/minio/minio/pkg/storage/drivers" "github.com/minio/minio/pkg/storage/drivers"
) )
func (memory *memoryDriver) NewMultipartUpload(bucket, key, contentType string) (string, error) { func (cache *cacheDriver) NewMultipartUpload(bucket, key, contentType string) (string, error) {
memory.lock.RLock() cache.lock.RLock()
if !drivers.IsValidBucket(bucket) { if !drivers.IsValidBucket(bucket) {
memory.lock.RUnlock() cache.lock.RUnlock()
return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil) return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
} }
if !drivers.IsValidObjectName(key) { if !drivers.IsValidObjectName(key) {
memory.lock.RUnlock() cache.lock.RUnlock()
return "", iodine.New(drivers.ObjectNameInvalid{Object: key}, nil) return "", iodine.New(drivers.ObjectNameInvalid{Object: key}, nil)
} }
if _, ok := memory.storedBuckets[bucket]; ok == false { if _, ok := cache.storedBuckets[bucket]; ok == false {
memory.lock.RUnlock() cache.lock.RUnlock()
return "", iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) return "", iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
} }
storedBucket := memory.storedBuckets[bucket] storedBucket := cache.storedBuckets[bucket]
objectKey := bucket + "/" + key objectKey := bucket + "/" + key
if _, ok := storedBucket.objectMetadata[objectKey]; ok == true { if _, ok := storedBucket.objectMetadata[objectKey]; ok == true {
memory.lock.RUnlock() cache.lock.RUnlock()
return "", iodine.New(drivers.ObjectExists{Bucket: bucket, Object: key}, nil) return "", iodine.New(drivers.ObjectExists{Bucket: bucket, Object: key}, nil)
} }
memory.lock.RUnlock() cache.lock.RUnlock()
memory.lock.Lock() cache.lock.Lock()
id := []byte(strconv.FormatInt(rand.Int63(), 10) + bucket + key + time.Now().String()) id := []byte(strconv.FormatInt(rand.Int63(), 10) + bucket + key + time.Now().String())
uploadIDSum := sha512.Sum512(id) uploadIDSum := sha512.Sum512(id)
uploadID := base64.URLEncoding.EncodeToString(uploadIDSum[:])[:47] uploadID := base64.URLEncoding.EncodeToString(uploadIDSum[:])[:47]
memory.storedBuckets[bucket].multiPartSession[key] = multiPartSession{ cache.storedBuckets[bucket].multiPartSession[key] = multiPartSession{
uploadID: uploadID, uploadID: uploadID,
initiated: time.Now(), initiated: time.Now(),
totalParts: 0, totalParts: 0,
} }
memory.lock.Unlock() cache.lock.Unlock()
return uploadID, nil return uploadID, nil
} }
func (memory *memoryDriver) AbortMultipartUpload(bucket, key, uploadID string) error { func (cache *cacheDriver) AbortMultipartUpload(bucket, key, uploadID string) error {
memory.lock.RLock() cache.lock.RLock()
storedBucket := memory.storedBuckets[bucket] storedBucket := cache.storedBuckets[bucket]
if storedBucket.multiPartSession[key].uploadID != uploadID { if storedBucket.multiPartSession[key].uploadID != uploadID {
memory.lock.RUnlock() cache.lock.RUnlock()
return iodine.New(drivers.InvalidUploadID{UploadID: uploadID}, nil) return iodine.New(drivers.InvalidUploadID{UploadID: uploadID}, nil)
} }
memory.lock.RUnlock() cache.lock.RUnlock()
memory.cleanupMultiparts(bucket, key, uploadID) cache.cleanupMultiparts(bucket, key, uploadID)
memory.cleanupMultipartSession(bucket, key, uploadID) cache.cleanupMultipartSession(bucket, key, uploadID)
return nil return nil
} }
@ -90,17 +90,17 @@ func getMultipartKey(key string, uploadID string, partNumber int) string {
return key + "?uploadId=" + uploadID + "&partNumber=" + strconv.Itoa(partNumber) return key + "?uploadId=" + uploadID + "&partNumber=" + strconv.Itoa(partNumber)
} }
func (memory *memoryDriver) CreateObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) { func (cache *cacheDriver) CreateObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) {
// Verify upload id // Verify upload id
memory.lock.RLock() cache.lock.RLock()
storedBucket := memory.storedBuckets[bucket] storedBucket := cache.storedBuckets[bucket]
if storedBucket.multiPartSession[key].uploadID != uploadID { if storedBucket.multiPartSession[key].uploadID != uploadID {
memory.lock.RUnlock() cache.lock.RUnlock()
return "", iodine.New(drivers.InvalidUploadID{UploadID: uploadID}, nil) return "", iodine.New(drivers.InvalidUploadID{UploadID: uploadID}, nil)
} }
memory.lock.RUnlock() cache.lock.RUnlock()
etag, err := memory.createObjectPart(bucket, key, uploadID, partID, "", expectedMD5Sum, size, data) etag, err := cache.createObjectPart(bucket, key, uploadID, partID, "", expectedMD5Sum, size, data)
if err != nil { if err != nil {
return "", iodine.New(err, nil) return "", iodine.New(err, nil)
} }
@ -109,29 +109,29 @@ func (memory *memoryDriver) CreateObjectPart(bucket, key, uploadID string, partI
return etag, nil return etag, nil
} }
// createObject - PUT object to memory buffer // createObject - PUT object to cache buffer
func (memory *memoryDriver) createObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) { func (cache *cacheDriver) createObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) {
memory.lock.RLock() cache.lock.RLock()
if !drivers.IsValidBucket(bucket) { if !drivers.IsValidBucket(bucket) {
memory.lock.RUnlock() cache.lock.RUnlock()
return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil) return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
} }
if !drivers.IsValidObjectName(key) { if !drivers.IsValidObjectName(key) {
memory.lock.RUnlock() cache.lock.RUnlock()
return "", iodine.New(drivers.ObjectNameInvalid{Object: key}, nil) return "", iodine.New(drivers.ObjectNameInvalid{Object: key}, nil)
} }
if _, ok := memory.storedBuckets[bucket]; ok == false { if _, ok := cache.storedBuckets[bucket]; ok == false {
memory.lock.RUnlock() cache.lock.RUnlock()
return "", iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) return "", iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
} }
storedBucket := memory.storedBuckets[bucket] storedBucket := cache.storedBuckets[bucket]
// get object key // get object key
partKey := bucket + "/" + getMultipartKey(key, uploadID, partID) partKey := bucket + "/" + getMultipartKey(key, uploadID, partID)
if _, ok := storedBucket.partMetadata[partKey]; ok == true { if _, ok := storedBucket.partMetadata[partKey]; ok == true {
memory.lock.RUnlock() cache.lock.RUnlock()
return storedBucket.partMetadata[partKey].ETag, nil return storedBucket.partMetadata[partKey].ETag, nil
} }
memory.lock.RUnlock() cache.lock.RUnlock()
if contentType == "" { if contentType == "" {
contentType = "application/octet-stream" contentType = "application/octet-stream"
@ -170,9 +170,9 @@ func (memory *memoryDriver) createObjectPart(bucket, key, uploadID string, partI
md5SumBytes := hash.Sum(nil) md5SumBytes := hash.Sum(nil)
totalLength := int64(len(readBytes)) totalLength := int64(len(readBytes))
memory.lock.Lock() cache.lock.Lock()
memory.multiPartObjects.Set(partKey, readBytes) cache.multiPartObjects.Set(partKey, readBytes)
memory.lock.Unlock() cache.lock.Unlock()
// setting up for de-allocation // setting up for de-allocation
readBytes = nil readBytes = nil
@ -190,31 +190,31 @@ func (memory *memoryDriver) createObjectPart(bucket, key, uploadID string, partI
Size: totalLength, Size: totalLength,
} }
memory.lock.Lock() cache.lock.Lock()
storedBucket.partMetadata[partKey] = newPart storedBucket.partMetadata[partKey] = newPart
multiPartSession := storedBucket.multiPartSession[key] multiPartSession := storedBucket.multiPartSession[key]
multiPartSession.totalParts++ multiPartSession.totalParts++
storedBucket.multiPartSession[key] = multiPartSession storedBucket.multiPartSession[key] = multiPartSession
memory.storedBuckets[bucket] = storedBucket cache.storedBuckets[bucket] = storedBucket
memory.lock.Unlock() cache.lock.Unlock()
return md5Sum, nil return md5Sum, nil
} }
func (memory *memoryDriver) cleanupMultipartSession(bucket, key, uploadID string) { func (cache *cacheDriver) cleanupMultipartSession(bucket, key, uploadID string) {
memory.lock.Lock() cache.lock.Lock()
defer memory.lock.Unlock() defer cache.lock.Unlock()
delete(memory.storedBuckets[bucket].multiPartSession, key) delete(cache.storedBuckets[bucket].multiPartSession, key)
} }
func (memory *memoryDriver) cleanupMultiparts(bucket, key, uploadID string) { func (cache *cacheDriver) cleanupMultiparts(bucket, key, uploadID string) {
for i := 1; i <= memory.storedBuckets[bucket].multiPartSession[key].totalParts; i++ { for i := 1; i <= cache.storedBuckets[bucket].multiPartSession[key].totalParts; i++ {
objectKey := bucket + "/" + getMultipartKey(key, uploadID, i) objectKey := bucket + "/" + getMultipartKey(key, uploadID, i)
memory.multiPartObjects.Delete(objectKey) cache.multiPartObjects.Delete(objectKey)
} }
} }
func (memory *memoryDriver) CompleteMultipartUpload(bucket, key, uploadID string, parts map[int]string) (string, error) { func (cache *cacheDriver) CompleteMultipartUpload(bucket, key, uploadID string, parts map[int]string) (string, error) {
if !drivers.IsValidBucket(bucket) { if !drivers.IsValidBucket(bucket) {
return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil) return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
} }
@ -222,26 +222,26 @@ func (memory *memoryDriver) CompleteMultipartUpload(bucket, key, uploadID string
return "", iodine.New(drivers.ObjectNameInvalid{Object: key}, nil) return "", iodine.New(drivers.ObjectNameInvalid{Object: key}, nil)
} }
// Verify upload id // Verify upload id
memory.lock.RLock() cache.lock.RLock()
if _, ok := memory.storedBuckets[bucket]; ok == false { if _, ok := cache.storedBuckets[bucket]; ok == false {
memory.lock.RUnlock() cache.lock.RUnlock()
return "", iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) return "", iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
} }
storedBucket := memory.storedBuckets[bucket] storedBucket := cache.storedBuckets[bucket]
if storedBucket.multiPartSession[key].uploadID != uploadID { if storedBucket.multiPartSession[key].uploadID != uploadID {
memory.lock.RUnlock() cache.lock.RUnlock()
return "", iodine.New(drivers.InvalidUploadID{UploadID: uploadID}, nil) return "", iodine.New(drivers.InvalidUploadID{UploadID: uploadID}, nil)
} }
memory.lock.RUnlock() cache.lock.RUnlock()
memory.lock.Lock() cache.lock.Lock()
var size int64 var size int64
var fullObject bytes.Buffer var fullObject bytes.Buffer
for i := 1; i <= len(parts); i++ { for i := 1; i <= len(parts); i++ {
recvMD5 := parts[i] recvMD5 := parts[i]
object, ok := memory.multiPartObjects.Get(bucket + "/" + getMultipartKey(key, uploadID, i)) object, ok := cache.multiPartObjects.Get(bucket + "/" + getMultipartKey(key, uploadID, i))
if ok == false { if ok == false {
memory.lock.Unlock() cache.lock.Unlock()
return "", iodine.New(errors.New("missing part: "+strconv.Itoa(i)), nil) return "", iodine.New(errors.New("missing part: "+strconv.Itoa(i)), nil)
} }
size += int64(len(object)) size += int64(len(object))
@ -261,20 +261,20 @@ func (memory *memoryDriver) CompleteMultipartUpload(bucket, key, uploadID string
object = nil object = nil
go debug.FreeOSMemory() go debug.FreeOSMemory()
} }
memory.lock.Unlock() cache.lock.Unlock()
md5sumSlice := md5.Sum(fullObject.Bytes()) md5sumSlice := md5.Sum(fullObject.Bytes())
// this is needed for final verification inside CreateObject, do not convert this to hex // this is needed for final verification inside CreateObject, do not convert this to hex
md5sum := base64.StdEncoding.EncodeToString(md5sumSlice[:]) md5sum := base64.StdEncoding.EncodeToString(md5sumSlice[:])
etag, err := memory.CreateObject(bucket, key, "", md5sum, size, &fullObject) etag, err := cache.CreateObject(bucket, key, "", md5sum, size, &fullObject)
if err != nil { if err != nil {
// No need to call internal cleanup functions here, caller will call AbortMultipartUpload() // No need to call internal cleanup functions here, caller will call AbortMultipartUpload()
// which would in-turn cleanup properly in accordance with S3 Spec // which would in-turn cleanup properly in accordance with S3 Spec
return "", iodine.New(err, nil) return "", iodine.New(err, nil)
} }
fullObject.Reset() fullObject.Reset()
memory.cleanupMultiparts(bucket, key, uploadID) cache.cleanupMultiparts(bucket, key, uploadID)
memory.cleanupMultipartSession(bucket, key, uploadID) cache.cleanupMultipartSession(bucket, key, uploadID)
return etag, nil return etag, nil
} }
@ -285,14 +285,14 @@ func (a byKey) Len() int { return len(a) }
func (a byKey) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a byKey) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byKey) Less(i, j int) bool { return a[i].Key < a[j].Key } func (a byKey) Less(i, j int) bool { return a[i].Key < a[j].Key }
func (memory *memoryDriver) ListMultipartUploads(bucket string, resources drivers.BucketMultipartResourcesMetadata) (drivers.BucketMultipartResourcesMetadata, error) { func (cache *cacheDriver) ListMultipartUploads(bucket string, resources drivers.BucketMultipartResourcesMetadata) (drivers.BucketMultipartResourcesMetadata, error) {
// TODO handle delimiter // TODO handle delimiter
memory.lock.RLock() cache.lock.RLock()
defer memory.lock.RUnlock() defer cache.lock.RUnlock()
if _, ok := memory.storedBuckets[bucket]; ok == false { if _, ok := cache.storedBuckets[bucket]; ok == false {
return drivers.BucketMultipartResourcesMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) return drivers.BucketMultipartResourcesMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
} }
storedBucket := memory.storedBuckets[bucket] storedBucket := cache.storedBuckets[bucket]
var uploads []*drivers.UploadMetadata var uploads []*drivers.UploadMetadata
for key, session := range storedBucket.multiPartSession { for key, session := range storedBucket.multiPartSession {
@ -346,14 +346,14 @@ func (a partNumber) Len() int { return len(a) }
func (a partNumber) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a partNumber) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a partNumber) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber } func (a partNumber) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
func (memory *memoryDriver) ListObjectParts(bucket, key string, resources drivers.ObjectResourcesMetadata) (drivers.ObjectResourcesMetadata, error) { func (cache *cacheDriver) ListObjectParts(bucket, key string, resources drivers.ObjectResourcesMetadata) (drivers.ObjectResourcesMetadata, error) {
// Verify upload id // Verify upload id
memory.lock.RLock() cache.lock.RLock()
defer memory.lock.RUnlock() defer cache.lock.RUnlock()
if _, ok := memory.storedBuckets[bucket]; ok == false { if _, ok := cache.storedBuckets[bucket]; ok == false {
return drivers.ObjectResourcesMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) return drivers.ObjectResourcesMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
} }
storedBucket := memory.storedBuckets[bucket] storedBucket := cache.storedBuckets[bucket]
if _, ok := storedBucket.multiPartSession[key]; ok == false { if _, ok := storedBucket.multiPartSession[key]; ok == false {
return drivers.ObjectResourcesMetadata{}, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: key}, nil) return drivers.ObjectResourcesMetadata{}, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: key}, nil)
} }
@ -390,10 +390,10 @@ func (memory *memoryDriver) ListObjectParts(bucket, key string, resources driver
return objectResourcesMetadata, nil return objectResourcesMetadata, nil
} }
func (memory *memoryDriver) expiredPart(a ...interface{}) { func (cache *cacheDriver) expiredPart(a ...interface{}) {
key := a[0].(string) key := a[0].(string)
// loop through all buckets // loop through all buckets
for _, storedBucket := range memory.storedBuckets { for _, storedBucket := range cache.storedBuckets {
delete(storedBucket.partMetadata, key) delete(storedBucket.partMetadata, key)
} }
debug.FreeOSMemory() debug.FreeOSMemory()

233
pkg/storage/drivers/memory/memory.go → pkg/storage/drivers/cache/cache.go vendored
View file

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
package memory package cache
import ( import (
"bufio" "bufio"
@ -37,8 +37,8 @@ import (
"github.com/minio/minio/pkg/storage/trove" "github.com/minio/minio/pkg/storage/trove"
) )
// memoryDriver - local variables // cacheDriver - local variables
type memoryDriver struct { type cacheDriver struct {
storedBuckets map[string]storedBucket storedBuckets map[string]storedBucket
lock *sync.RWMutex lock *sync.RWMutex
objects *trove.Cache objects *trove.Cache
@ -64,65 +64,65 @@ const (
totalBuckets = 100 totalBuckets = 100
) )
// NewDriver instantiate a new memory driver // NewDriver instantiate a new cache driver
func NewDriver(maxSize uint64, expiration time.Duration) (drivers.Driver, error) { func NewDriver(maxSize uint64, expiration time.Duration) (drivers.Driver, error) {
memory := new(memoryDriver) cache := new(cacheDriver)
memory.storedBuckets = make(map[string]storedBucket) cache.storedBuckets = make(map[string]storedBucket)
memory.objects = trove.NewCache(maxSize, expiration) cache.objects = trove.NewCache(maxSize, expiration)
memory.maxSize = maxSize cache.maxSize = maxSize
memory.expiration = expiration cache.expiration = expiration
memory.multiPartObjects = trove.NewCache(0, time.Duration(0)) cache.multiPartObjects = trove.NewCache(0, time.Duration(0))
memory.lock = new(sync.RWMutex) cache.lock = new(sync.RWMutex)
memory.objects.OnExpired = memory.expiredObject cache.objects.OnExpired = cache.expiredObject
memory.multiPartObjects.OnExpired = memory.expiredPart cache.multiPartObjects.OnExpired = cache.expiredPart
// set up memory expiration // set up cache expiration
memory.objects.ExpireObjects(time.Second * 5) cache.objects.ExpireObjects(time.Second * 5)
return memory, nil return cache, nil
} }
// GetObject - GET object from memory buffer // GetObject - GET object from cache buffer
func (memory *memoryDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) { func (cache *cacheDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) {
memory.lock.RLock() cache.lock.RLock()
if !drivers.IsValidBucket(bucket) { if !drivers.IsValidBucket(bucket) {
memory.lock.RUnlock() cache.lock.RUnlock()
return 0, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil) return 0, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
} }
if !drivers.IsValidObjectName(object) { if !drivers.IsValidObjectName(object) {
memory.lock.RUnlock() cache.lock.RUnlock()
return 0, iodine.New(drivers.ObjectNameInvalid{Object: object}, nil) return 0, iodine.New(drivers.ObjectNameInvalid{Object: object}, nil)
} }
if _, ok := memory.storedBuckets[bucket]; ok == false { if _, ok := cache.storedBuckets[bucket]; ok == false {
memory.lock.RUnlock() cache.lock.RUnlock()
return 0, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) return 0, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
} }
objectKey := bucket + "/" + object objectKey := bucket + "/" + object
data, ok := memory.objects.Get(objectKey) data, ok := cache.objects.Get(objectKey)
if !ok { if !ok {
memory.lock.RUnlock() cache.lock.RUnlock()
return 0, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: object}, nil) return 0, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: object}, nil)
} }
written, err := io.Copy(w, bytes.NewBuffer(data)) written, err := io.Copy(w, bytes.NewBuffer(data))
memory.lock.RUnlock() cache.lock.RUnlock()
return written, iodine.New(err, nil) return written, iodine.New(err, nil)
} }
// GetPartialObject - GET object from memory buffer range // GetPartialObject - GET object from cache buffer range
func (memory *memoryDriver) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) { func (cache *cacheDriver) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) {
errParams := map[string]string{ errParams := map[string]string{
"bucket": bucket, "bucket": bucket,
"object": object, "object": object,
"start": strconv.FormatInt(start, 10), "start": strconv.FormatInt(start, 10),
"length": strconv.FormatInt(length, 10), "length": strconv.FormatInt(length, 10),
} }
memory.lock.RLock() cache.lock.RLock()
if !drivers.IsValidBucket(bucket) { if !drivers.IsValidBucket(bucket) {
memory.lock.RUnlock() cache.lock.RUnlock()
return 0, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, errParams) return 0, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, errParams)
} }
if !drivers.IsValidObjectName(object) { if !drivers.IsValidObjectName(object) {
memory.lock.RUnlock() cache.lock.RUnlock()
return 0, iodine.New(drivers.ObjectNameInvalid{Object: object}, errParams) return 0, iodine.New(drivers.ObjectNameInvalid{Object: object}, errParams)
} }
if start < 0 { if start < 0 {
@ -132,49 +132,49 @@ func (memory *memoryDriver) GetPartialObject(w io.Writer, bucket, object string,
}, errParams) }, errParams)
} }
objectKey := bucket + "/" + object objectKey := bucket + "/" + object
data, ok := memory.objects.Get(objectKey) data, ok := cache.objects.Get(objectKey)
if !ok { if !ok {
memory.lock.RUnlock() cache.lock.RUnlock()
return 0, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: object}, errParams) return 0, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: object}, errParams)
} }
written, err := io.CopyN(w, bytes.NewBuffer(data[start:]), length) written, err := io.CopyN(w, bytes.NewBuffer(data[start:]), length)
memory.lock.RUnlock() cache.lock.RUnlock()
return written, iodine.New(err, nil) return written, iodine.New(err, nil)
} }
// GetBucketMetadata - // GetBucketMetadata -
func (memory *memoryDriver) GetBucketMetadata(bucket string) (drivers.BucketMetadata, error) { func (cache *cacheDriver) GetBucketMetadata(bucket string) (drivers.BucketMetadata, error) {
memory.lock.RLock() cache.lock.RLock()
defer memory.lock.RUnlock() defer cache.lock.RUnlock()
if !drivers.IsValidBucket(bucket) { if !drivers.IsValidBucket(bucket) {
return drivers.BucketMetadata{}, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil) return drivers.BucketMetadata{}, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
} }
if _, ok := memory.storedBuckets[bucket]; ok == false { if _, ok := cache.storedBuckets[bucket]; ok == false {
return drivers.BucketMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) return drivers.BucketMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
} }
return memory.storedBuckets[bucket].bucketMetadata, nil return cache.storedBuckets[bucket].bucketMetadata, nil
} }
// SetBucketMetadata - // SetBucketMetadata -
func (memory *memoryDriver) SetBucketMetadata(bucket, acl string) error { func (cache *cacheDriver) SetBucketMetadata(bucket, acl string) error {
memory.lock.RLock() cache.lock.RLock()
if !drivers.IsValidBucket(bucket) { if !drivers.IsValidBucket(bucket) {
memory.lock.RUnlock() cache.lock.RUnlock()
return iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil) return iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
} }
if _, ok := memory.storedBuckets[bucket]; ok == false { if _, ok := cache.storedBuckets[bucket]; ok == false {
memory.lock.RUnlock() cache.lock.RUnlock()
return iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) return iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
} }
if strings.TrimSpace(acl) == "" { if strings.TrimSpace(acl) == "" {
acl = "private" acl = "private"
} }
memory.lock.RUnlock() cache.lock.RUnlock()
memory.lock.Lock() cache.lock.Lock()
defer memory.lock.Unlock() defer cache.lock.Unlock()
storedBucket := memory.storedBuckets[bucket] storedBucket := cache.storedBuckets[bucket]
storedBucket.bucketMetadata.ACL = drivers.BucketACL(acl) storedBucket.bucketMetadata.ACL = drivers.BucketACL(acl)
memory.storedBuckets[bucket] = storedBucket cache.storedBuckets[bucket] = storedBucket
return nil return nil
} }
@ -197,44 +197,44 @@ func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) error {
return iodine.New(errors.New("invalid argument"), nil) return iodine.New(errors.New("invalid argument"), nil)
} }
func (memory *memoryDriver) CreateObject(bucket, key, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) { func (cache *cacheDriver) CreateObject(bucket, key, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) {
if size > int64(memory.maxSize) { if size > int64(cache.maxSize) {
generic := drivers.GenericObjectError{Bucket: bucket, Object: key} generic := drivers.GenericObjectError{Bucket: bucket, Object: key}
return "", iodine.New(drivers.EntityTooLarge{ return "", iodine.New(drivers.EntityTooLarge{
GenericObjectError: generic, GenericObjectError: generic,
Size: strconv.FormatInt(size, 10), Size: strconv.FormatInt(size, 10),
MaxSize: strconv.FormatUint(memory.maxSize, 10), MaxSize: strconv.FormatUint(cache.maxSize, 10),
}, nil) }, nil)
} }
md5sum, err := memory.createObject(bucket, key, contentType, expectedMD5Sum, size, data) md5sum, err := cache.createObject(bucket, key, contentType, expectedMD5Sum, size, data)
// free // free
debug.FreeOSMemory() debug.FreeOSMemory()
return md5sum, iodine.New(err, nil) return md5sum, iodine.New(err, nil)
} }
// createObject - PUT object to memory buffer // createObject - PUT object to cache buffer
func (memory *memoryDriver) createObject(bucket, key, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) { func (cache *cacheDriver) createObject(bucket, key, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) {
memory.lock.RLock() cache.lock.RLock()
if !drivers.IsValidBucket(bucket) { if !drivers.IsValidBucket(bucket) {
memory.lock.RUnlock() cache.lock.RUnlock()
return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil) return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
} }
if !drivers.IsValidObjectName(key) { if !drivers.IsValidObjectName(key) {
memory.lock.RUnlock() cache.lock.RUnlock()
return "", iodine.New(drivers.ObjectNameInvalid{Object: key}, nil) return "", iodine.New(drivers.ObjectNameInvalid{Object: key}, nil)
} }
if _, ok := memory.storedBuckets[bucket]; ok == false { if _, ok := cache.storedBuckets[bucket]; ok == false {
memory.lock.RUnlock() cache.lock.RUnlock()
return "", iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) return "", iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
} }
storedBucket := memory.storedBuckets[bucket] storedBucket := cache.storedBuckets[bucket]
// get object key // get object key
objectKey := bucket + "/" + key objectKey := bucket + "/" + key
if _, ok := storedBucket.objectMetadata[objectKey]; ok == true { if _, ok := storedBucket.objectMetadata[objectKey]; ok == true {
memory.lock.RUnlock() cache.lock.RUnlock()
return "", iodine.New(drivers.ObjectExists{Bucket: bucket, Object: key}, nil) return "", iodine.New(drivers.ObjectExists{Bucket: bucket, Object: key}, nil)
} }
memory.lock.RUnlock() cache.lock.RUnlock()
if contentType == "" { if contentType == "" {
contentType = "application/octet-stream" contentType = "application/octet-stream"
@ -272,12 +272,12 @@ func (memory *memoryDriver) createObject(bucket, key, contentType, expectedMD5Su
md5SumBytes := hash.Sum(nil) md5SumBytes := hash.Sum(nil)
totalLength := len(readBytes) totalLength := len(readBytes)
memory.lock.Lock() cache.lock.Lock()
ok := memory.objects.Set(objectKey, readBytes) ok := cache.objects.Set(objectKey, readBytes)
// setting up for de-allocation // setting up for de-allocation
readBytes = nil readBytes = nil
go debug.FreeOSMemory() go debug.FreeOSMemory()
memory.lock.Unlock() cache.lock.Unlock()
if !ok { if !ok {
return "", iodine.New(drivers.InternalError{}, nil) return "", iodine.New(drivers.InternalError{}, nil)
} }
@ -300,33 +300,33 @@ func (memory *memoryDriver) createObject(bucket, key, contentType, expectedMD5Su
Size: int64(totalLength), Size: int64(totalLength),
} }
memory.lock.Lock() cache.lock.Lock()
storedBucket.objectMetadata[objectKey] = newObject storedBucket.objectMetadata[objectKey] = newObject
memory.storedBuckets[bucket] = storedBucket cache.storedBuckets[bucket] = storedBucket
memory.lock.Unlock() cache.lock.Unlock()
return newObject.Md5, nil return newObject.Md5, nil
} }
// CreateBucket - create bucket in memory // CreateBucket - create bucket in cache
func (memory *memoryDriver) CreateBucket(bucketName, acl string) error { func (cache *cacheDriver) CreateBucket(bucketName, acl string) error {
memory.lock.RLock() cache.lock.RLock()
if len(memory.storedBuckets) == totalBuckets { if len(cache.storedBuckets) == totalBuckets {
memory.lock.RUnlock() cache.lock.RUnlock()
return iodine.New(drivers.TooManyBuckets{Bucket: bucketName}, nil) return iodine.New(drivers.TooManyBuckets{Bucket: bucketName}, nil)
} }
if !drivers.IsValidBucket(bucketName) { if !drivers.IsValidBucket(bucketName) {
memory.lock.RUnlock() cache.lock.RUnlock()
return iodine.New(drivers.BucketNameInvalid{Bucket: bucketName}, nil) return iodine.New(drivers.BucketNameInvalid{Bucket: bucketName}, nil)
} }
if !drivers.IsValidBucketACL(acl) { if !drivers.IsValidBucketACL(acl) {
memory.lock.RUnlock() cache.lock.RUnlock()
return iodine.New(drivers.InvalidACL{ACL: acl}, nil) return iodine.New(drivers.InvalidACL{ACL: acl}, nil)
} }
if _, ok := memory.storedBuckets[bucketName]; ok == true { if _, ok := cache.storedBuckets[bucketName]; ok == true {
memory.lock.RUnlock() cache.lock.RUnlock()
return iodine.New(drivers.BucketExists{Bucket: bucketName}, nil) return iodine.New(drivers.BucketExists{Bucket: bucketName}, nil)
} }
memory.lock.RUnlock() cache.lock.RUnlock()
if strings.TrimSpace(acl) == "" { if strings.TrimSpace(acl) == "" {
// default is private // default is private
@ -340,9 +340,9 @@ func (memory *memoryDriver) CreateBucket(bucketName, acl string) error {
newBucket.bucketMetadata.Name = bucketName newBucket.bucketMetadata.Name = bucketName
newBucket.bucketMetadata.Created = time.Now().UTC() newBucket.bucketMetadata.Created = time.Now().UTC()
newBucket.bucketMetadata.ACL = drivers.BucketACL(acl) newBucket.bucketMetadata.ACL = drivers.BucketACL(acl)
memory.lock.Lock() cache.lock.Lock()
defer memory.lock.Unlock() cache.storedBuckets[bucketName] = newBucket
memory.storedBuckets[bucketName] = newBucket cache.lock.Unlock()
return nil return nil
} }
@ -364,7 +364,7 @@ func appendUniq(slice []string, i string) []string {
return append(slice, i) return append(slice, i)
} }
func (memory *memoryDriver) filterDelimiterPrefix(keys []string, key, delim string, r drivers.BucketResourcesMetadata) ([]string, drivers.BucketResourcesMetadata) { func (cache *cacheDriver) filterDelimiterPrefix(keys []string, key, delim string, r drivers.BucketResourcesMetadata) ([]string, drivers.BucketResourcesMetadata) {
switch true { switch true {
case key == r.Prefix: case key == r.Prefix:
keys = appendUniq(keys, key) keys = appendUniq(keys, key)
@ -377,7 +377,7 @@ func (memory *memoryDriver) filterDelimiterPrefix(keys []string, key, delim stri
return keys, r return keys, r
} }
func (memory *memoryDriver) listObjects(keys []string, key string, r drivers.BucketResourcesMetadata) ([]string, drivers.BucketResourcesMetadata) { func (cache *cacheDriver) listObjects(keys []string, key string, r drivers.BucketResourcesMetadata) ([]string, drivers.BucketResourcesMetadata) {
switch true { switch true {
// Prefix absent, delimit object key based on delimiter // Prefix absent, delimit object key based on delimiter
case r.IsDelimiterSet(): case r.IsDelimiterSet():
@ -393,7 +393,7 @@ func (memory *memoryDriver) listObjects(keys []string, key string, r drivers.Buc
if strings.HasPrefix(key, r.Prefix) { if strings.HasPrefix(key, r.Prefix) {
trimmedName := strings.TrimPrefix(key, r.Prefix) trimmedName := strings.TrimPrefix(key, r.Prefix)
delim := delimiter(trimmedName, r.Delimiter) delim := delimiter(trimmedName, r.Delimiter)
keys, r = memory.filterDelimiterPrefix(keys, key, delim, r) keys, r = cache.filterDelimiterPrefix(keys, key, delim, r)
} }
// Prefix present, nothing to delimit // Prefix present, nothing to delimit
case r.IsPrefixSet(): case r.IsPrefixSet():
@ -405,26 +405,26 @@ func (memory *memoryDriver) listObjects(keys []string, key string, r drivers.Buc
return keys, r return keys, r
} }
// ListObjects - list objects from memory // ListObjects - list objects from cache
func (memory *memoryDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) { func (cache *cacheDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
memory.lock.RLock() cache.lock.RLock()
defer memory.lock.RUnlock() defer cache.lock.RUnlock()
if !drivers.IsValidBucket(bucket) { if !drivers.IsValidBucket(bucket) {
return nil, drivers.BucketResourcesMetadata{IsTruncated: false}, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil) return nil, drivers.BucketResourcesMetadata{IsTruncated: false}, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
} }
if !drivers.IsValidObjectName(resources.Prefix) { if !drivers.IsValidObjectName(resources.Prefix) {
return nil, drivers.BucketResourcesMetadata{IsTruncated: false}, iodine.New(drivers.ObjectNameInvalid{Object: resources.Prefix}, nil) return nil, drivers.BucketResourcesMetadata{IsTruncated: false}, iodine.New(drivers.ObjectNameInvalid{Object: resources.Prefix}, nil)
} }
if _, ok := memory.storedBuckets[bucket]; ok == false { if _, ok := cache.storedBuckets[bucket]; ok == false {
return nil, drivers.BucketResourcesMetadata{IsTruncated: false}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) return nil, drivers.BucketResourcesMetadata{IsTruncated: false}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
} }
var results []drivers.ObjectMetadata var results []drivers.ObjectMetadata
var keys []string var keys []string
storedBucket := memory.storedBuckets[bucket] storedBucket := cache.storedBuckets[bucket]
for key := range storedBucket.objectMetadata { for key := range storedBucket.objectMetadata {
if strings.HasPrefix(key, bucket+"/") { if strings.HasPrefix(key, bucket+"/") {
key = key[len(bucket)+1:] key = key[len(bucket)+1:]
keys, resources = memory.listObjects(keys, key, resources) keys, resources = cache.listObjects(keys, key, resources)
} }
} }
var newKeys []string var newKeys []string
@ -453,34 +453,29 @@ func (memory *memoryDriver) ListObjects(bucket string, resources drivers.BucketR
return results, resources, nil return results, resources, nil
} }
// ByBucketName is a type for sorting bucket metadata by bucket name // byBucketName is a type for sorting bucket metadata by bucket name
type ByBucketName []drivers.BucketMetadata type byBucketName []drivers.BucketMetadata
// Len of bucket name func (b byBucketName) Len() int { return len(b) }
func (b ByBucketName) Len() int { return len(b) } func (b byBucketName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
func (b byBucketName) Less(i, j int) bool { return b[i].Name < b[j].Name }
// Swap bucket i, j // ListBuckets - List buckets from cache
func (b ByBucketName) Swap(i, j int) { b[i], b[j] = b[j], b[i] } func (cache *cacheDriver) ListBuckets() ([]drivers.BucketMetadata, error) {
cache.lock.RLock()
// Less defer cache.lock.RUnlock()
func (b ByBucketName) Less(i, j int) bool { return b[i].Name < b[j].Name }
// ListBuckets - List buckets from memory
func (memory *memoryDriver) ListBuckets() ([]drivers.BucketMetadata, error) {
memory.lock.RLock()
defer memory.lock.RUnlock()
var results []drivers.BucketMetadata var results []drivers.BucketMetadata
for _, bucket := range memory.storedBuckets { for _, bucket := range cache.storedBuckets {
results = append(results, bucket.bucketMetadata) results = append(results, bucket.bucketMetadata)
} }
sort.Sort(ByBucketName(results)) sort.Sort(byBucketName(results))
return results, nil return results, nil
} }
// GetObjectMetadata - get object metadata from memory // GetObjectMetadata - get object metadata from cache
func (memory *memoryDriver) GetObjectMetadata(bucket, key string) (drivers.ObjectMetadata, error) { func (cache *cacheDriver) GetObjectMetadata(bucket, key string) (drivers.ObjectMetadata, error) {
memory.lock.RLock() cache.lock.RLock()
defer memory.lock.RUnlock() defer cache.lock.RUnlock()
// check if bucket exists // check if bucket exists
if !drivers.IsValidBucket(bucket) { if !drivers.IsValidBucket(bucket) {
return drivers.ObjectMetadata{}, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil) return drivers.ObjectMetadata{}, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
@ -488,10 +483,10 @@ func (memory *memoryDriver) GetObjectMetadata(bucket, key string) (drivers.Objec
if !drivers.IsValidObjectName(key) { if !drivers.IsValidObjectName(key) {
return drivers.ObjectMetadata{}, iodine.New(drivers.ObjectNameInvalid{Object: key}, nil) return drivers.ObjectMetadata{}, iodine.New(drivers.ObjectNameInvalid{Object: key}, nil)
} }
if _, ok := memory.storedBuckets[bucket]; ok == false { if _, ok := cache.storedBuckets[bucket]; ok == false {
return drivers.ObjectMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) return drivers.ObjectMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
} }
storedBucket := memory.storedBuckets[bucket] storedBucket := cache.storedBuckets[bucket]
objectKey := bucket + "/" + key objectKey := bucket + "/" + key
if object, ok := storedBucket.objectMetadata[objectKey]; ok == true { if object, ok := storedBucket.objectMetadata[objectKey]; ok == true {
return object, nil return object, nil
@ -499,18 +494,18 @@ func (memory *memoryDriver) GetObjectMetadata(bucket, key string) (drivers.Objec
return drivers.ObjectMetadata{}, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: key}, nil) return drivers.ObjectMetadata{}, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: key}, nil)
} }
func (memory *memoryDriver) expiredObject(a ...interface{}) { func (cache *cacheDriver) expiredObject(a ...interface{}) {
cacheStats := memory.objects.Stats() cacheStats := cache.objects.Stats()
log.Printf("CurrentSize: %d, CurrentItems: %d, TotalExpirations: %d", log.Printf("CurrentSize: %d, CurrentItems: %d, TotalExpirations: %d",
cacheStats.Bytes, cacheStats.Items, cacheStats.Expired) cacheStats.Bytes, cacheStats.Items, cacheStats.Expired)
key := a[0].(string) key := a[0].(string)
// loop through all buckets // loop through all buckets
for bucket, storedBucket := range memory.storedBuckets { for bucket, storedBucket := range cache.storedBuckets {
delete(storedBucket.objectMetadata, key) delete(storedBucket.objectMetadata, key)
// remove bucket if no objects found anymore // remove bucket if no objects found anymore
if len(storedBucket.objectMetadata) == 0 { if len(storedBucket.objectMetadata) == 0 {
if time.Since(memory.storedBuckets[bucket].bucketMetadata.Created) > memory.expiration { if time.Since(cache.storedBuckets[bucket].bucketMetadata.Created) > cache.expiration {
delete(memory.storedBuckets, bucket) delete(cache.storedBuckets, bucket)
} }
} }
} }

2
pkg/storage/drivers/memory/memory_test.go → pkg/storage/drivers/cache/cache_test.go vendored
View file

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
package memory package cache
import ( import (
"testing" "testing"

0
pkg/storage/drivers/donut/multipart.go → pkg/storage/drivers/donut/donut-multipart.go
View file