Merge pull request #344 from fkautz/pr_out_exorcising_donutbox
This commit is contained in:
commit
f471915646
|
@ -1,89 +0,0 @@
|
||||||
package donutbox
|
|
||||||
|
|
||||||
import "io"
|
|
||||||
|
|
||||||
// DonutBox is an interface specifying how the storage driver should interact with its underlying system.
|
|
||||||
type DonutBox interface {
|
|
||||||
// system operations
|
|
||||||
ListBuckets() ([]string, error)
|
|
||||||
|
|
||||||
// bucket operations
|
|
||||||
CreateBucket(bucket string) error
|
|
||||||
ListObjectsInBucket(bucket, prefix string) ([]string, error)
|
|
||||||
GetBucketMetadata(bucket string) (map[string]string, error)
|
|
||||||
SetBucketMetadata(bucket string, metadata map[string]string) error
|
|
||||||
|
|
||||||
// object operations
|
|
||||||
GetObjectWriter(bucket, object string, column uint) (*NewObject, error)
|
|
||||||
GetObjectReader(bucket, object string, column uint) (io.Reader, error)
|
|
||||||
GetObjectMetadata(bucket, object string, column uint) (map[string]string, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Bucket contains major operations on a bucket
|
|
||||||
type Bucket interface {
|
|
||||||
AddDisk(Disk) error
|
|
||||||
RemoveDisk(Disk)
|
|
||||||
GetDisk(i uint) (Disk, error)
|
|
||||||
GetDisks() ([]Disk, error)
|
|
||||||
GetMetadata() (map[string]string, error)
|
|
||||||
ListObjects(prefix string) ([]string, error)
|
|
||||||
SetMetadata(metadata map[string]string) error
|
|
||||||
}
|
|
||||||
|
|
||||||
// Disk represents major operations on a bucket's disk
|
|
||||||
type Disk interface {
|
|
||||||
GetObjectMetadata(object string) (map[string]string, error)
|
|
||||||
GetObjectReader(object string) (io.Reader, error)
|
|
||||||
GetObjectWriter(object string) (*NewObject, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Result is a result for async tasks
|
|
||||||
type Result struct {
|
|
||||||
Err error
|
|
||||||
}
|
|
||||||
|
|
||||||
// CreateNewObject creates a new object wrapping a writer. Clients are not expected to use this directly. This is exposed for storage drivers.
|
|
||||||
func CreateNewObject(writer *io.PipeWriter) *NewObject {
|
|
||||||
return &NewObject{writer: writer}
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewObject wraps a writer and allows setting metadata. On a successful close, the object is committed by the backend.
|
|
||||||
type NewObject struct {
|
|
||||||
metadata map[string]string
|
|
||||||
writer *io.PipeWriter
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write data
|
|
||||||
func (newObject *NewObject) Write(data []byte) (int, error) {
|
|
||||||
return newObject.writer.Write(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetMetadata sets metadata for an object
|
|
||||||
func (newObject *NewObject) SetMetadata(metadata map[string]string) {
|
|
||||||
newMetadata := make(map[string]string)
|
|
||||||
for k, v := range metadata {
|
|
||||||
newMetadata[k] = v
|
|
||||||
}
|
|
||||||
newObject.metadata = newMetadata
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close and commit the object
|
|
||||||
func (newObject *NewObject) Close() error {
|
|
||||||
return newObject.writer.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// CloseWithError closes the object with an error, causing the backend to abandon the object
|
|
||||||
func (newObject *NewObject) CloseWithError(err error) error {
|
|
||||||
return newObject.writer.CloseWithError(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetMetadata returns a copy of the metadata set metadata
|
|
||||||
func (newObject *NewObject) GetMetadata() map[string]string {
|
|
||||||
newMetadata := make(map[string]string)
|
|
||||||
if newMetadata != nil {
|
|
||||||
for k, v := range newObject.metadata {
|
|
||||||
newMetadata[k] = v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return newMetadata
|
|
||||||
}
|
|
|
@ -1 +0,0 @@
|
||||||
package donutfs
|
|
|
@ -1 +0,0 @@
|
||||||
package donutfs
|
|
|
@ -1,246 +0,0 @@
|
||||||
package donutmem
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"errors"
|
|
||||||
"github.com/minio-io/minio/pkg/donutbox"
|
|
||||||
"io"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
type bucket struct {
|
|
||||||
name string
|
|
||||||
metadata map[string]string
|
|
||||||
objects map[string]*object
|
|
||||||
lock *sync.RWMutex
|
|
||||||
}
|
|
||||||
|
|
||||||
type object struct {
|
|
||||||
name string
|
|
||||||
data []byte
|
|
||||||
metadata map[string]string
|
|
||||||
lock *sync.RWMutex
|
|
||||||
}
|
|
||||||
|
|
||||||
type donutMem struct {
|
|
||||||
buckets map[string]*bucket
|
|
||||||
lock *sync.RWMutex
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewDonutMem creates a new in memory donut
|
|
||||||
func NewDonutMem() donutbox.DonutBox {
|
|
||||||
return donutMem{
|
|
||||||
buckets: make(map[string]*bucket),
|
|
||||||
lock: new(sync.RWMutex),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// system operations
|
|
||||||
func (donutMem donutMem) ListBuckets() ([]string, error) {
|
|
||||||
donutMem.lock.RLock()
|
|
||||||
defer donutMem.lock.RUnlock()
|
|
||||||
var buckets []string
|
|
||||||
for k := range donutMem.buckets {
|
|
||||||
buckets = append(buckets, k)
|
|
||||||
}
|
|
||||||
return buckets, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// bucket operations
|
|
||||||
func (donutMem donutMem) CreateBucket(b string) error {
|
|
||||||
donutMem.lock.Lock()
|
|
||||||
defer donutMem.lock.Unlock()
|
|
||||||
b = strings.ToLower(b)
|
|
||||||
if _, ok := donutMem.buckets[b]; ok {
|
|
||||||
return errors.New("Bucket Exists")
|
|
||||||
}
|
|
||||||
metadata := make(map[string]string)
|
|
||||||
metadata["name"] = b
|
|
||||||
metadata["created"] = time.Now().Format(time.RFC3339Nano)
|
|
||||||
newBucket := bucket{
|
|
||||||
name: b,
|
|
||||||
metadata: metadata,
|
|
||||||
objects: make(map[string]*object),
|
|
||||||
lock: new(sync.RWMutex),
|
|
||||||
}
|
|
||||||
donutMem.buckets[b] = &newBucket
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (donutMem donutMem) ListObjectsInBucket(bucketKey, prefixKey string) ([]string, error) {
|
|
||||||
donutMem.lock.RLock()
|
|
||||||
defer donutMem.lock.RUnlock()
|
|
||||||
if curBucket, ok := donutMem.buckets[bucketKey]; ok {
|
|
||||||
curBucket.lock.RLock()
|
|
||||||
defer curBucket.lock.RUnlock()
|
|
||||||
objectMap := make(map[string]string)
|
|
||||||
for objectKey := range curBucket.objects {
|
|
||||||
objectName := strings.Split(objectKey, "#")[0]
|
|
||||||
if strings.HasPrefix(objectName, prefixKey) {
|
|
||||||
objectMap[objectName] = objectName
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var objects []string
|
|
||||||
for k := range objectMap {
|
|
||||||
objects = append(objects, k)
|
|
||||||
}
|
|
||||||
return objects, nil
|
|
||||||
}
|
|
||||||
return nil, errors.New("Bucket does not exist")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (donutMem donutMem) GetBucketMetadata(bucketKey string) (map[string]string, error) {
|
|
||||||
donutMem.lock.RLock()
|
|
||||||
defer donutMem.lock.RUnlock()
|
|
||||||
|
|
||||||
if curBucket, ok := donutMem.buckets[bucketKey]; ok {
|
|
||||||
curBucket.lock.RLock()
|
|
||||||
defer curBucket.lock.RUnlock()
|
|
||||||
result := make(map[string]string)
|
|
||||||
for k, v := range curBucket.metadata {
|
|
||||||
result[k] = v
|
|
||||||
}
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
return nil, errors.New("Bucket not found")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (donutMem donutMem) SetBucketMetadata(bucketKey string, metadata map[string]string) error {
|
|
||||||
donutMem.lock.RLock()
|
|
||||||
defer donutMem.lock.RUnlock()
|
|
||||||
if curBucket, ok := donutMem.buckets[bucketKey]; ok {
|
|
||||||
curBucket.lock.Lock()
|
|
||||||
defer curBucket.lock.Unlock()
|
|
||||||
newMetadata := make(map[string]string)
|
|
||||||
for k, v := range metadata {
|
|
||||||
newMetadata[k] = v
|
|
||||||
}
|
|
||||||
curBucket.metadata = newMetadata
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return errors.New("Bucket not found")
|
|
||||||
}
|
|
||||||
|
|
||||||
// object operations
|
|
||||||
func (donutMem donutMem) GetObjectWriter(bucketKey, objectKey string, column uint) (*donutbox.NewObject, error) {
|
|
||||||
key := getKey(bucketKey, objectKey, column)
|
|
||||||
reader, writer := io.Pipe()
|
|
||||||
returnObject := donutbox.CreateNewObject(writer)
|
|
||||||
donutMem.lock.RLock()
|
|
||||||
defer donutMem.lock.RUnlock()
|
|
||||||
if curBucket, ok := donutMem.buckets[bucketKey]; ok {
|
|
||||||
curBucket.lock.Lock()
|
|
||||||
defer curBucket.lock.Unlock()
|
|
||||||
if _, ok := curBucket.objects[key]; !ok {
|
|
||||||
newObject := object{
|
|
||||||
name: key,
|
|
||||||
data: make([]byte, 0),
|
|
||||||
lock: new(sync.RWMutex),
|
|
||||||
}
|
|
||||||
|
|
||||||
newObject.lock.Lock()
|
|
||||||
curBucket.objects[key] = &newObject
|
|
||||||
go func() {
|
|
||||||
defer newObject.lock.Unlock()
|
|
||||||
var objBuffer bytes.Buffer
|
|
||||||
|
|
||||||
_, err := io.Copy(&objBuffer, reader)
|
|
||||||
if err == nil {
|
|
||||||
newObject.data = objBuffer.Bytes()
|
|
||||||
writer.Close()
|
|
||||||
|
|
||||||
metadata := returnObject.GetMetadata()
|
|
||||||
for k, v := range metadata {
|
|
||||||
metadata[k] = v
|
|
||||||
}
|
|
||||||
metadata["key"] = objectKey
|
|
||||||
metadata["column"] = strconv.FormatUint(uint64(column), 10)
|
|
||||||
newObject.metadata = metadata
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
donutMem.lock.RLock()
|
|
||||||
defer donutMem.lock.RUnlock()
|
|
||||||
bucket, _ := donutMem.buckets[bucketKey]
|
|
||||||
bucket.lock.Lock()
|
|
||||||
defer bucket.lock.Unlock()
|
|
||||||
delete(bucket.objects, key)
|
|
||||||
writer.CloseWithError(err)
|
|
||||||
}()
|
|
||||||
return returnObject, nil
|
|
||||||
}
|
|
||||||
writer.CloseWithError(errors.New("Object exists"))
|
|
||||||
return nil, errors.New("Object exists")
|
|
||||||
}
|
|
||||||
writer.CloseWithError(errors.New("Bucket does not exist"))
|
|
||||||
return nil, errors.New("Bucket does not exist")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (donutMem donutMem) GetObjectReader(bucketKey, objectKey string, column uint) (io.Reader, error) {
|
|
||||||
key := getKey(bucketKey, objectKey, column)
|
|
||||||
donutMem.lock.RLock()
|
|
||||||
defer donutMem.lock.RUnlock()
|
|
||||||
if curBucket, ok := donutMem.buckets[bucketKey]; ok {
|
|
||||||
curBucket.lock.RLock()
|
|
||||||
defer curBucket.lock.RUnlock()
|
|
||||||
if curObject, ok := curBucket.objects[key]; ok {
|
|
||||||
curObject.lock.RLock()
|
|
||||||
defer curObject.lock.RUnlock()
|
|
||||||
return bytes.NewBuffer(curObject.data), nil
|
|
||||||
}
|
|
||||||
return nil, errors.New("Object not found")
|
|
||||||
}
|
|
||||||
return nil, errors.New("Bucket not found")
|
|
||||||
}
|
|
||||||
|
|
||||||
//func (donutMem donutMem) SetObjectMetadata(bucketKey, objectKey string, column uint, metadata map[string]string) error {
|
|
||||||
// key := getKey(bucketKey, objectKey, column)
|
|
||||||
// donutMem.lock.RLock()
|
|
||||||
// defer donutMem.lock.RUnlock()
|
|
||||||
// if curBucket, ok := donutMem.buckets[bucketKey]; ok {
|
|
||||||
// curBucket.lock.RLock()
|
|
||||||
// defer curBucket.lock.RUnlock()
|
|
||||||
// if curObject, ok := curBucket.objects[key]; ok {
|
|
||||||
// curObject.lock.Lock()
|
|
||||||
// defer curObject.lock.Unlock()
|
|
||||||
// newMetadata := make(map[string]string)
|
|
||||||
// for k, v := range metadata {
|
|
||||||
// newMetadata[k] = v
|
|
||||||
// }
|
|
||||||
// curObject.metadata = newMetadata
|
|
||||||
// return nil
|
|
||||||
// }
|
|
||||||
// return errors.New("Object not found")
|
|
||||||
// }
|
|
||||||
// return errors.New("Bucket not found")
|
|
||||||
//}
|
|
||||||
|
|
||||||
func (donutMem donutMem) GetObjectMetadata(bucketKey, objectKey string, column uint) (map[string]string, error) {
|
|
||||||
key := getKey(bucketKey, objectKey, column)
|
|
||||||
donutMem.lock.RLock()
|
|
||||||
defer donutMem.lock.RUnlock()
|
|
||||||
|
|
||||||
if curBucket, ok := donutMem.buckets[bucketKey]; ok {
|
|
||||||
curBucket.lock.RLock()
|
|
||||||
defer curBucket.lock.RUnlock()
|
|
||||||
if curObject, ok := curBucket.objects[key]; ok {
|
|
||||||
curObject.lock.RLock()
|
|
||||||
defer curObject.lock.RUnlock()
|
|
||||||
result := make(map[string]string)
|
|
||||||
for k, v := range curObject.metadata {
|
|
||||||
result[k] = v
|
|
||||||
}
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
return nil, errors.New("Object not Found: " + bucketKey + "#" + objectKey)
|
|
||||||
}
|
|
||||||
return nil, errors.New("Bucket not found")
|
|
||||||
}
|
|
||||||
|
|
||||||
func getKey(bucketKey, objectKey string, column uint) string {
|
|
||||||
return objectKey + "#" + strconv.FormatUint(uint64(column), 10)
|
|
||||||
}
|
|
|
@ -1,151 +0,0 @@
|
||||||
package donutmem
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
. "gopkg.in/check.v1"
|
|
||||||
"io/ioutil"
|
|
||||||
"sort"
|
|
||||||
"strconv"
|
|
||||||
)
|
|
||||||
|
|
||||||
func Test(t *testing.T) { TestingT(t) }
|
|
||||||
|
|
||||||
type MySuite struct{}
|
|
||||||
|
|
||||||
var _ = Suite(&MySuite{})
|
|
||||||
|
|
||||||
func (s *MySuite) TestCreateAndReadObject(c *C) {
|
|
||||||
data := "Hello World"
|
|
||||||
donut := NewDonutMem()
|
|
||||||
|
|
||||||
writer, err := donut.GetObjectWriter("foo", "bar", 0)
|
|
||||||
c.Assert(writer, IsNil)
|
|
||||||
c.Assert(err, Not(IsNil))
|
|
||||||
|
|
||||||
err = donut.CreateBucket("foo")
|
|
||||||
c.Assert(err, IsNil)
|
|
||||||
|
|
||||||
writer, err = donut.GetObjectWriter("foo", "bar", 0)
|
|
||||||
c.Assert(err, IsNil)
|
|
||||||
count, err := writer.Write([]byte(data))
|
|
||||||
c.Assert(count, Equals, len(data))
|
|
||||||
c.Assert(err, IsNil)
|
|
||||||
err = writer.Close()
|
|
||||||
c.Assert(err, IsNil)
|
|
||||||
|
|
||||||
// data should be available
|
|
||||||
reader, err := donut.GetObjectReader("foo", "bar", 0)
|
|
||||||
c.Assert(err, IsNil)
|
|
||||||
result, err := ioutil.ReadAll(reader)
|
|
||||||
c.Assert(result, DeepEquals, []byte(data))
|
|
||||||
|
|
||||||
// try writing, should see error
|
|
||||||
writer, err = donut.GetObjectWriter("foo", "bar", 0)
|
|
||||||
c.Assert(writer, IsNil)
|
|
||||||
c.Assert(err, Not(IsNil))
|
|
||||||
|
|
||||||
// data should not change
|
|
||||||
reader, err = donut.GetObjectReader("foo", "bar", 0)
|
|
||||||
c.Assert(err, IsNil)
|
|
||||||
result, err = ioutil.ReadAll(reader)
|
|
||||||
c.Assert(result, DeepEquals, []byte(data))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *MySuite) TestBucketList(c *C) {
|
|
||||||
donut := NewDonutMem()
|
|
||||||
|
|
||||||
results, err := donut.ListBuckets()
|
|
||||||
c.Assert(len(results), Equals, 0)
|
|
||||||
|
|
||||||
var buckets []string
|
|
||||||
for i := 0; i < 10; i++ {
|
|
||||||
bucket := "foo" + strconv.Itoa(i)
|
|
||||||
buckets = append(buckets, bucket)
|
|
||||||
err := donut.CreateBucket(bucket)
|
|
||||||
c.Assert(err, IsNil)
|
|
||||||
}
|
|
||||||
sort.Strings(buckets)
|
|
||||||
results, err = donut.ListBuckets()
|
|
||||||
c.Assert(err, IsNil)
|
|
||||||
sort.Strings(results)
|
|
||||||
c.Assert(results, DeepEquals, buckets)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *MySuite) TestObjectList(c *C) {
|
|
||||||
donut := NewDonutMem()
|
|
||||||
donut.CreateBucket("foo")
|
|
||||||
|
|
||||||
results, err := donut.ListObjectsInBucket("foo", "")
|
|
||||||
c.Assert(len(results), Equals, 0)
|
|
||||||
|
|
||||||
var objects []string
|
|
||||||
for i := 0; i < 10; i++ {
|
|
||||||
object := "foo" + strconv.Itoa(i)
|
|
||||||
objects = append(objects, object)
|
|
||||||
writer, err := donut.GetObjectWriter("foo", object, 0)
|
|
||||||
c.Assert(err, IsNil)
|
|
||||||
writer.Write([]byte(object))
|
|
||||||
writer.Close()
|
|
||||||
c.Assert(err, IsNil)
|
|
||||||
}
|
|
||||||
sort.Strings(objects)
|
|
||||||
results, err = donut.ListObjectsInBucket("foo", "")
|
|
||||||
c.Assert(err, IsNil)
|
|
||||||
c.Assert(len(results), Equals, 10)
|
|
||||||
sort.Strings(results)
|
|
||||||
c.Assert(results, DeepEquals, objects)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *MySuite) TestBucketMetadata(c *C) {
|
|
||||||
donut := NewDonutMem()
|
|
||||||
donut.CreateBucket("foo")
|
|
||||||
|
|
||||||
metadata := make(map[string]string)
|
|
||||||
|
|
||||||
metadata["hello"] = "world"
|
|
||||||
metadata["foo"] = "bar"
|
|
||||||
|
|
||||||
err := donut.SetBucketMetadata("foo", metadata)
|
|
||||||
c.Assert(err, IsNil)
|
|
||||||
|
|
||||||
result, err := donut.GetBucketMetadata("foo")
|
|
||||||
c.Assert(result, DeepEquals, metadata)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *MySuite) TestObjectMetadata(c *C) {
|
|
||||||
donut := NewDonutMem()
|
|
||||||
donut.CreateBucket("foo")
|
|
||||||
|
|
||||||
metadata := make(map[string]string)
|
|
||||||
|
|
||||||
metadata["hello"] = "world"
|
|
||||||
metadata["foo"] = "bar"
|
|
||||||
|
|
||||||
result, err := donut.GetObjectMetadata("foo", "bar", 1)
|
|
||||||
c.Assert(result, IsNil)
|
|
||||||
c.Assert(err, Not(IsNil))
|
|
||||||
|
|
||||||
writer, err := donut.GetObjectWriter("foo", "bar", 1)
|
|
||||||
c.Assert(err, IsNil)
|
|
||||||
_, err = writer.Write([]byte("Hello World"))
|
|
||||||
c.Assert(err, IsNil)
|
|
||||||
writer.SetMetadata(metadata)
|
|
||||||
err = writer.Close()
|
|
||||||
c.Assert(err, IsNil)
|
|
||||||
|
|
||||||
expectedMetadata := make(map[string]string)
|
|
||||||
for k, v := range metadata {
|
|
||||||
expectedMetadata[k] = v
|
|
||||||
}
|
|
||||||
expectedMetadata["key"] = "bar"
|
|
||||||
expectedMetadata["column"] = "1"
|
|
||||||
|
|
||||||
result, err = donut.GetObjectMetadata("foo", "bar", 1)
|
|
||||||
c.Assert(err, IsNil)
|
|
||||||
c.Assert(result, DeepEquals, expectedMetadata)
|
|
||||||
|
|
||||||
result, err = donut.GetObjectMetadata("foo", "bar", 0)
|
|
||||||
c.Assert(err, Not(IsNil))
|
|
||||||
c.Assert(result, IsNil)
|
|
||||||
}
|
|
|
@ -17,38 +17,23 @@
|
||||||
package donutstorage
|
package donutstorage
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
|
||||||
"sort"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"crypto/md5"
|
|
||||||
"encoding/hex"
|
|
||||||
|
|
||||||
"github.com/minio-io/minio/pkg/donutbox"
|
|
||||||
"github.com/minio-io/minio/pkg/encoding/erasure"
|
|
||||||
"github.com/minio-io/minio/pkg/storage"
|
"github.com/minio-io/minio/pkg/storage"
|
||||||
"github.com/minio-io/minio/pkg/utils/split"
|
"io"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DonutDriver creates a new single disk storage driver using donut without encoding.
|
// DonutDriver creates a new single disk storage driver using donut without encoding.
|
||||||
type DonutDriver struct {
|
type DonutDriver struct{}
|
||||||
donutBox donutbox.DonutBox
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
const (
|
||||||
blockSize = 10 * 1024 * 1024
|
blockSize = 10 * 1024 * 1024
|
||||||
)
|
)
|
||||||
|
|
||||||
// Start a single disk subsystem
|
// Start a single disk subsystem
|
||||||
func Start(donutBox donutbox.DonutBox) (chan<- string, <-chan error, storage.Storage) {
|
func Start() (chan<- string, <-chan error, storage.Storage) {
|
||||||
ctrlChannel := make(chan string)
|
ctrlChannel := make(chan string)
|
||||||
errorChannel := make(chan error)
|
errorChannel := make(chan error)
|
||||||
s := new(DonutDriver)
|
s := new(DonutDriver)
|
||||||
s.donutBox = donutBox
|
|
||||||
go start(ctrlChannel, errorChannel, s)
|
go start(ctrlChannel, errorChannel, s)
|
||||||
return ctrlChannel, errorChannel, s
|
return ctrlChannel, errorChannel, s
|
||||||
}
|
}
|
||||||
|
@ -59,60 +44,17 @@ func start(ctrlChannel <-chan string, errorChannel chan<- error, s *DonutDriver)
|
||||||
|
|
||||||
// ListBuckets returns a list of buckets
|
// ListBuckets returns a list of buckets
|
||||||
func (donutStorage DonutDriver) ListBuckets() (results []storage.BucketMetadata, err error) {
|
func (donutStorage DonutDriver) ListBuckets() (results []storage.BucketMetadata, err error) {
|
||||||
buckets, err := donutStorage.donutBox.ListBuckets()
|
return nil, errors.New("Not Implemented")
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
sort.Strings(buckets)
|
|
||||||
for _, bucket := range buckets {
|
|
||||||
metadata, err := donutStorage.donutBox.GetBucketMetadata(bucket)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
created, err := time.Parse(time.RFC3339Nano, metadata["created"])
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
bucketMetadata := storage.BucketMetadata{
|
|
||||||
Name: bucket,
|
|
||||||
Created: created,
|
|
||||||
}
|
|
||||||
results = append(results, bucketMetadata)
|
|
||||||
}
|
|
||||||
return results, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// CreateBucket creates a new bucket
|
// CreateBucket creates a new bucket
|
||||||
func (donutStorage DonutDriver) CreateBucket(bucket string) error {
|
func (donutStorage DonutDriver) CreateBucket(bucket string) error {
|
||||||
err := donutStorage.donutBox.CreateBucket(bucket)
|
return errors.New("Not Implemented")
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
metadataBucket := storage.BucketMetadata{
|
|
||||||
Name: bucket,
|
|
||||||
Created: time.Now(),
|
|
||||||
}
|
|
||||||
metadata := createBucketMetadata(metadataBucket)
|
|
||||||
err = donutStorage.donutBox.SetBucketMetadata(bucket, metadata)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetBucketMetadata retrieves an bucket's metadata
|
// GetBucketMetadata retrieves an bucket's metadata
|
||||||
func (donutStorage DonutDriver) GetBucketMetadata(bucket string) (storage.BucketMetadata, error) {
|
func (donutStorage DonutDriver) GetBucketMetadata(bucket string) (storage.BucketMetadata, error) {
|
||||||
metadata, err := donutStorage.donutBox.GetBucketMetadata(bucket)
|
return storage.BucketMetadata{}, errors.New("Not Implemented")
|
||||||
if err != nil {
|
|
||||||
return storage.BucketMetadata{}, err
|
|
||||||
}
|
|
||||||
created, err := time.Parse(time.RFC3339Nano, metadata["created"])
|
|
||||||
bucketMetadata := storage.BucketMetadata{
|
|
||||||
Name: bucket,
|
|
||||||
Created: created,
|
|
||||||
}
|
|
||||||
return bucketMetadata, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// CreateBucketPolicy sets a bucket's access policy
|
// CreateBucketPolicy sets a bucket's access policy
|
||||||
|
@ -127,72 +69,7 @@ func (donutStorage DonutDriver) GetBucketPolicy(bucket string) (storage.BucketPo
|
||||||
|
|
||||||
// GetObject retrieves an object and writes it to a writer
|
// GetObject retrieves an object and writes it to a writer
|
||||||
func (donutStorage DonutDriver) GetObject(target io.Writer, bucket, key string) (int64, error) {
|
func (donutStorage DonutDriver) GetObject(target io.Writer, bucket, key string) (int64, error) {
|
||||||
metadata, err := donutStorage.donutBox.GetObjectMetadata(bucket, key, 0)
|
return 0, errors.New("Not Implemented")
|
||||||
if err != nil {
|
|
||||||
// TODO strongly type and properly handle error cases
|
|
||||||
return 0, storage.ObjectNotFound{Bucket: bucket, Object: key}
|
|
||||||
}
|
|
||||||
k, err := strconv.Atoi(metadata["erasureK"])
|
|
||||||
if err != nil {
|
|
||||||
return 0, errors.New("Cannot parse erasureK")
|
|
||||||
}
|
|
||||||
m, err := strconv.Atoi(metadata["erasureM"])
|
|
||||||
if err != nil {
|
|
||||||
return 0, errors.New("Cannot parse erasureM")
|
|
||||||
}
|
|
||||||
columnCount := k + m
|
|
||||||
bs, err := strconv.Atoi(metadata["blockSize"])
|
|
||||||
if err != nil {
|
|
||||||
return 0, errors.New("Cannot parse blockSize")
|
|
||||||
}
|
|
||||||
size, err := strconv.Atoi(metadata["size"])
|
|
||||||
if err != nil {
|
|
||||||
return 0, errors.New("Cannot parse length")
|
|
||||||
}
|
|
||||||
chunkCount := size/bs + 1
|
|
||||||
var readers []io.Reader
|
|
||||||
for column := 0; column < columnCount; column++ {
|
|
||||||
reader, err := donutStorage.donutBox.GetObjectReader(bucket, key, uint(column))
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
readers = append(readers, reader)
|
|
||||||
}
|
|
||||||
|
|
||||||
totalWritten := int64(size)
|
|
||||||
totalRemaining := int64(size)
|
|
||||||
if err != err {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
params, err := erasure.ParseEncoderParams(8, 8, erasure.Cauchy)
|
|
||||||
decoder := erasure.NewEncoder(params)
|
|
||||||
for chunk := 0; chunk < chunkCount; chunk++ {
|
|
||||||
blocks := make([][]byte, columnCount)
|
|
||||||
for column := 0; column < columnCount; column++ {
|
|
||||||
var block bytes.Buffer
|
|
||||||
limitReader := io.LimitReader(readers[column], int64(blockSize))
|
|
||||||
_, err := io.Copy(&block, limitReader)
|
|
||||||
if err != nil {
|
|
||||||
return totalWritten, err
|
|
||||||
}
|
|
||||||
blocks[column] = block.Bytes()
|
|
||||||
}
|
|
||||||
curBlockSize := blockSize
|
|
||||||
if totalRemaining < int64(blockSize) {
|
|
||||||
curBlockSize = int(totalRemaining)
|
|
||||||
}
|
|
||||||
original, err := decoder.Decode(blocks, curBlockSize)
|
|
||||||
if err != nil {
|
|
||||||
return totalWritten, err
|
|
||||||
}
|
|
||||||
curWritten, err := io.Copy(target, bytes.NewBuffer(original))
|
|
||||||
totalRemaining = totalRemaining - curWritten
|
|
||||||
if err != nil {
|
|
||||||
return totalWritten, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return totalWritten, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetPartialObject retrieves an object and writes it to a writer
|
// GetPartialObject retrieves an object and writes it to a writer
|
||||||
|
@ -202,192 +79,15 @@ func (donutStorage DonutDriver) GetPartialObject(w io.Writer, bucket, object str
|
||||||
|
|
||||||
// GetObjectMetadata retrieves an object's metadata
|
// GetObjectMetadata retrieves an object's metadata
|
||||||
func (donutStorage DonutDriver) GetObjectMetadata(bucket, key string, prefix string) (storage.ObjectMetadata, error) {
|
func (donutStorage DonutDriver) GetObjectMetadata(bucket, key string, prefix string) (storage.ObjectMetadata, error) {
|
||||||
metadata, err := donutStorage.donutBox.GetObjectMetadata(bucket, key, 0)
|
return storage.ObjectMetadata{}, errors.New("Not Implemented")
|
||||||
if err != nil {
|
|
||||||
return storage.ObjectMetadata{}, err
|
|
||||||
}
|
|
||||||
created, err := time.Parse(time.RFC3339Nano, metadata["created"])
|
|
||||||
size, err := strconv.ParseInt(metadata["size"], 10, 64)
|
|
||||||
objectMetadata := storage.ObjectMetadata{
|
|
||||||
Bucket: bucket,
|
|
||||||
Key: key,
|
|
||||||
ContentType: metadata["contentType"],
|
|
||||||
Created: created,
|
|
||||||
Md5: metadata["md5"],
|
|
||||||
Size: size,
|
|
||||||
}
|
|
||||||
return objectMetadata, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListObjects lists objects
|
// ListObjects lists objects
|
||||||
func (donutStorage DonutDriver) ListObjects(bucket string, resources storage.BucketResourcesMetadata) ([]storage.ObjectMetadata, storage.BucketResourcesMetadata, error) {
|
func (donutStorage DonutDriver) ListObjects(bucket string, resources storage.BucketResourcesMetadata) ([]storage.ObjectMetadata, storage.BucketResourcesMetadata, error) {
|
||||||
objects, err := donutStorage.donutBox.ListObjectsInBucket(bucket, resources.Prefix)
|
return nil, storage.BucketResourcesMetadata{}, errors.New("Not Implemented")
|
||||||
if err != nil {
|
|
||||||
return nil, storage.BucketResourcesMetadata{}, err
|
|
||||||
}
|
|
||||||
var results []storage.ObjectMetadata
|
|
||||||
sort.Strings(objects)
|
|
||||||
for _, object := range withoutDelimiter(objects, resources.Prefix, resources.Delimiter) {
|
|
||||||
if len(results) < resources.Maxkeys {
|
|
||||||
objectMetadata, err := donutStorage.GetObjectMetadata(bucket, object, "")
|
|
||||||
if err != nil {
|
|
||||||
return nil, storage.BucketResourcesMetadata{}, err
|
|
||||||
}
|
|
||||||
results = append(results, objectMetadata)
|
|
||||||
} else {
|
|
||||||
resources.IsTruncated = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if resources.Delimiter != "" {
|
|
||||||
objects = trimPrefixWithDelimiter(objects, resources.Prefix, resources.Delimiter)
|
|
||||||
objects = beforeDelimiter(objects, resources.Delimiter)
|
|
||||||
resources.CommonPrefixes = objects
|
|
||||||
}
|
|
||||||
return results, resources, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func appendUniq(slice []string, i string) []string {
|
|
||||||
for _, ele := range slice {
|
|
||||||
if ele == i {
|
|
||||||
return slice
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return append(slice, i)
|
|
||||||
}
|
|
||||||
|
|
||||||
func withoutDelimiter(inputs []string, prefix, delim string) (results []string) {
|
|
||||||
if delim == "" {
|
|
||||||
return inputs
|
|
||||||
}
|
|
||||||
for _, input := range inputs {
|
|
||||||
input = strings.TrimPrefix(input, prefix)
|
|
||||||
if !strings.Contains(input, delim) {
|
|
||||||
results = appendUniq(results, prefix+input)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return results
|
|
||||||
}
|
|
||||||
|
|
||||||
func trimPrefixWithDelimiter(inputs []string, prefix, delim string) (results []string) {
|
|
||||||
for _, input := range inputs {
|
|
||||||
input = strings.TrimPrefix(input, prefix)
|
|
||||||
if strings.Contains(input, delim) {
|
|
||||||
results = appendUniq(results, input)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return results
|
|
||||||
}
|
|
||||||
|
|
||||||
func beforeDelimiter(inputs []string, delim string) (results []string) {
|
|
||||||
for _, input := range inputs {
|
|
||||||
results = appendUniq(results, strings.Split(input, delim)[0]+delim)
|
|
||||||
}
|
|
||||||
return results
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// CreateObject creates a new object
|
// CreateObject creates a new object
|
||||||
func (donutStorage DonutDriver) CreateObject(bucketKey, objectKey, contentType, md5sum string, reader io.Reader) error {
|
func (donutStorage DonutDriver) CreateObject(bucketKey, objectKey, contentType, md5sum string, reader io.Reader) error {
|
||||||
// set defaults
|
return errors.New("Not Implemented")
|
||||||
if contentType == "" {
|
|
||||||
contentType = "application/octet-stream"
|
|
||||||
}
|
|
||||||
contentType = strings.TrimSpace(contentType)
|
|
||||||
// split stream
|
|
||||||
splitStream := split.Stream(reader, uint64(blockSize))
|
|
||||||
writers := make([]*donutbox.NewObject, 16)
|
|
||||||
for i := 0; i < 16; i++ {
|
|
||||||
newWriter, err := donutStorage.donutBox.GetObjectWriter(bucketKey, objectKey, uint(i))
|
|
||||||
if err != nil {
|
|
||||||
closeAllWritersWithError(writers, err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
writers[i] = newWriter
|
|
||||||
}
|
|
||||||
totalLength := uint64(0)
|
|
||||||
chunkCount := 0
|
|
||||||
hasher := md5.New()
|
|
||||||
for chunk := range splitStream {
|
|
||||||
params, err := erasure.ParseEncoderParams(8, 8, erasure.Cauchy)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
hasher.Write(chunk.Data)
|
|
||||||
totalLength = totalLength + uint64(len(chunk.Data))
|
|
||||||
chunkCount = chunkCount + 1
|
|
||||||
encoder := erasure.NewEncoder(params)
|
|
||||||
if chunk.Err == nil {
|
|
||||||
parts, _ := encoder.Encode(chunk.Data)
|
|
||||||
for index, part := range parts {
|
|
||||||
if _, err := writers[index].Write(part); err != nil {
|
|
||||||
closeAllWritersWithError(writers, err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
closeAllWritersWithError(writers, chunk.Err)
|
|
||||||
return chunk.Err
|
|
||||||
}
|
|
||||||
// encode data
|
|
||||||
// write
|
|
||||||
}
|
|
||||||
// close connections
|
|
||||||
|
|
||||||
metadataObj := storage.ObjectMetadata{
|
|
||||||
Bucket: bucketKey,
|
|
||||||
Key: objectKey,
|
|
||||||
|
|
||||||
ContentType: contentType,
|
|
||||||
Created: time.Now(),
|
|
||||||
Md5: hex.EncodeToString(hasher.Sum(nil)),
|
|
||||||
Size: int64(totalLength),
|
|
||||||
}
|
|
||||||
|
|
||||||
metadata := createObjectMetadata(metadataObj, blockSize, 8, 8, "Cauchy")
|
|
||||||
|
|
||||||
for column := uint(0); column < 16; column++ {
|
|
||||||
writers[column].SetMetadata(metadata)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO capture errors in writers, enough should pass before returning
|
|
||||||
closeAllWriters(writers)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func closeAllWriters(writers []*donutbox.NewObject) {
|
|
||||||
for _, writer := range writers {
|
|
||||||
if writer != nil {
|
|
||||||
writer.Close()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func closeAllWritersWithError(writers []*donutbox.NewObject, err error) {
|
|
||||||
for _, writer := range writers {
|
|
||||||
if writer != nil {
|
|
||||||
writer.CloseWithError(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func createBucketMetadata(metadataBucket storage.BucketMetadata) map[string]string {
|
|
||||||
metadata := make(map[string]string)
|
|
||||||
metadata["bucket"] = metadataBucket.Name
|
|
||||||
metadata["created"] = metadataBucket.Created.Format(time.RFC3339Nano)
|
|
||||||
return metadata
|
|
||||||
}
|
|
||||||
|
|
||||||
func createObjectMetadata(metadataObject storage.ObjectMetadata, blockSize int, k, m uint8, technique string) map[string]string {
|
|
||||||
metadata := make(map[string]string)
|
|
||||||
metadata["bucket"] = metadataObject.Bucket
|
|
||||||
metadata["key"] = metadataObject.Key
|
|
||||||
metadata["contentType"] = metadataObject.ContentType
|
|
||||||
metadata["created"] = metadataObject.Created.Format(time.RFC3339Nano)
|
|
||||||
metadata["md5"] = metadataObject.Md5
|
|
||||||
metadata["size"] = strconv.FormatInt(metadataObject.Size, 10)
|
|
||||||
metadata["blockSize"] = strconv.FormatUint(uint64(blockSize), 10)
|
|
||||||
metadata["erasureK"] = strconv.FormatUint(uint64(k), 10)
|
|
||||||
metadata["erasureM"] = strconv.FormatUint(uint64(m), 10)
|
|
||||||
metadata["erasureTechnique"] = technique
|
|
||||||
return metadata
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,7 +23,6 @@ import (
|
||||||
|
|
||||||
mstorage "github.com/minio-io/minio/pkg/storage"
|
mstorage "github.com/minio-io/minio/pkg/storage"
|
||||||
|
|
||||||
"github.com/minio-io/minio/pkg/donutbox/donutmem"
|
|
||||||
. "gopkg.in/check.v1"
|
. "gopkg.in/check.v1"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -34,12 +33,13 @@ type MySuite struct{}
|
||||||
var _ = Suite(&MySuite{})
|
var _ = Suite(&MySuite{})
|
||||||
|
|
||||||
func (s *MySuite) TestAPISuite(c *C) {
|
func (s *MySuite) TestAPISuite(c *C) {
|
||||||
|
c.Skip("Not Implemented")
|
||||||
var storageList []string
|
var storageList []string
|
||||||
create := func() mstorage.Storage {
|
create := func() mstorage.Storage {
|
||||||
path, err := ioutil.TempDir(os.TempDir(), "minio-fs-")
|
path, err := ioutil.TempDir(os.TempDir(), "minio-fs-")
|
||||||
c.Check(err, IsNil)
|
c.Check(err, IsNil)
|
||||||
storageList = append(storageList, path)
|
storageList = append(storageList, path)
|
||||||
_, _, store := Start(donutmem.NewDonutMem()) // TODO Make InMemory driver
|
_, _, store := Start() // TODO Make InMemory driver
|
||||||
return store
|
return store
|
||||||
}
|
}
|
||||||
mstorage.APITestSuite(c, create)
|
mstorage.APITestSuite(c, create)
|
||||||
|
|
|
@ -24,6 +24,7 @@ import (
|
||||||
mstorage "github.com/minio-io/minio/pkg/storage"
|
mstorage "github.com/minio-io/minio/pkg/storage"
|
||||||
|
|
||||||
. "gopkg.in/check.v1"
|
. "gopkg.in/check.v1"
|
||||||
|
"log"
|
||||||
)
|
)
|
||||||
|
|
||||||
func Test(t *testing.T) { TestingT(t) }
|
func Test(t *testing.T) { TestingT(t) }
|
||||||
|
@ -46,6 +47,7 @@ func (s *MySuite) TestAPISuite(c *C) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func removeRoots(c *C, roots []string) {
|
func removeRoots(c *C, roots []string) {
|
||||||
|
log.Println(roots)
|
||||||
for _, root := range roots {
|
for _, root := range roots {
|
||||||
err := os.RemoveAll(root)
|
err := os.RemoveAll(root)
|
||||||
c.Check(err, IsNil)
|
c.Check(err, IsNil)
|
||||||
|
|
Loading…
Reference in a new issue