maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge pull request #357 from fkautz/pr_out_populate_buckets_at_start

Browse source
This commit is contained in:
Frederick F. Kautz IV 2015-03-24 21:04:48 -07:00
parent a311a9c4b8 dd5ba865ff
commit 3fc54eea3a
6 changed files with 86 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
pkg/drivers/donut/donut.go
View file

@ -44,7 +44,11 @@ func Start(path string) (chan<- string, <-chan error, drivers.Driver) {
s := new(donutDriver)
// TODO donut driver should be passed in as Start param and driven by config
s.donut = donut.NewDonut(path)
var err error
s.donut, err = donut.NewDonut(path)
if err != nil {
errorChannel <- err
}
go start(ctrlChannel, errorChannel, s)
return ctrlChannel, errorChannel, s

22
pkg/storage/donut/bucket.go
View file

@ -1,5 +1,11 @@
package donut
import (
"errors"
"strconv"
"strings"
)
type donutBucket struct {
nodes []string
objects map[string][]byte
@ -13,3 +19,19 @@ func (b donutBucket) GetNodes() ([]string, error) {
}
return nodes, nil
}
func (b donutBucket) AddNode(nodeID, bucketID string) error {
tokens := strings.Split(bucketID, ":")
if len(tokens) != 3 {
return errors.New("Bucket ID malformed: " + bucketID)
}
// bucketName := tokens[0]
// aggregate := tokens[1]
// aggregate := "0"
part, err := strconv.Atoi(tokens[2])
if err != nil {
return errors.New("Part malformed: " + tokens[2])
}
b.nodes[part] = nodeID
return nil
}

31
pkg/storage/donut/donut.go
View file

@ -13,15 +13,36 @@ type donut struct {
nodes map[string]Node
}
// NewDonut - instantiate new donut
func NewDonut(root string) Donut {
// NewDonut - instantiate new donut driver
func NewDonut(root string) (Donut, error) {
nodes := make(map[string]Node)
nodes["localhost"] = localDirectoryNode{root: root}
d := donut{
nodes["localhost"] = &localDirectoryNode{root: root}
driver := &donut{
buckets: make(map[string]Bucket),
nodes: nodes,
}
return d
for nodeID, node := range nodes {
bucketIDs, err := node.GetBuckets()
if err != nil {
return nil, err
}
for _, bucketID := range bucketIDs {
tokens := strings.Split(bucketID, ":")
if _, ok := driver.buckets[tokens[0]]; ok {
// found bucket, skip creating
} else {
bucket := donutBucket{
nodes: make([]string, 16),
}
// TODO catch errors
driver.buckets[tokens[0]] = bucket
}
if err = driver.buckets[tokens[0]].AddNode(nodeID, bucketID); err != nil {
return nil, err
}
}
}
return driver, nil
}
// CreateBucket - create a new bucket

30
pkg/storage/donut/donut_test.go
View file

@ -21,7 +21,8 @@ func (s *MySuite) TestEmptyBucket(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut := NewDonut(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
// check buckets are empty
buckets, err := donut.ListBuckets()
@ -33,7 +34,8 @@ func (s *MySuite) TestBucketWithoutNameFails(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut := NewDonut(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
// fail to create new bucket without a name
err = donut.CreateBucket("")
c.Assert(err, Not(IsNil))
@ -46,7 +48,8 @@ func (s *MySuite) TestCreateBucketAndList(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut := NewDonut(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
// create bucket
err = donut.CreateBucket("foo")
c.Assert(err, IsNil)
@ -61,7 +64,8 @@ func (s *MySuite) TestCreateBucketWithSameNameFails(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut := NewDonut(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
err = donut.CreateBucket("foo")
c.Assert(err, IsNil)
@ -73,7 +77,8 @@ func (s *MySuite) TestCreateMultipleBucketsAndList(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut := NewDonut(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
// add a second bucket
err = donut.CreateBucket("foo")
c.Assert(err, IsNil)
@ -97,7 +102,8 @@ func (s *MySuite) TestNewObjectFailsWithoutBucket(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut := NewDonut(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
writer, err := donut.GetObjectWriter("foo", "obj")
c.Assert(err, Not(IsNil))
@ -108,7 +114,8 @@ func (s *MySuite) TestNewObjectFailsWithEmptyName(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut := NewDonut(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
writer, err := donut.GetObjectWriter("foo", "")
c.Assert(err, Not(IsNil))
@ -123,7 +130,8 @@ func (s *MySuite) TestNewObjectCanBeWritten(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut := NewDonut(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
err = donut.CreateBucket("foo")
c.Assert(err, IsNil)
@ -175,7 +183,8 @@ func (s *MySuite) TestMultipleNewObjects(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut := NewDonut(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
c.Assert(donut.CreateBucket("foo"), IsNil)
writer, err := donut.GetObjectWriter("foo", "obj1")
@ -215,7 +224,8 @@ func (s *MySuite) TestSysPrefixShouldFail(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut := NewDonut(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
c.Assert(donut.CreateBucket("foo"), IsNil)
writer, err := donut.GetObjectWriter("foo", "obj1")

1
pkg/storage/donut/interfaces.go
View file

@ -19,6 +19,7 @@ type Donut interface {
// Bucket interface
type Bucket interface {
GetNodes() ([]string, error)
AddNode(nodeID, bucketID string) error
}
// Node interface

14
pkg/storage/donut/node_local.go
View file

@ -1,7 +1,6 @@
package donut
import (
"errors"
"io"
"os"
"path"
@ -9,6 +8,7 @@ import (
"strings"
"encoding/json"
"io/ioutil"
"path/filepath"
)
@ -22,7 +22,17 @@ func (d localDirectoryNode) CreateBucket(bucket string) error {
}
func (d localDirectoryNode) GetBuckets() ([]string, error) {
return nil, errors.New("Not Implemented")
files, err := ioutil.ReadDir(d.root)
if err != nil {
return nil, err
}
var results []string
for _, file := range files {
if file.IsDir() {
results = append(results, file.Name())
}
}
return results, nil
}
func (d localDirectoryNode) GetWriter(bucket, object string) (Writer, error) {