Populate buckets at start
This commit is contained in:
parent
3c4012f1e7
commit
cf2550b267
6 changed files with 86 additions and 19 deletions
|
@ -1,8 +1,13 @@
|
||||||
package donut
|
package donut
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
type bucketDriver struct {
|
type bucketDriver struct {
|
||||||
nodes []string
|
nodes []string
|
||||||
objects map[string][]byte
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b bucketDriver) GetNodes() ([]string, error) {
|
func (b bucketDriver) GetNodes() ([]string, error) {
|
||||||
|
@ -12,3 +17,19 @@ func (b bucketDriver) GetNodes() ([]string, error) {
|
||||||
}
|
}
|
||||||
return nodes, nil
|
return nodes, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (b bucketDriver) AddNode(nodeID, bucketID string) error {
|
||||||
|
tokens := strings.Split(bucketID, ":")
|
||||||
|
if len(tokens) != 3 {
|
||||||
|
return errors.New("Bucket ID malformed: " + bucketID)
|
||||||
|
}
|
||||||
|
// bucketName := tokens[0]
|
||||||
|
// aggregate := tokens[1]
|
||||||
|
// aggregate := "0"
|
||||||
|
part, err := strconv.Atoi(tokens[2])
|
||||||
|
if err != nil {
|
||||||
|
return errors.New("Part malformed: " + tokens[2])
|
||||||
|
}
|
||||||
|
b.nodes[part] = nodeID
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
|
@ -19,6 +19,7 @@ type Donut interface {
|
||||||
// Bucket interface
|
// Bucket interface
|
||||||
type Bucket interface {
|
type Bucket interface {
|
||||||
GetNodes() ([]string, error)
|
GetNodes() ([]string, error)
|
||||||
|
AddNode(nodeID, bucketID string) error
|
||||||
}
|
}
|
||||||
|
|
||||||
// Node interface
|
// Node interface
|
||||||
|
|
|
@ -14,14 +14,35 @@ type donutDriver struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewDonutDriver - instantiate new donut driver
|
// NewDonutDriver - instantiate new donut driver
|
||||||
func NewDonutDriver(root string) Donut {
|
func NewDonutDriver(root string) (Donut, error) {
|
||||||
nodes := make(map[string]Node)
|
nodes := make(map[string]Node)
|
||||||
nodes["localhost"] = localDirectoryNode{root: root}
|
nodes["localhost"] = &localDirectoryNode{root: root}
|
||||||
driver := donutDriver{
|
driver := &donutDriver{
|
||||||
buckets: make(map[string]Bucket),
|
buckets: make(map[string]Bucket),
|
||||||
nodes: nodes,
|
nodes: nodes,
|
||||||
}
|
}
|
||||||
return driver
|
for nodeID, node := range nodes {
|
||||||
|
bucketIDs, err := node.GetBuckets()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, bucketID := range bucketIDs {
|
||||||
|
tokens := strings.Split(bucketID, ":")
|
||||||
|
if _, ok := driver.buckets[tokens[0]]; ok {
|
||||||
|
// found bucket, skip creating
|
||||||
|
} else {
|
||||||
|
bucket := bucketDriver{
|
||||||
|
nodes: make([]string, 16),
|
||||||
|
}
|
||||||
|
// TODO catch errors
|
||||||
|
driver.buckets[tokens[0]] = bucket
|
||||||
|
}
|
||||||
|
if err = driver.buckets[tokens[0]].AddNode(nodeID, bucketID); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return driver, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (driver donutDriver) CreateBucket(bucketName string) error {
|
func (driver donutDriver) CreateBucket(bucketName string) error {
|
||||||
|
|
|
@ -21,7 +21,8 @@ func (s *MySuite) TestEmptyBucket(c *C) {
|
||||||
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
||||||
c.Assert(err, IsNil)
|
c.Assert(err, IsNil)
|
||||||
defer os.RemoveAll(root)
|
defer os.RemoveAll(root)
|
||||||
donut := NewDonutDriver(root)
|
donut, err := NewDonutDriver(root)
|
||||||
|
c.Assert(err, IsNil)
|
||||||
|
|
||||||
// check buckets are empty
|
// check buckets are empty
|
||||||
buckets, err := donut.ListBuckets()
|
buckets, err := donut.ListBuckets()
|
||||||
|
@ -33,7 +34,8 @@ func (s *MySuite) TestBucketWithoutNameFails(c *C) {
|
||||||
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
||||||
c.Assert(err, IsNil)
|
c.Assert(err, IsNil)
|
||||||
defer os.RemoveAll(root)
|
defer os.RemoveAll(root)
|
||||||
donut := NewDonutDriver(root)
|
donut, err := NewDonutDriver(root)
|
||||||
|
c.Assert(err, IsNil)
|
||||||
// fail to create new bucket without a name
|
// fail to create new bucket without a name
|
||||||
err = donut.CreateBucket("")
|
err = donut.CreateBucket("")
|
||||||
c.Assert(err, Not(IsNil))
|
c.Assert(err, Not(IsNil))
|
||||||
|
@ -46,7 +48,8 @@ func (s *MySuite) TestCreateBucketAndList(c *C) {
|
||||||
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
||||||
c.Assert(err, IsNil)
|
c.Assert(err, IsNil)
|
||||||
defer os.RemoveAll(root)
|
defer os.RemoveAll(root)
|
||||||
donut := NewDonutDriver(root)
|
donut, err := NewDonutDriver(root)
|
||||||
|
c.Assert(err, IsNil)
|
||||||
// create bucket
|
// create bucket
|
||||||
err = donut.CreateBucket("foo")
|
err = donut.CreateBucket("foo")
|
||||||
c.Assert(err, IsNil)
|
c.Assert(err, IsNil)
|
||||||
|
@ -61,7 +64,8 @@ func (s *MySuite) TestCreateBucketWithSameNameFails(c *C) {
|
||||||
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
||||||
c.Assert(err, IsNil)
|
c.Assert(err, IsNil)
|
||||||
defer os.RemoveAll(root)
|
defer os.RemoveAll(root)
|
||||||
donut := NewDonutDriver(root)
|
donut, err := NewDonutDriver(root)
|
||||||
|
c.Assert(err, IsNil)
|
||||||
err = donut.CreateBucket("foo")
|
err = donut.CreateBucket("foo")
|
||||||
c.Assert(err, IsNil)
|
c.Assert(err, IsNil)
|
||||||
|
|
||||||
|
@ -73,7 +77,8 @@ func (s *MySuite) TestCreateMultipleBucketsAndList(c *C) {
|
||||||
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
||||||
c.Assert(err, IsNil)
|
c.Assert(err, IsNil)
|
||||||
defer os.RemoveAll(root)
|
defer os.RemoveAll(root)
|
||||||
donut := NewDonutDriver(root)
|
donut, err := NewDonutDriver(root)
|
||||||
|
c.Assert(err, IsNil)
|
||||||
// add a second bucket
|
// add a second bucket
|
||||||
err = donut.CreateBucket("foo")
|
err = donut.CreateBucket("foo")
|
||||||
c.Assert(err, IsNil)
|
c.Assert(err, IsNil)
|
||||||
|
@ -97,7 +102,8 @@ func (s *MySuite) TestNewObjectFailsWithoutBucket(c *C) {
|
||||||
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
||||||
c.Assert(err, IsNil)
|
c.Assert(err, IsNil)
|
||||||
defer os.RemoveAll(root)
|
defer os.RemoveAll(root)
|
||||||
donut := NewDonutDriver(root)
|
donut, err := NewDonutDriver(root)
|
||||||
|
c.Assert(err, IsNil)
|
||||||
|
|
||||||
writer, err := donut.GetObjectWriter("foo", "obj")
|
writer, err := donut.GetObjectWriter("foo", "obj")
|
||||||
c.Assert(err, Not(IsNil))
|
c.Assert(err, Not(IsNil))
|
||||||
|
@ -108,7 +114,8 @@ func (s *MySuite) TestNewObjectFailsWithEmptyName(c *C) {
|
||||||
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
||||||
c.Assert(err, IsNil)
|
c.Assert(err, IsNil)
|
||||||
defer os.RemoveAll(root)
|
defer os.RemoveAll(root)
|
||||||
donut := NewDonutDriver(root)
|
donut, err := NewDonutDriver(root)
|
||||||
|
c.Assert(err, IsNil)
|
||||||
|
|
||||||
writer, err := donut.GetObjectWriter("foo", "")
|
writer, err := donut.GetObjectWriter("foo", "")
|
||||||
c.Assert(err, Not(IsNil))
|
c.Assert(err, Not(IsNil))
|
||||||
|
@ -123,7 +130,8 @@ func (s *MySuite) TestNewObjectCanBeWritten(c *C) {
|
||||||
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
||||||
c.Assert(err, IsNil)
|
c.Assert(err, IsNil)
|
||||||
defer os.RemoveAll(root)
|
defer os.RemoveAll(root)
|
||||||
donut := NewDonutDriver(root)
|
donut, err := NewDonutDriver(root)
|
||||||
|
c.Assert(err, IsNil)
|
||||||
|
|
||||||
err = donut.CreateBucket("foo")
|
err = donut.CreateBucket("foo")
|
||||||
c.Assert(err, IsNil)
|
c.Assert(err, IsNil)
|
||||||
|
@ -175,7 +183,8 @@ func (s *MySuite) TestMultipleNewObjects(c *C) {
|
||||||
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
||||||
c.Assert(err, IsNil)
|
c.Assert(err, IsNil)
|
||||||
defer os.RemoveAll(root)
|
defer os.RemoveAll(root)
|
||||||
donut := NewDonutDriver(root)
|
donut, err := NewDonutDriver(root)
|
||||||
|
c.Assert(err, IsNil)
|
||||||
|
|
||||||
c.Assert(donut.CreateBucket("foo"), IsNil)
|
c.Assert(donut.CreateBucket("foo"), IsNil)
|
||||||
writer, err := donut.GetObjectWriter("foo", "obj1")
|
writer, err := donut.GetObjectWriter("foo", "obj1")
|
||||||
|
@ -215,7 +224,8 @@ func (s *MySuite) TestSysPrefixShouldFail(c *C) {
|
||||||
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
root, err := ioutil.TempDir(os.TempDir(), "donut-")
|
||||||
c.Assert(err, IsNil)
|
c.Assert(err, IsNil)
|
||||||
defer os.RemoveAll(root)
|
defer os.RemoveAll(root)
|
||||||
donut := NewDonutDriver(root)
|
donut, err := NewDonutDriver(root)
|
||||||
|
c.Assert(err, IsNil)
|
||||||
|
|
||||||
c.Assert(donut.CreateBucket("foo"), IsNil)
|
c.Assert(donut.CreateBucket("foo"), IsNil)
|
||||||
writer, err := donut.GetObjectWriter("foo", "obj1")
|
writer, err := donut.GetObjectWriter("foo", "obj1")
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
package donut
|
package donut
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
|
@ -9,6 +8,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"io/ioutil"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -22,7 +22,17 @@ func (d localDirectoryNode) CreateBucket(bucket string) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d localDirectoryNode) GetBuckets() ([]string, error) {
|
func (d localDirectoryNode) GetBuckets() ([]string, error) {
|
||||||
return nil, errors.New("Not Implemented")
|
files, err := ioutil.ReadDir(d.root)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var results []string
|
||||||
|
for _, file := range files {
|
||||||
|
if file.IsDir() {
|
||||||
|
results = append(results, file.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return results, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d localDirectoryNode) GetWriter(bucket, object string) (Writer, error) {
|
func (d localDirectoryNode) GetWriter(bucket, object string) (Writer, error) {
|
||||||
|
|
|
@ -44,7 +44,11 @@ func Start(path string) (chan<- string, <-chan error, storage.Storage) {
|
||||||
s := new(Storage)
|
s := new(Storage)
|
||||||
|
|
||||||
// TODO donut driver should be passed in as Start param and driven by config
|
// TODO donut driver should be passed in as Start param and driven by config
|
||||||
s.donut = donut.NewDonutDriver(path)
|
var err error
|
||||||
|
s.donut, err = donut.NewDonutDriver(path)
|
||||||
|
if err != nil {
|
||||||
|
errorChannel <- err
|
||||||
|
}
|
||||||
|
|
||||||
go start(ctrlChannel, errorChannel, s)
|
go start(ctrlChannel, errorChannel, s)
|
||||||
return ctrlChannel, errorChannel, s
|
return ctrlChannel, errorChannel, s
|
||||||
|
|
Loading…
Reference in a new issue