From cf2550b2676d0effa70692e45f89706bcb05d2c6 Mon Sep 17 00:00:00 2001 From: "Frederick F. Kautz IV" Date: Tue, 24 Mar 2015 19:58:03 -0700 Subject: [PATCH] Populate buckets at start --- pkg/storage/donut/bucketdriver.go | 25 ++++++++++++++++++-- pkg/storage/donut/donut.go | 1 + pkg/storage/donut/donutdriver.go | 29 +++++++++++++++++++---- pkg/storage/donut/donutdriver_test.go | 30 ++++++++++++++++-------- pkg/storage/donut/local.go | 14 +++++++++-- pkg/storage/donutstorage/donutstorage.go | 6 ++++- 6 files changed, 86 insertions(+), 19 deletions(-) diff --git a/pkg/storage/donut/bucketdriver.go b/pkg/storage/donut/bucketdriver.go index 783fa9878..480a5920d 100644 --- a/pkg/storage/donut/bucketdriver.go +++ b/pkg/storage/donut/bucketdriver.go @@ -1,8 +1,13 @@ package donut +import ( + "errors" + "strconv" + "strings" +) + type bucketDriver struct { - nodes []string - objects map[string][]byte + nodes []string } func (b bucketDriver) GetNodes() ([]string, error) { @@ -12,3 +17,19 @@ func (b bucketDriver) GetNodes() ([]string, error) { } return nodes, nil } + +func (b bucketDriver) AddNode(nodeID, bucketID string) error { + tokens := strings.Split(bucketID, ":") + if len(tokens) != 3 { + return errors.New("Bucket ID malformed: " + bucketID) + } + // bucketName := tokens[0] + // aggregate := tokens[1] + // aggregate := "0" + part, err := strconv.Atoi(tokens[2]) + if err != nil { + return errors.New("Part malformed: " + tokens[2]) + } + b.nodes[part] = nodeID + return nil +} diff --git a/pkg/storage/donut/donut.go b/pkg/storage/donut/donut.go index 0514f91af..debbd915e 100644 --- a/pkg/storage/donut/donut.go +++ b/pkg/storage/donut/donut.go @@ -19,6 +19,7 @@ type Donut interface { // Bucket interface type Bucket interface { GetNodes() ([]string, error) + AddNode(nodeID, bucketID string) error } // Node interface diff --git a/pkg/storage/donut/donutdriver.go b/pkg/storage/donut/donutdriver.go index f06555a75..6c2851a8a 100644 --- a/pkg/storage/donut/donutdriver.go +++ b/pkg/storage/donut/donutdriver.go @@ -14,14 +14,35 @@ type donutDriver struct { } // NewDonutDriver - instantiate new donut driver -func NewDonutDriver(root string) Donut { +func NewDonutDriver(root string) (Donut, error) { nodes := make(map[string]Node) - nodes["localhost"] = localDirectoryNode{root: root} - driver := donutDriver{ + nodes["localhost"] = &localDirectoryNode{root: root} + driver := &donutDriver{ buckets: make(map[string]Bucket), nodes: nodes, } - return driver + for nodeID, node := range nodes { + bucketIDs, err := node.GetBuckets() + if err != nil { + return nil, err + } + for _, bucketID := range bucketIDs { + tokens := strings.Split(bucketID, ":") + if _, ok := driver.buckets[tokens[0]]; ok { + // found bucket, skip creating + } else { + bucket := bucketDriver{ + nodes: make([]string, 16), + } + // TODO catch errors + driver.buckets[tokens[0]] = bucket + } + if err = driver.buckets[tokens[0]].AddNode(nodeID, bucketID); err != nil { + return nil, err + } + } + } + return driver, nil } func (driver donutDriver) CreateBucket(bucketName string) error { diff --git a/pkg/storage/donut/donutdriver_test.go b/pkg/storage/donut/donutdriver_test.go index 19cde193e..edf0921df 100644 --- a/pkg/storage/donut/donutdriver_test.go +++ b/pkg/storage/donut/donutdriver_test.go @@ -21,7 +21,8 @@ func (s *MySuite) TestEmptyBucket(c *C) { root, err := ioutil.TempDir(os.TempDir(), "donut-") c.Assert(err, IsNil) defer os.RemoveAll(root) - donut := NewDonutDriver(root) + donut, err := NewDonutDriver(root) + c.Assert(err, IsNil) // check buckets are empty buckets, err := donut.ListBuckets() @@ -33,7 +34,8 @@ func (s *MySuite) TestBucketWithoutNameFails(c *C) { root, err := ioutil.TempDir(os.TempDir(), "donut-") c.Assert(err, IsNil) defer os.RemoveAll(root) - donut := NewDonutDriver(root) + donut, err := NewDonutDriver(root) + c.Assert(err, IsNil) // fail to create new bucket without a name err = donut.CreateBucket("") c.Assert(err, Not(IsNil)) @@ -46,7 +48,8 @@ func (s *MySuite) TestCreateBucketAndList(c *C) { root, err := ioutil.TempDir(os.TempDir(), "donut-") c.Assert(err, IsNil) defer os.RemoveAll(root) - donut := NewDonutDriver(root) + donut, err := NewDonutDriver(root) + c.Assert(err, IsNil) // create bucket err = donut.CreateBucket("foo") c.Assert(err, IsNil) @@ -61,7 +64,8 @@ func (s *MySuite) TestCreateBucketWithSameNameFails(c *C) { root, err := ioutil.TempDir(os.TempDir(), "donut-") c.Assert(err, IsNil) defer os.RemoveAll(root) - donut := NewDonutDriver(root) + donut, err := NewDonutDriver(root) + c.Assert(err, IsNil) err = donut.CreateBucket("foo") c.Assert(err, IsNil) @@ -73,7 +77,8 @@ func (s *MySuite) TestCreateMultipleBucketsAndList(c *C) { root, err := ioutil.TempDir(os.TempDir(), "donut-") c.Assert(err, IsNil) defer os.RemoveAll(root) - donut := NewDonutDriver(root) + donut, err := NewDonutDriver(root) + c.Assert(err, IsNil) // add a second bucket err = donut.CreateBucket("foo") c.Assert(err, IsNil) @@ -97,7 +102,8 @@ func (s *MySuite) TestNewObjectFailsWithoutBucket(c *C) { root, err := ioutil.TempDir(os.TempDir(), "donut-") c.Assert(err, IsNil) defer os.RemoveAll(root) - donut := NewDonutDriver(root) + donut, err := NewDonutDriver(root) + c.Assert(err, IsNil) writer, err := donut.GetObjectWriter("foo", "obj") c.Assert(err, Not(IsNil)) @@ -108,7 +114,8 @@ func (s *MySuite) TestNewObjectFailsWithEmptyName(c *C) { root, err := ioutil.TempDir(os.TempDir(), "donut-") c.Assert(err, IsNil) defer os.RemoveAll(root) - donut := NewDonutDriver(root) + donut, err := NewDonutDriver(root) + c.Assert(err, IsNil) writer, err := donut.GetObjectWriter("foo", "") c.Assert(err, Not(IsNil)) @@ -123,7 +130,8 @@ func (s *MySuite) TestNewObjectCanBeWritten(c *C) { root, err := ioutil.TempDir(os.TempDir(), "donut-") c.Assert(err, IsNil) defer os.RemoveAll(root) - donut := NewDonutDriver(root) + donut, err := NewDonutDriver(root) + c.Assert(err, IsNil) err = donut.CreateBucket("foo") c.Assert(err, IsNil) @@ -175,7 +183,8 @@ func (s *MySuite) TestMultipleNewObjects(c *C) { root, err := ioutil.TempDir(os.TempDir(), "donut-") c.Assert(err, IsNil) defer os.RemoveAll(root) - donut := NewDonutDriver(root) + donut, err := NewDonutDriver(root) + c.Assert(err, IsNil) c.Assert(donut.CreateBucket("foo"), IsNil) writer, err := donut.GetObjectWriter("foo", "obj1") @@ -215,7 +224,8 @@ func (s *MySuite) TestSysPrefixShouldFail(c *C) { root, err := ioutil.TempDir(os.TempDir(), "donut-") c.Assert(err, IsNil) defer os.RemoveAll(root) - donut := NewDonutDriver(root) + donut, err := NewDonutDriver(root) + c.Assert(err, IsNil) c.Assert(donut.CreateBucket("foo"), IsNil) writer, err := donut.GetObjectWriter("foo", "obj1") diff --git a/pkg/storage/donut/local.go b/pkg/storage/donut/local.go index c1fcff915..6ed994eab 100644 --- a/pkg/storage/donut/local.go +++ b/pkg/storage/donut/local.go @@ -1,7 +1,6 @@ package donut import ( - "errors" "io" "os" "path" @@ -9,6 +8,7 @@ import ( "strings" "encoding/json" + "io/ioutil" "path/filepath" ) @@ -22,7 +22,17 @@ func (d localDirectoryNode) CreateBucket(bucket string) error { } func (d localDirectoryNode) GetBuckets() ([]string, error) { - return nil, errors.New("Not Implemented") + files, err := ioutil.ReadDir(d.root) + if err != nil { + return nil, err + } + var results []string + for _, file := range files { + if file.IsDir() { + results = append(results, file.Name()) + } + } + return results, nil } func (d localDirectoryNode) GetWriter(bucket, object string) (Writer, error) { diff --git a/pkg/storage/donutstorage/donutstorage.go b/pkg/storage/donutstorage/donutstorage.go index 19e345c05..15e6f2231 100644 --- a/pkg/storage/donutstorage/donutstorage.go +++ b/pkg/storage/donutstorage/donutstorage.go @@ -44,7 +44,11 @@ func Start(path string) (chan<- string, <-chan error, storage.Storage) { s := new(Storage) // TODO donut driver should be passed in as Start param and driven by config - s.donut = donut.NewDonutDriver(path) + var err error + s.donut, err = donut.NewDonutDriver(path) + if err != nil { + errorChannel <- err + } go start(ctrlChannel, errorChannel, s) return ctrlChannel, errorChannel, s