From 03f51eac2994657a9de44cf04b1754d6bdb3f091 Mon Sep 17 00:00:00 2001 From: "Frederick F. Kautz IV" Date: Sun, 22 Mar 2015 15:33:52 -0700 Subject: [PATCH] Adding list objects to donut --- pkg/storage/donut/donut.go | 17 +++++---- pkg/storage/donut/donutdriver.go | 55 +++++++++++++++++++++------ pkg/storage/donut/donutdriver_test.go | 5 +++ 3 files changed, 58 insertions(+), 19 deletions(-) diff --git a/pkg/storage/donut/donut.go b/pkg/storage/donut/donut.go index 8e02fa16f..d6d6bf720 100644 --- a/pkg/storage/donut/donut.go +++ b/pkg/storage/donut/donut.go @@ -7,11 +7,11 @@ import "io" // Donut interface type Donut interface { CreateBucket(bucket string) error - ListBuckets() ([]string, error) - GetObjectWriter(bucket, object string) (ObjectWriter, error) GetObject(bucket, object string) (io.ReadCloser, error) GetObjectMetadata(bucket, object string) (map[string]string, error) - // ListObjects(bucket string) ([]string, error) + GetObjectWriter(bucket, object string) (ObjectWriter, error) + ListBuckets() ([]string, error) + ListObjects(bucket string) ([]string, error) } // Bucket interface @@ -22,19 +22,20 @@ type Bucket interface { // Node interface type Node interface { GetBuckets() ([]string, error) - GetWriter(bucket, object string) (Writer, error) - GetReader(bucket, object string) (io.ReadCloser, error) - GetMetadata(bucket, object string) (map[string]string, error) GetDonutMetadata(bucket, object string) (map[string]string, error) + GetMetadata(bucket, object string) (map[string]string, error) + GetReader(bucket, object string) (io.ReadCloser, error) + GetWriter(bucket, object string) (Writer, error) + ListObjects(bucket string) ([]string, error) } // ObjectWriter interface type ObjectWriter interface { - Write([]byte) (int, error) Close() error CloseWithError(error) error - SetMetadata(map[string]string) error GetMetadata() (map[string]string, error) + SetMetadata(map[string]string) error + Write([]byte) (int, error) } // Writer interface diff --git a/pkg/storage/donut/donutdriver.go b/pkg/storage/donut/donutdriver.go index b387fe2ae..ade871809 100644 --- a/pkg/storage/donut/donutdriver.go +++ b/pkg/storage/donut/donutdriver.go @@ -16,6 +16,7 @@ import ( "github.com/minio-io/minio/pkg/encoding/erasure" "github.com/minio-io/minio/pkg/utils/split" + "path/filepath" ) type donutDriver struct { @@ -128,6 +129,19 @@ func (driver donutDriver) GetObjectMetadata(bucketName, object string) (map[stri return nil, errors.New("Bucket not found") } +func (driver donutDriver) ListObjects(bucketName string) ([]string, error) { + if bucket, ok := driver.buckets[bucketName]; ok { + nodes, err := bucket.GetNodes() + if err != nil { + return nil, err + } + if node, ok := driver.nodes[nodes[0]]; ok { + return node.ListObjects(bucketName + ":0:0") + } + } + return nil, errors.New("Bucket not found") +} + func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, writer *io.PipeWriter) { totalChunks, _ := strconv.Atoi(donutMetadata["chunkCount"]) totalLeft, _ := strconv.Atoi(donutMetadata["totalLength"]) @@ -259,12 +273,12 @@ type localDirectoryNode struct { root string } -func (d localDirectoryNode) GetBuckets() ([]string, error) { +func (node localDirectoryNode) GetBuckets() ([]string, error) { return nil, errors.New("Not Implemented") } -func (d localDirectoryNode) GetWriter(bucket, object string) (Writer, error) { - objectPath := path.Join(d.root, bucket, object) +func (node localDirectoryNode) GetWriter(bucket, object string) (Writer, error) { + objectPath := path.Join(node.root, bucket, object) err := os.MkdirAll(objectPath, 0700) if err != nil { return nil, err @@ -272,19 +286,19 @@ func (d localDirectoryNode) GetWriter(bucket, object string) (Writer, error) { return newDonutFileWriter(objectPath) } -func (d localDirectoryNode) GetReader(bucket, object string) (io.ReadCloser, error) { - return os.Open(path.Join(d.root, bucket, object, "data")) +func (node localDirectoryNode) GetReader(bucket, object string) (io.ReadCloser, error) { + return os.Open(path.Join(node.root, bucket, object, "data")) } -func (d localDirectoryNode) GetMetadata(bucket, object string) (map[string]string, error) { - return d.getMetadata(bucket, object, "metadata.json") +func (node localDirectoryNode) GetMetadata(bucket, object string) (map[string]string, error) { + return node.getMetadata(bucket, object, "metadata.json") } -func (d localDirectoryNode) GetDonutMetadata(bucket, object string) (map[string]string, error) { - return d.getMetadata(bucket, object, "donutMetadata.json") +func (node localDirectoryNode) GetDonutMetadata(bucket, object string) (map[string]string, error) { + return node.getMetadata(bucket, object, "donutMetadata.json") } -func (d localDirectoryNode) getMetadata(bucket, object, fileName string) (map[string]string, error) { - file, err := os.Open(path.Join(d.root, bucket, object, fileName)) +func (node localDirectoryNode) getMetadata(bucket, object, fileName string) (map[string]string, error) { + file, err := os.Open(path.Join(node.root, bucket, object, fileName)) defer file.Close() if err != nil { return nil, err @@ -295,7 +309,26 @@ func (d localDirectoryNode) getMetadata(bucket, object, fileName string) (map[st return nil, err } return metadata, nil +} +func (node localDirectoryNode) ListObjects(bucketName string) ([]string, error) { + prefix := path.Join(node.root, bucketName) + var objects []string + if err := filepath.Walk(prefix, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(path, "data") { + object := strings.TrimPrefix(path, prefix+"/") + object = strings.TrimSuffix(object, "/data") + objects = append(objects, object) + } + return nil + }); err != nil { + return nil, err + } + sort.Strings(objects) + return objects, nil } func newDonutFileWriter(objectDir string) (Writer, error) { diff --git a/pkg/storage/donut/donutdriver_test.go b/pkg/storage/donut/donutdriver_test.go index 8988e147e..9ca7973ac 100644 --- a/pkg/storage/donut/donutdriver_test.go +++ b/pkg/storage/donut/donutdriver_test.go @@ -200,4 +200,9 @@ func (s *MySuite) TestMultipleNewObjects(c *C) { _, err = io.Copy(&readerBuffer2, reader) c.Assert(err, IsNil) c.Assert(readerBuffer2.Bytes(), DeepEquals, []byte("two")) + + // test list objects + listObjects, err := donut.ListObjects("foo") + c.Assert(err, IsNil) + c.Assert(listObjects, DeepEquals, []string{"obj1", "obj2"}) }