Adding content type to backend storage drivers
This commit is contained in:
parent
e5cd3dcb7c
commit
288bc3fcc7
6 changed files with 144 additions and 46 deletions
|
@ -192,7 +192,7 @@ func (server *minioApi) putObjectHandler(w http.ResponseWriter, req *http.Reques
|
|||
vars := mux.Vars(req)
|
||||
bucket := vars["bucket"]
|
||||
object := vars["object"]
|
||||
err := server.storage.StoreObject(bucket, object, req.Body)
|
||||
err := server.storage.StoreObject(bucket, object, "", req.Body)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
w.Write([]byte(err.Error()))
|
||||
|
|
|
@ -57,7 +57,7 @@ func (s *MySuite) TestEmptyObject(c *C) {
|
|||
|
||||
buffer := bytes.NewBufferString("")
|
||||
storage.StoreBucket("bucket")
|
||||
storage.StoreObject("bucket", "object", buffer)
|
||||
storage.StoreObject("bucket", "object", "", buffer)
|
||||
|
||||
response, err := http.Get(testServer.URL + "/bucket/object")
|
||||
c.Assert(err, IsNil)
|
||||
|
@ -82,7 +82,7 @@ func (s *MySuite) TestObject(c *C) {
|
|||
|
||||
buffer := bytes.NewBufferString("hello world")
|
||||
storage.StoreBucket("bucket")
|
||||
storage.StoreObject("bucket", "object", buffer)
|
||||
storage.StoreObject("bucket", "object", "", buffer)
|
||||
|
||||
response, err := http.Get(testServer.URL + "/bucket/object")
|
||||
c.Assert(err, IsNil)
|
||||
|
@ -108,9 +108,9 @@ func (s *MySuite) TestMultipleObjects(c *C) {
|
|||
buffer3 := bytes.NewBufferString("hello three")
|
||||
|
||||
storage.StoreBucket("bucket")
|
||||
storage.StoreObject("bucket", "object1", buffer1)
|
||||
storage.StoreObject("bucket", "object2", buffer2)
|
||||
storage.StoreObject("bucket", "object3", buffer3)
|
||||
storage.StoreObject("bucket", "object1", "", buffer1)
|
||||
storage.StoreObject("bucket", "object2", "", buffer2)
|
||||
storage.StoreObject("bucket", "object3", "", buffer3)
|
||||
|
||||
// test non-existant object
|
||||
response, err := http.Get(testServer.URL + "/bucket/object")
|
||||
|
@ -200,7 +200,7 @@ func (s *MySuite) TestHeader(c *C) {
|
|||
|
||||
buffer := bytes.NewBufferString("hello world")
|
||||
storage.StoreBucket("bucket")
|
||||
storage.StoreObject("bucket", "object", buffer)
|
||||
storage.StoreObject("bucket", "object", "", buffer)
|
||||
|
||||
response, err = http.Get(testServer.URL + "/bucket/object")
|
||||
c.Assert(err, IsNil)
|
||||
|
|
|
@ -1,6 +1,8 @@
|
|||
package fs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
|
@ -16,6 +18,10 @@ type storage struct {
|
|||
writeLock sync.Mutex
|
||||
}
|
||||
|
||||
type SerializedMetadata struct {
|
||||
ContentType string
|
||||
}
|
||||
|
||||
type MkdirFailedError struct{}
|
||||
|
||||
func (self MkdirFailedError) Error() string {
|
||||
|
@ -146,12 +152,41 @@ func (storage *storage) GetObjectMetadata(bucket string, object string) (mstorag
|
|||
return mstorage.ObjectMetadata{}, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
|
||||
}
|
||||
|
||||
_, err = os.Stat(objectPath + "$metadata")
|
||||
if os.IsNotExist(err) {
|
||||
return mstorage.ObjectMetadata{}, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
|
||||
}
|
||||
|
||||
file, err := os.Open(objectPath + "$metadata")
|
||||
defer file.Close()
|
||||
if err != nil {
|
||||
return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
|
||||
}
|
||||
|
||||
metadataBuffer, err := ioutil.ReadAll(file)
|
||||
if err != nil {
|
||||
return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
|
||||
}
|
||||
|
||||
var deserializedMetadata SerializedMetadata
|
||||
err = json.Unmarshal(metadataBuffer, &deserializedMetadata)
|
||||
if err != nil {
|
||||
return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
|
||||
}
|
||||
|
||||
contentType := "application/octet-stream"
|
||||
if deserializedMetadata.ContentType != "" {
|
||||
contentType = deserializedMetadata.ContentType
|
||||
}
|
||||
contentType = strings.TrimSpace(contentType)
|
||||
|
||||
metadata := mstorage.ObjectMetadata{
|
||||
Bucket: bucket,
|
||||
Key: object,
|
||||
Created: stat.ModTime(),
|
||||
Size: stat.Size(),
|
||||
ETag: bucket + "#" + object,
|
||||
Bucket: bucket,
|
||||
Key: object,
|
||||
Created: stat.ModTime(),
|
||||
Size: stat.Size(),
|
||||
ETag: bucket + "#" + object,
|
||||
ContentType: contentType,
|
||||
}
|
||||
|
||||
return metadata, nil
|
||||
|
@ -179,24 +214,26 @@ func (storage *storage) ListObjects(bucket, prefix string, count int) ([]mstorag
|
|||
|
||||
var metadataList []mstorage.ObjectMetadata
|
||||
for _, file := range files {
|
||||
if len(metadataList) >= count {
|
||||
return metadataList, true, nil
|
||||
}
|
||||
if strings.HasPrefix(file.Name(), prefix) {
|
||||
metadata := mstorage.ObjectMetadata{
|
||||
Bucket: bucket,
|
||||
Key: file.Name(),
|
||||
Created: file.ModTime(),
|
||||
Size: file.Size(),
|
||||
ETag: bucket + "#" + file.Name(),
|
||||
if !strings.HasSuffix(file.Name(), "$metadata") {
|
||||
if len(metadataList) >= count {
|
||||
return metadataList, true, nil
|
||||
}
|
||||
if strings.HasPrefix(file.Name(), prefix) {
|
||||
metadata := mstorage.ObjectMetadata{
|
||||
Bucket: bucket,
|
||||
Key: file.Name(),
|
||||
Created: file.ModTime(),
|
||||
Size: file.Size(),
|
||||
ETag: bucket + "#" + file.Name(),
|
||||
}
|
||||
metadataList = append(metadataList, metadata)
|
||||
}
|
||||
metadataList = append(metadataList, metadata)
|
||||
}
|
||||
}
|
||||
return metadataList, false, nil
|
||||
}
|
||||
|
||||
func (storage *storage) StoreObject(bucket string, key string, data io.Reader) error {
|
||||
func (storage *storage) StoreObject(bucket, key, contentType string, data io.Reader) error {
|
||||
// TODO Commits should stage then move instead of writing directly
|
||||
storage.writeLock.Lock()
|
||||
defer storage.writeLock.Unlock()
|
||||
|
@ -216,6 +253,12 @@ func (storage *storage) StoreObject(bucket string, key string, data io.Reader) e
|
|||
return mstorage.ObjectNameInvalid{Bucket: bucket, Object: key}
|
||||
}
|
||||
|
||||
// verify content type
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
contentType = strings.TrimSpace(contentType)
|
||||
|
||||
// get object path
|
||||
objectPath := path.Join(storage.root, bucket, key)
|
||||
objectDir := path.Dir(objectPath)
|
||||
|
@ -246,5 +289,24 @@ func (storage *storage) StoreObject(bucket string, key string, data io.Reader) e
|
|||
return mstorage.EmbedError(bucket, key, err)
|
||||
}
|
||||
|
||||
// serialize metadata to json
|
||||
|
||||
metadataBuffer, err := json.Marshal(SerializedMetadata{ContentType: contentType})
|
||||
if err != nil {
|
||||
return mstorage.EmbedError(bucket, key, err)
|
||||
}
|
||||
|
||||
//
|
||||
file, err = os.OpenFile(objectPath+"$metadata", os.O_WRONLY|os.O_CREATE, 0600)
|
||||
defer file.Close()
|
||||
if err != nil {
|
||||
return mstorage.EmbedError(bucket, key, err)
|
||||
}
|
||||
|
||||
_, err = io.Copy(file, bytes.NewBuffer(metadataBuffer))
|
||||
if err != nil {
|
||||
return mstorage.EmbedError(bucket, key, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -41,7 +41,7 @@ func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object st
|
|||
}
|
||||
}
|
||||
|
||||
func (storage *storage) StoreObject(bucket string, key string, data io.Reader) error {
|
||||
func (storage *storage) StoreObject(bucket, key, contentType string, data io.Reader) error {
|
||||
objectKey := bucket + ":" + key
|
||||
|
||||
if _, ok := storage.bucketdata[bucket]; ok == false {
|
||||
|
@ -51,17 +51,26 @@ func (storage *storage) StoreObject(bucket string, key string, data io.Reader) e
|
|||
if _, ok := storage.objectdata[objectKey]; ok == true {
|
||||
return mstorage.ObjectExists{Bucket: bucket, Key: key}
|
||||
}
|
||||
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
|
||||
contentType = strings.TrimSpace(contentType)
|
||||
|
||||
var bytesBuffer bytes.Buffer
|
||||
var newObject = storedObject{}
|
||||
if _, ok := io.Copy(&bytesBuffer, data); ok == nil {
|
||||
size := bytesBuffer.Len()
|
||||
etag := fmt.Sprintf("%x", sha256.Sum256(bytesBuffer.Bytes()))
|
||||
newObject.metadata = mstorage.ObjectMetadata{
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
Created: time.Now(),
|
||||
Size: int64(size),
|
||||
ETag: etag,
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
|
||||
ContentType: contentType,
|
||||
Created: time.Now(),
|
||||
ETag: etag,
|
||||
Size: int64(size),
|
||||
}
|
||||
newObject.data = bytesBuffer.Bytes()
|
||||
}
|
||||
|
|
|
@ -31,7 +31,7 @@ type Storage interface {
|
|||
CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error)
|
||||
GetObjectMetadata(bucket string, object string) (ObjectMetadata, error)
|
||||
ListObjects(bucket, prefix string, count int) ([]ObjectMetadata, bool, error)
|
||||
StoreObject(bucket string, key string, data io.Reader) error
|
||||
StoreObject(bucket string, key string, contentType string, data io.Reader) error
|
||||
}
|
||||
|
||||
type BucketMetadata struct {
|
||||
|
@ -40,11 +40,13 @@ type BucketMetadata struct {
|
|||
}
|
||||
|
||||
type ObjectMetadata struct {
|
||||
Bucket string
|
||||
Key string
|
||||
Created time.Time
|
||||
Size int64
|
||||
ETag string
|
||||
Bucket string
|
||||
Key string
|
||||
|
||||
ContentType string
|
||||
Created time.Time
|
||||
ETag string
|
||||
Size int64
|
||||
}
|
||||
|
||||
func IsValidBucket(bucket string) bool {
|
||||
|
|
|
@ -21,6 +21,7 @@ func APITestSuite(c *C, create func() Storage) {
|
|||
testListObjectsTestsForNonExistantBucket(c, create)
|
||||
testNonExistantObjectInBucket(c, create)
|
||||
testGetDirectoryReturnsObjectNotFound(c, create)
|
||||
testDefaultContentType(c, create)
|
||||
}
|
||||
|
||||
func testCreateBucket(c *C, create func() Storage) {
|
||||
|
@ -40,7 +41,7 @@ func testMultipleObjectCreation(c *C, create func() Storage) {
|
|||
}
|
||||
key := "obj" + strconv.Itoa(i)
|
||||
objects[key] = []byte(randomString)
|
||||
err := storage.StoreObject("bucket", key, bytes.NewBufferString(randomString))
|
||||
err := storage.StoreObject("bucket", key, "", bytes.NewBufferString(randomString))
|
||||
c.Assert(err, IsNil)
|
||||
}
|
||||
|
||||
|
@ -72,7 +73,7 @@ func testPaging(c *C, create func() Storage) {
|
|||
// check before paging occurs
|
||||
for i := 0; i < 5; i++ {
|
||||
key := "obj" + strconv.Itoa(i)
|
||||
storage.StoreObject("bucket", key, bytes.NewBufferString(key))
|
||||
storage.StoreObject("bucket", key, "", bytes.NewBufferString(key))
|
||||
objects, isTruncated, err = storage.ListObjects("bucket", "", 5)
|
||||
c.Assert(len(objects), Equals, i+1)
|
||||
c.Assert(isTruncated, Equals, false)
|
||||
|
@ -81,7 +82,7 @@ func testPaging(c *C, create func() Storage) {
|
|||
// check after paging occurs pages work
|
||||
for i := 6; i <= 10; i++ {
|
||||
key := "obj" + strconv.Itoa(i)
|
||||
storage.StoreObject("bucket", key, bytes.NewBufferString(key))
|
||||
storage.StoreObject("bucket", key, "", bytes.NewBufferString(key))
|
||||
objects, isTruncated, err = storage.ListObjects("bucket", "", 5)
|
||||
c.Assert(len(objects), Equals, 5)
|
||||
c.Assert(isTruncated, Equals, true)
|
||||
|
@ -89,8 +90,8 @@ func testPaging(c *C, create func() Storage) {
|
|||
}
|
||||
// check paging with prefix at end returns less objects
|
||||
{
|
||||
storage.StoreObject("bucket", "newPrefix", bytes.NewBufferString("prefix1"))
|
||||
storage.StoreObject("bucket", "newPrefix2", bytes.NewBufferString("prefix2"))
|
||||
storage.StoreObject("bucket", "newPrefix", "", bytes.NewBufferString("prefix1"))
|
||||
storage.StoreObject("bucket", "newPrefix2", "", bytes.NewBufferString("prefix2"))
|
||||
objects, isTruncated, err = storage.ListObjects("bucket", "new", 5)
|
||||
c.Assert(len(objects), Equals, 2)
|
||||
}
|
||||
|
@ -124,9 +125,9 @@ func testPaging(c *C, create func() Storage) {
|
|||
func testObjectOverwriteFails(c *C, create func() Storage) {
|
||||
storage := create()
|
||||
storage.StoreBucket("bucket")
|
||||
err := storage.StoreObject("bucket", "object", bytes.NewBufferString("one"))
|
||||
err := storage.StoreObject("bucket", "object", "", bytes.NewBufferString("one"))
|
||||
c.Assert(err, IsNil)
|
||||
err = storage.StoreObject("bucket", "object", bytes.NewBufferString("three"))
|
||||
err = storage.StoreObject("bucket", "object", "", bytes.NewBufferString("three"))
|
||||
c.Assert(err, Not(IsNil))
|
||||
var bytesBuffer bytes.Buffer
|
||||
length, err := storage.CopyObjectToWriter(&bytesBuffer, "bucket", "object")
|
||||
|
@ -137,7 +138,7 @@ func testObjectOverwriteFails(c *C, create func() Storage) {
|
|||
|
||||
func testNonExistantBucketOperations(c *C, create func() Storage) {
|
||||
storage := create()
|
||||
err := storage.StoreObject("bucket", "object", bytes.NewBufferString("one"))
|
||||
err := storage.StoreObject("bucket", "object", "", bytes.NewBufferString("one"))
|
||||
c.Assert(err, Not(IsNil))
|
||||
}
|
||||
|
||||
|
@ -153,7 +154,7 @@ func testPutObjectInSubdir(c *C, create func() Storage) {
|
|||
storage := create()
|
||||
err := storage.StoreBucket("bucket")
|
||||
c.Assert(err, IsNil)
|
||||
err = storage.StoreObject("bucket", "dir1/dir2/object", bytes.NewBufferString("hello world"))
|
||||
err = storage.StoreObject("bucket", "dir1/dir2/object", "", bytes.NewBufferString("hello world"))
|
||||
c.Assert(err, IsNil)
|
||||
var bytesBuffer bytes.Buffer
|
||||
length, err := storage.CopyObjectToWriter(&bytesBuffer, "bucket", "dir1/dir2/object")
|
||||
|
@ -250,7 +251,7 @@ func testGetDirectoryReturnsObjectNotFound(c *C, create func() Storage) {
|
|||
err := storage.StoreBucket("bucket")
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
err = storage.StoreObject("bucket", "dir1/dir2/object", bytes.NewBufferString("hello world"))
|
||||
err = storage.StoreObject("bucket", "dir1/dir2/object", "", bytes.NewBufferString("hello world"))
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
var byteBuffer bytes.Buffer
|
||||
|
@ -287,3 +288,27 @@ func testGetDirectoryReturnsObjectNotFound(c *C, create func() Storage) {
|
|||
}
|
||||
c.Assert(len(byteBuffer2.Bytes()), Equals, 0)
|
||||
}
|
||||
|
||||
func testDefaultContentType(c *C, create func() Storage) {
|
||||
storage := create()
|
||||
err := storage.StoreBucket("bucket")
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
// test empty
|
||||
err = storage.StoreObject("bucket", "one", "", bytes.NewBufferString("one"))
|
||||
metadata, err := storage.GetObjectMetadata("bucket", "one")
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(metadata.ContentType, Equals, "application/octet-stream")
|
||||
|
||||
// test custom
|
||||
storage.StoreObject("bucket", "two", "application/text", bytes.NewBufferString("two"))
|
||||
metadata, err = storage.GetObjectMetadata("bucket", "two")
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(metadata.ContentType, Equals, "application/text")
|
||||
|
||||
// test trim space
|
||||
storage.StoreObject("bucket", "three", "\tapplication/json ", bytes.NewBufferString("three"))
|
||||
metadata, err = storage.GetObjectMetadata("bucket", "three")
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(metadata.ContentType, Equals, "application/json")
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue