diff --git a/api-headers.go b/api-headers.go index c585730dc..235fdb70d 100644 --- a/api-headers.go +++ b/api-headers.go @@ -23,8 +23,6 @@ import ( "net/http" "runtime" "strconv" - - "github.com/minio/minio/pkg/fs" ) //// helpers @@ -60,7 +58,7 @@ func encodeResponse(response interface{}) []byte { } // Write object header -func setObjectHeaders(w http.ResponseWriter, objectInfo fs.ObjectInfo, contentRange *httpRange) { +func setObjectHeaders(w http.ResponseWriter, objectInfo ObjectInfo, contentRange *httpRange) { // set common headers setCommonHeaders(w) diff --git a/api-resources.go b/api-resources.go index 1a1ed4d39..71b27a9bc 100644 --- a/api-resources.go +++ b/api-resources.go @@ -19,8 +19,6 @@ package main import ( "net/url" "strconv" - - "github.com/minio/minio/pkg/fs" ) // parse bucket url queries @@ -34,7 +32,7 @@ func getBucketResources(values url.Values) (prefix, marker, delimiter string, ma } // part bucket url queries for ?uploads -func getBucketMultipartResources(values url.Values) (v fs.BucketMultipartResourcesMetadata) { +func getBucketMultipartResources(values url.Values) (v BucketMultipartResourcesMetadata) { v.Prefix = values.Get("prefix") v.KeyMarker = values.Get("key-marker") v.MaxUploads, _ = strconv.Atoi(values.Get("max-uploads")) @@ -45,10 +43,15 @@ func getBucketMultipartResources(values url.Values) (v fs.BucketMultipartResourc } // parse object url queries -func getObjectResources(values url.Values) (v fs.ObjectResourcesMetadata) { +func getObjectResources(values url.Values) (v ObjectResourcesMetadata) { v.UploadID = values.Get("uploadId") v.PartNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker")) v.MaxParts, _ = strconv.Atoi(values.Get("max-parts")) v.EncodingType = values.Get("encoding-type") return } + +// get upload id. +func getUploadID(values url.Values) (uploadID string) { + return getObjectResources(values).UploadID +} diff --git a/api-response.go b/api-response.go index 9467c8c9d..9fd380747 100644 --- a/api-response.go +++ b/api-response.go @@ -20,8 +20,6 @@ import ( "encoding/xml" "net/http" "time" - - "github.com/minio/minio/pkg/fs" ) const ( @@ -225,7 +223,7 @@ func getLocation(r *http.Request) string { // // output: // populated struct that can be serialized to match xml and json api spec output -func generateListBucketsResponse(buckets []fs.BucketInfo) ListBucketsResponse { +func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse { var listbuckets []Bucket var data = ListBucketsResponse{} var owner = Owner{} @@ -247,7 +245,7 @@ func generateListBucketsResponse(buckets []fs.BucketInfo) ListBucketsResponse { } // generates an ListObjects response for the said bucket with other enumerated options. -func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKeys int, resp fs.ListObjectsResult) ListObjectsResponse { +func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKeys int, resp ListObjectsResult) ListObjectsResponse { var contents []Object var prefixes []CommonPrefix var owner = Owner{} @@ -319,7 +317,7 @@ func generateCompleteMultpartUploadResponse(bucket, key, location, etag string) } // generateListPartsResult -func generateListPartsResponse(objectMetadata fs.ObjectResourcesMetadata) ListPartsResponse { +func generateListPartsResponse(objectMetadata ObjectResourcesMetadata) ListPartsResponse { // TODO - support EncodingType in xml decoding listPartsResponse := ListPartsResponse{} listPartsResponse.Bucket = objectMetadata.Bucket @@ -349,7 +347,7 @@ func generateListPartsResponse(objectMetadata fs.ObjectResourcesMetadata) ListPa } // generateListMultipartUploadsResponse -func generateListMultipartUploadsResponse(bucket string, metadata fs.BucketMultipartResourcesMetadata) ListMultipartUploadsResponse { +func generateListMultipartUploadsResponse(bucket string, metadata BucketMultipartResourcesMetadata) ListMultipartUploadsResponse { listMultipartUploadsResponse := ListMultipartUploadsResponse{} listMultipartUploadsResponse.Bucket = bucket listMultipartUploadsResponse.Delimiter = metadata.Delimiter diff --git a/api-router.go b/api-router.go index 9a2aae15f..50986f687 100644 --- a/api-router.go +++ b/api-router.go @@ -16,19 +16,15 @@ package main -import ( - router "github.com/gorilla/mux" - "github.com/minio/minio/pkg/fs" -) +import router "github.com/gorilla/mux" -// storageAPI container for S3 compatible API. -type storageAPI struct { - // Filesystem instance. - Filesystem fs.Filesystem +// objectStorageAPI container for S3 compatible API. +type objectStorageAPI struct { + ObjectAPI ObjectAPI } // registerAPIRouter - registers S3 compatible APIs. -func registerAPIRouter(mux *router.Router, api storageAPI) { +func registerAPIRouter(mux *router.Router, api objectStorageAPI) { // API Router apiRouter := mux.NewRoute().PathPrefix("/").Subrouter() diff --git a/bucket-handlers.go b/bucket-handlers.go index 61117e7a7..106867a36 100644 --- a/bucket-handlers.go +++ b/bucket-handlers.go @@ -27,7 +27,6 @@ import ( "strings" mux "github.com/gorilla/mux" - "github.com/minio/minio/pkg/fs" "github.com/minio/minio/pkg/probe" ) @@ -38,9 +37,9 @@ func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error if err != nil { errorIf(err.Trace(bucket), "GetBucketPolicy failed.", nil) switch err.ToGoError().(type) { - case fs.BucketNotFound: + case BucketNotFound: return ErrNoSuchBucket - case fs.BucketNameInvalid: + case BucketNameInvalid: return ErrInvalidBucketName default: // For any other error just return AccessDenied. @@ -73,7 +72,7 @@ func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error // GetBucketLocationHandler - GET Bucket location. // ------------------------- // This operation returns bucket location. -func (api storageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] @@ -95,13 +94,13 @@ func (api storageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Re } } - _, err := api.Filesystem.GetBucketInfo(bucket) + _, err := api.ObjectAPI.GetBucketInfo(bucket) if err != nil { errorIf(err.Trace(), "GetBucketInfo failed.", nil) switch err.ToGoError().(type) { - case fs.BucketNotFound: + case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) - case fs.BucketNameInvalid: + case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) @@ -130,7 +129,7 @@ func (api storageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Re // completed or aborted. This operation returns at most 1,000 multipart // uploads in the response. // -func (api storageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] @@ -161,11 +160,11 @@ func (api storageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http resources.MaxUploads = maxObjectList } - resources, err := api.Filesystem.ListMultipartUploads(bucket, resources) + resources, err := api.ObjectAPI.ListMultipartUploads(bucket, resources) if err != nil { errorIf(err.Trace(), "ListMultipartUploads failed.", nil) switch err.ToGoError().(type) { - case fs.BucketNotFound: + case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) @@ -187,7 +186,7 @@ func (api storageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http // of the objects in a bucket. You can use the request parameters as selection // criteria to return a subset of the objects in a bucket. // -func (api storageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] @@ -219,7 +218,7 @@ func (api storageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request) maxkeys = maxObjectList } - listResp, err := api.Filesystem.ListObjects(bucket, prefix, marker, delimiter, maxkeys) + listResp, err := api.ObjectAPI.ListObjects(bucket, prefix, marker, delimiter, maxkeys) if err == nil { // generate response response := generateListObjectsResponse(bucket, prefix, marker, delimiter, maxkeys, listResp) @@ -231,13 +230,13 @@ func (api storageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request) return } switch err.ToGoError().(type) { - case fs.BucketNameInvalid: + case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) - case fs.BucketNotFound: + case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) - case fs.ObjectNotFound: + case ObjectNotFound: writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) - case fs.ObjectNameInvalid: + case ObjectNameInvalid: writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) default: errorIf(err.Trace(), "ListObjects failed.", nil) @@ -249,7 +248,7 @@ func (api storageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request) // ----------- // This implementation of the GET operation returns a list of all buckets // owned by the authenticated sender of the request. -func (api storageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { // List buckets does not support bucket policies. switch getRequestAuthType(r) { default: @@ -263,7 +262,7 @@ func (api storageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request) } } - buckets, err := api.Filesystem.ListBuckets() + buckets, err := api.ObjectAPI.ListBuckets() if err == nil { // generate response response := generateListBucketsResponse(buckets) @@ -279,7 +278,7 @@ func (api storageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request) } // DeleteMultipleObjectsHandler - deletes multiple objects. -func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] @@ -337,7 +336,7 @@ func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *htt var deletedObjects []ObjectIdentifier // Loop through all the objects and delete them sequentially. for _, object := range deleteObjects.Objects { - err := api.Filesystem.DeleteObject(bucket, object.ObjectName) + err := api.ObjectAPI.DeleteObject(bucket, object.ObjectName) if err == nil { deletedObjects = append(deletedObjects, ObjectIdentifier{ ObjectName: object.ObjectName, @@ -345,25 +344,25 @@ func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *htt } else { errorIf(err.Trace(object.ObjectName), "DeleteObject failed.", nil) switch err.ToGoError().(type) { - case fs.BucketNameInvalid: + case BucketNameInvalid: deleteErrors = append(deleteErrors, DeleteError{ Code: errorCodeResponse[ErrInvalidBucketName].Code, Message: errorCodeResponse[ErrInvalidBucketName].Description, Key: object.ObjectName, }) - case fs.BucketNotFound: + case BucketNotFound: deleteErrors = append(deleteErrors, DeleteError{ Code: errorCodeResponse[ErrNoSuchBucket].Code, Message: errorCodeResponse[ErrNoSuchBucket].Description, Key: object.ObjectName, }) - case fs.ObjectNotFound: + case ObjectNotFound: deleteErrors = append(deleteErrors, DeleteError{ Code: errorCodeResponse[ErrNoSuchKey].Code, Message: errorCodeResponse[ErrNoSuchKey].Description, Key: object.ObjectName, }) - case fs.ObjectNameInvalid: + case ObjectNameInvalid: deleteErrors = append(deleteErrors, DeleteError{ Code: errorCodeResponse[ErrNoSuchKey].Code, Message: errorCodeResponse[ErrNoSuchKey].Description, @@ -390,7 +389,7 @@ func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *htt // PutBucketHandler - PUT Bucket // ---------- // This implementation of the PUT operation creates a new bucket for authenticated request -func (api storageAPI) PutBucketHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) PutBucketHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] @@ -408,13 +407,13 @@ func (api storageAPI) PutBucketHandler(w http.ResponseWriter, r *http.Request) { } // Make bucket. - err := api.Filesystem.MakeBucket(bucket) + err := api.ObjectAPI.MakeBucket(bucket) if err != nil { errorIf(err.Trace(), "MakeBucket failed.", nil) switch err.ToGoError().(type) { - case fs.BucketNameInvalid: + case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) - case fs.BucketExists: + case BucketExists: writeErrorResponse(w, r, ErrBucketAlreadyExists, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) @@ -455,7 +454,7 @@ func extractHTTPFormValues(reader *multipart.Reader) (io.Reader, map[string]stri // ---------- // This implementation of the POST operation handles object creation with a specified // signature policy in multipart/form-data -func (api storageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) { // Here the parameter is the size of the form data that should // be loaded in memory, the remaining being put in temporary files. reader, e := r.MultipartReader() @@ -485,19 +484,19 @@ func (api storageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Req writeErrorResponse(w, r, apiErr, r.URL.Path) return } - objectInfo, err := api.Filesystem.CreateObject(bucket, object, -1, fileBody, nil) + objectInfo, err := api.ObjectAPI.PutObject(bucket, object, -1, fileBody, nil) if err != nil { - errorIf(err.Trace(), "CreateObject failed.", nil) + errorIf(err.Trace(), "PutObject failed.", nil) switch err.ToGoError().(type) { - case fs.RootPathFull: + case RootPathFull: writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path) - case fs.BucketNotFound: + case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) - case fs.BucketNameInvalid: + case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) - case fs.BadDigest: + case BadDigest: writeErrorResponse(w, r, ErrBadDigest, r.URL.Path) - case fs.IncompleteBody: + case IncompleteBody: writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) @@ -516,7 +515,7 @@ func (api storageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Req // The operation returns a 200 OK if the bucket exists and you // have permission to access it. Otherwise, the operation might // return responses such as 404 Not Found and 403 Forbidden. -func (api storageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] @@ -532,13 +531,13 @@ func (api storageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Request) } } - _, err := api.Filesystem.GetBucketInfo(bucket) + _, err := api.ObjectAPI.GetBucketInfo(bucket) if err != nil { errorIf(err.Trace(), "GetBucketInfo failed.", nil) switch err.ToGoError().(type) { - case fs.BucketNotFound: + case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) - case fs.BucketNameInvalid: + case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) @@ -549,7 +548,7 @@ func (api storageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Request) } // DeleteBucketHandler - Delete bucket -func (api storageAPI) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] @@ -565,13 +564,13 @@ func (api storageAPI) DeleteBucketHandler(w http.ResponseWriter, r *http.Request } } - err := api.Filesystem.DeleteBucket(bucket) + err := api.ObjectAPI.DeleteBucket(bucket) if err != nil { errorIf(err.Trace(), "DeleteBucket failed.", nil) switch err.ToGoError().(type) { - case fs.BucketNotFound: + case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) - case fs.BucketNotEmpty: + case BucketNotEmpty: writeErrorResponse(w, r, ErrBucketNotEmpty, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) diff --git a/bucket-policy-handlers.go b/bucket-policy-handlers.go index 4ea72dabe..a62c71a81 100644 --- a/bucket-policy-handlers.go +++ b/bucket-policy-handlers.go @@ -25,7 +25,6 @@ import ( "strings" mux "github.com/gorilla/mux" - "github.com/minio/minio/pkg/fs" "github.com/minio/minio/pkg/probe" ) @@ -128,7 +127,7 @@ func bucketPolicyConditionMatch(conditions map[string]string, statement policySt // ----------------- // This implementation of the PUT operation uses the policy // subresource to add to or replace a policy on a bucket -func (api storageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] @@ -188,7 +187,7 @@ func (api storageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Requ if err != nil { errorIf(err.Trace(bucket, string(bucketPolicyBuf)), "SaveBucketPolicy failed.", nil) switch err.ToGoError().(type) { - case fs.BucketNameInvalid: + case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) @@ -202,7 +201,7 @@ func (api storageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Requ // ----------------- // This implementation of the DELETE operation uses the policy // subresource to add to remove a policy on a bucket. -func (api storageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] @@ -223,9 +222,9 @@ func (api storageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.R if err != nil { errorIf(err.Trace(bucket), "DeleteBucketPolicy failed.", nil) switch err.ToGoError().(type) { - case fs.BucketNameInvalid: + case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) - case fs.BucketPolicyNotFound: + case BucketPolicyNotFound: writeErrorResponse(w, r, ErrNoSuchBucketPolicy, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) @@ -239,7 +238,7 @@ func (api storageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.R // ----------------- // This operation uses the policy // subresource to return the policy of a specified bucket. -func (api storageAPI) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] @@ -260,9 +259,9 @@ func (api storageAPI) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Requ if err != nil { errorIf(err.Trace(bucket), "GetBucketPolicy failed.", nil) switch err.ToGoError().(type) { - case fs.BucketNameInvalid: + case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) - case fs.BucketPolicyNotFound: + case BucketPolicyNotFound: writeErrorResponse(w, r, ErrNoSuchBucketPolicy, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) diff --git a/bucket-policy.go b/bucket-policy.go index 3e5b8183e..3791321a8 100644 --- a/bucket-policy.go +++ b/bucket-policy.go @@ -21,7 +21,6 @@ import ( "os" "path/filepath" - "github.com/minio/minio/pkg/fs" "github.com/minio/minio/pkg/probe" ) @@ -70,8 +69,8 @@ func createBucketConfigPath(bucket string) *probe.Error { // readBucketPolicy - read bucket policy. func readBucketPolicy(bucket string) ([]byte, *probe.Error) { // Verify bucket is valid. - if !fs.IsValidBucketName(bucket) { - return nil, probe.NewError(fs.BucketNameInvalid{Bucket: bucket}) + if !IsValidBucketName(bucket) { + return nil, probe.NewError(BucketNameInvalid{Bucket: bucket}) } bucketConfigPath, err := getBucketConfigPath(bucket) @@ -83,7 +82,7 @@ func readBucketPolicy(bucket string) ([]byte, *probe.Error) { bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json") if _, e := os.Stat(bucketPolicyFile); e != nil { if os.IsNotExist(e) { - return nil, probe.NewError(fs.BucketPolicyNotFound{Bucket: bucket}) + return nil, probe.NewError(BucketPolicyNotFound{Bucket: bucket}) } return nil, probe.NewError(e) } @@ -98,8 +97,8 @@ func readBucketPolicy(bucket string) ([]byte, *probe.Error) { // removeBucketPolicy - remove bucket policy. func removeBucketPolicy(bucket string) *probe.Error { // Verify bucket is valid. - if !fs.IsValidBucketName(bucket) { - return probe.NewError(fs.BucketNameInvalid{Bucket: bucket}) + if !IsValidBucketName(bucket) { + return probe.NewError(BucketNameInvalid{Bucket: bucket}) } bucketConfigPath, err := getBucketConfigPath(bucket) @@ -111,7 +110,7 @@ func removeBucketPolicy(bucket string) *probe.Error { bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json") if _, e := os.Stat(bucketPolicyFile); e != nil { if os.IsNotExist(e) { - return probe.NewError(fs.BucketPolicyNotFound{Bucket: bucket}) + return probe.NewError(BucketPolicyNotFound{Bucket: bucket}) } return probe.NewError(e) } @@ -121,8 +120,8 @@ func removeBucketPolicy(bucket string) *probe.Error { // writeBucketPolicy - save bucket policy. func writeBucketPolicy(bucket string, accessPolicyBytes []byte) *probe.Error { // Verify if bucket path legal - if !fs.IsValidBucketName(bucket) { - return probe.NewError(fs.BucketNameInvalid{Bucket: bucket}) + if !IsValidBucketName(bucket) { + return probe.NewError(BucketNameInvalid{Bucket: bucket}) } // Create bucket config path. diff --git a/pkg/fs/fs-backend-metadata.go b/fs-backend-metadata.go similarity index 99% rename from pkg/fs/fs-backend-metadata.go rename to fs-backend-metadata.go index 18e1a0cc4..87747fe6c 100644 --- a/pkg/fs/fs-backend-metadata.go +++ b/fs-backend-metadata.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package fs +package main import ( "github.com/minio/minio/pkg/probe" diff --git a/pkg/fs/fs-bucket-listobjects.go b/fs-bucket-listobjects.go similarity index 98% rename from pkg/fs/fs-bucket-listobjects.go rename to fs-bucket-listobjects.go index a789d87bf..e6f677689 100644 --- a/pkg/fs/fs-bucket-listobjects.go +++ b/fs-bucket-listobjects.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package fs +package main import ( "fmt" @@ -38,7 +38,7 @@ func (fs Filesystem) ListObjects(bucket, prefix, marker, delimiter string, maxKe return result, probe.NewError(BucketNameInvalid{Bucket: bucket}) } - bucket = fs.denormalizeBucket(bucket) + bucket = getActualBucketname(fs.path, bucket) // Get the right bucket name. bucketDir := filepath.Join(fs.path, bucket) // Verify if bucket exists. if status, err := isDirExist(bucketDir); !status { diff --git a/pkg/fs/fs-bucket-listobjects_test.go b/fs-bucket-listobjects_test.go similarity index 94% rename from pkg/fs/fs-bucket-listobjects_test.go rename to fs-bucket-listobjects_test.go index 9d69693f3..49fa60cf4 100644 --- a/pkg/fs/fs-bucket-listobjects_test.go +++ b/fs-bucket-listobjects_test.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package fs +package main import ( "bytes" @@ -27,15 +27,15 @@ import ( ) func TestListObjects(t *testing.T) { - // Make a temporary directory to use as the filesystem. + // Make a temporary directory to use as the fs. directory, e := ioutil.TempDir("", "minio-list-object-test") if e != nil { t.Fatal(e) } defer os.RemoveAll(directory) - // Create the filesystem. - fs, err := New(directory) + // Create the fs. + fs, err := newFS(directory) if err != nil { t.Fatal(err) } @@ -57,36 +57,36 @@ func TestListObjects(t *testing.T) { } defer os.Remove(tmpfile.Name()) // clean up - _, err = fs.CreateObject("test-bucket-list-object", "Asia-maps", int64(len("asia-maps")), bytes.NewBufferString("asia-maps"), nil) + _, err = fs.PutObject("test-bucket-list-object", "Asia-maps", int64(len("asia-maps")), bytes.NewBufferString("asia-maps"), nil) if err != nil { t.Fatal(e) } - _, err = fs.CreateObject("test-bucket-list-object", "Asia/India/India-summer-photos-1", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil) + _, err = fs.PutObject("test-bucket-list-object", "Asia/India/India-summer-photos-1", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil) if err != nil { t.Fatal(e) } - _, err = fs.CreateObject("test-bucket-list-object", "Asia/India/Karnataka/Bangalore/Koramangala/pics", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil) + _, err = fs.PutObject("test-bucket-list-object", "Asia/India/Karnataka/Bangalore/Koramangala/pics", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil) if err != nil { t.Fatal(e) } for i := 0; i < 2; i++ { key := "newPrefix" + strconv.Itoa(i) - _, err = fs.CreateObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil) + _, err = fs.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil) if err != nil { t.Fatal(err) } } - _, err = fs.CreateObject("test-bucket-list-object", "newzen/zen/recurse/again/again/again/pics", int64(len("recurse")), bytes.NewBufferString("recurse"), nil) + _, err = fs.PutObject("test-bucket-list-object", "newzen/zen/recurse/again/again/again/pics", int64(len("recurse")), bytes.NewBufferString("recurse"), nil) if err != nil { t.Fatal(e) } for i := 0; i < 3; i++ { key := "obj" + strconv.Itoa(i) - _, err = fs.CreateObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil) + _, err = fs.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil) if err != nil { t.Fatal(err) } @@ -570,28 +570,28 @@ func TestListObjects(t *testing.T) { } func BenchmarkListObjects(b *testing.B) { - // Make a temporary directory to use as the filesystem. + // Make a temporary directory to use as the fs. directory, e := ioutil.TempDir("", "minio-list-benchmark") if e != nil { b.Fatal(e) } defer os.RemoveAll(directory) - // Create the filesystem. - filesystem, err := New(directory) + // Create the fs. + fs, err := newFS(directory) if err != nil { b.Fatal(err) } // Create a bucket. - err = filesystem.MakeBucket("ls-benchmark-bucket") + err = fs.MakeBucket("ls-benchmark-bucket") if err != nil { b.Fatal(err) } for i := 0; i < 20000; i++ { key := "obj" + strconv.Itoa(i) - _, err = filesystem.CreateObject("ls-benchmark-bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) + _, err = fs.PutObject("ls-benchmark-bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) if err != nil { b.Fatal(err) } @@ -601,7 +601,7 @@ func BenchmarkListObjects(b *testing.B) { // List the buckets over and over and over. for i := 0; i < b.N; i++ { - _, err = filesystem.ListObjects("ls-benchmark-bucket", "", "obj9000", "", -1) + _, err = fs.ListObjects("ls-benchmark-bucket", "", "obj9000", "", -1) if err != nil { b.Fatal(err) } diff --git a/pkg/fs/fs-bucket.go b/fs-bucket.go similarity index 88% rename from pkg/fs/fs-bucket.go rename to fs-bucket.go index 35950676b..199c25433 100644 --- a/pkg/fs/fs-bucket.go +++ b/fs-bucket.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package fs +package main import ( "io/ioutil" @@ -35,7 +35,7 @@ func (fs Filesystem) DeleteBucket(bucket string) *probe.Error { if !IsValidBucketName(bucket) { return probe.NewError(BucketNameInvalid{Bucket: bucket}) } - bucket = fs.denormalizeBucket(bucket) + bucket = getActualBucketname(fs.path, bucket) bucketDir := filepath.Join(fs.path, bucket) if e := os.Remove(bucketDir); e != nil { // Error if there was no bucket in the first place. @@ -129,7 +129,7 @@ func (fs Filesystem) MakeBucket(bucket string) *probe.Error { return probe.NewError(BucketNameInvalid{Bucket: bucket}) } - bucket = fs.denormalizeBucket(bucket) + bucket = getActualBucketname(fs.path, bucket) bucketDir := filepath.Join(fs.path, bucket) if _, e := os.Stat(bucketDir); e == nil { return probe.NewError(BucketExists{Bucket: bucket}) @@ -142,19 +142,23 @@ func (fs Filesystem) MakeBucket(bucket string) *probe.Error { return nil } -// denormalizeBucket - will convert incoming bucket names to -// corresponding valid bucketnames on the backend in a platform +// getActualBucketname - will convert incoming bucket names to +// corresponding actual bucketnames on the backend in a platform // compatible way for all operating systems. -func (fs Filesystem) denormalizeBucket(bucket string) string { - buckets, e := ioutil.ReadDir(fs.path) +func getActualBucketname(fsPath, bucket string) string { + fd, e := os.Open(fsPath) + if e != nil { + return bucket + } + buckets, e := fd.Readdirnames(-1) if e != nil { return bucket } for _, b := range buckets { - // Verify if lowercase version of the bucket is equal to the - // incoming bucket, then use the proper name. - if strings.ToLower(b.Name()) == bucket { - return b.Name() + // Verify if lowercase version of the bucket is equal + // to the incoming bucket, then use the proper name. + if strings.ToLower(b) == bucket { + return b } } return bucket @@ -165,7 +169,7 @@ func (fs Filesystem) GetBucketInfo(bucket string) (BucketInfo, *probe.Error) { if !IsValidBucketName(bucket) { return BucketInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) } - bucket = fs.denormalizeBucket(bucket) + bucket = getActualBucketname(fs.path, bucket) // Get bucket path. bucketDir := filepath.Join(fs.path, bucket) fi, e := os.Stat(bucketDir) diff --git a/pkg/fs/fs-bucket_test.go b/fs-bucket_test.go similarity index 81% rename from pkg/fs/fs-bucket_test.go rename to fs-bucket_test.go index bb7d1781a..15b5c1d41 100644 --- a/pkg/fs/fs-bucket_test.go +++ b/fs-bucket_test.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package fs +package main import ( "io/ioutil" @@ -28,22 +28,22 @@ import ( // But also includes test cases for which the function should fail. // For those cases for which it fails, its also asserted whether the function fails as expected. func TestGetBucketInfo(t *testing.T) { - // Make a temporary directory to use as the filesystem. + // Make a temporary directory to use as the fs. directory, e := ioutil.TempDir("", "minio-metadata-test") if e != nil { t.Fatal(e) } defer os.RemoveAll(directory) - // Create the filesystem. - filesystem, err := New(directory) + // Create the fs. + fs, err := newFS(directory) if err != nil { t.Fatal(err) } // Creating few buckets. for i := 0; i < 4; i++ { - err = filesystem.MakeBucket("meta-test-bucket." + strconv.Itoa(i)) + err = fs.MakeBucket("meta-test-bucket." + strconv.Itoa(i)) if err != nil { t.Fatal(err) } @@ -70,7 +70,7 @@ func TestGetBucketInfo(t *testing.T) { } for i, testCase := range testCases { // The err returned is of type *probe.Error. - bucketInfo, err := filesystem.GetBucketInfo(testCase.bucketName) + bucketInfo, err := fs.GetBucketInfo(testCase.bucketName) if err != nil && testCase.shouldPass { t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Cause.Error()) @@ -96,29 +96,29 @@ func TestGetBucketInfo(t *testing.T) { } func TestListBuckets(t *testing.T) { - // Make a temporary directory to use as the filesystem. + // Make a temporary directory to use as the fs. directory, e := ioutil.TempDir("", "minio-benchmark") if e != nil { t.Fatal(e) } defer os.RemoveAll(directory) - // Create the filesystem. - filesystem, err := New(directory) + // Create the fs. + fs, err := newFS(directory) if err != nil { t.Fatal(err) } // Create a few buckets. for i := 0; i < 10; i++ { - err = filesystem.MakeBucket("testbucket." + strconv.Itoa(i)) + err = fs.MakeBucket("testbucket." + strconv.Itoa(i)) if err != nil { t.Fatal(err) } } // List, and ensure that they are all there. - metadatas, err := filesystem.ListBuckets() + metadatas, err := fs.ListBuckets() if err != nil { t.Fatal(err) } @@ -136,43 +136,43 @@ func TestListBuckets(t *testing.T) { } func TestDeleteBucket(t *testing.T) { - // Make a temporary directory to use as the filesystem. + // Make a temporary directory to use as the fs. directory, e := ioutil.TempDir("", "minio-benchmark") if e != nil { t.Fatal(e) } defer os.RemoveAll(directory) - // Create the filesystem. - filesystem, err := New(directory) + // Create the fs. + fs, err := newFS(directory) if err != nil { t.Fatal(err) } // Deleting a bucket that doesn't exist should error. - err = filesystem.DeleteBucket("bucket") + err = fs.DeleteBucket("bucket") if !strings.Contains(err.Cause.Error(), "Bucket not found:") { t.Fail() } } func BenchmarkListBuckets(b *testing.B) { - // Make a temporary directory to use as the filesystem. + // Make a temporary directory to use as the fs. directory, e := ioutil.TempDir("", "minio-benchmark") if e != nil { b.Fatal(e) } defer os.RemoveAll(directory) - // Create the filesystem. - filesystem, err := New(directory) + // Create the fs. + fs, err := newFS(directory) if err != nil { b.Fatal(err) } // Create a few buckets. for i := 0; i < 20; i++ { - err = filesystem.MakeBucket("bucket." + strconv.Itoa(i)) + err = fs.MakeBucket("bucket." + strconv.Itoa(i)) if err != nil { b.Fatal(err) } @@ -182,7 +182,7 @@ func BenchmarkListBuckets(b *testing.B) { // List the buckets over and over and over. for i := 0; i < b.N; i++ { - _, err = filesystem.ListBuckets() + _, err = fs.ListBuckets() if err != nil { b.Fatal(err) } @@ -190,15 +190,15 @@ func BenchmarkListBuckets(b *testing.B) { } func BenchmarkDeleteBucket(b *testing.B) { - // Make a temporary directory to use as the filesystem. + // Make a temporary directory to use as the fs. directory, e := ioutil.TempDir("", "minio-benchmark") if e != nil { b.Fatal(e) } defer os.RemoveAll(directory) - // Create the filesystem. - filesystem, err := New(directory) + // Create the fs. + fs, err := newFS(directory) if err != nil { b.Fatal(err) } @@ -210,14 +210,14 @@ func BenchmarkDeleteBucket(b *testing.B) { b.StopTimer() // Create and delete the bucket over and over. - err = filesystem.MakeBucket("bucket") + err = fs.MakeBucket("bucket") if err != nil { b.Fatal(err) } b.StartTimer() - err = filesystem.DeleteBucket("bucket") + err = fs.DeleteBucket("bucket") if err != nil { b.Fatal(err) } @@ -225,21 +225,21 @@ func BenchmarkDeleteBucket(b *testing.B) { } func BenchmarkGetBucketInfo(b *testing.B) { - // Make a temporary directory to use as the filesystem. + // Make a temporary directory to use as the fs. directory, e := ioutil.TempDir("", "minio-benchmark") if e != nil { b.Fatal(e) } defer os.RemoveAll(directory) - // Create the filesystem. - filesystem, err := New(directory) + // Create the fs. + fs, err := newFS(directory) if err != nil { b.Fatal(err) } // Put up a bucket with some metadata. - err = filesystem.MakeBucket("bucket") + err = fs.MakeBucket("bucket") if err != nil { b.Fatal(err) } @@ -248,7 +248,7 @@ func BenchmarkGetBucketInfo(b *testing.B) { for i := 0; i < b.N; i++ { // Retrieve the metadata! - _, err := filesystem.GetBucketInfo("bucket") + _, err := fs.GetBucketInfo("bucket") if err != nil { b.Fatal(err) } diff --git a/pkg/fs/fs-datatypes.go b/fs-datatypes.go similarity index 98% rename from pkg/fs/fs-datatypes.go rename to fs-datatypes.go index 13a5fef21..d5824ef06 100644 --- a/pkg/fs/fs-datatypes.go +++ b/fs-datatypes.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package fs +package main import "time" @@ -87,5 +87,5 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part // CompleteMultipartUpload container for completing multipart upload type CompleteMultipartUpload struct { - Part []CompletePart + Parts []CompletePart `xml:"Part"` } diff --git a/pkg/fs/dir.go b/fs-dir.go similarity index 99% rename from pkg/fs/dir.go rename to fs-dir.go index e57228c10..ed5e9765b 100644 --- a/pkg/fs/dir.go +++ b/fs-dir.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package fs +package main import ( "io" diff --git a/pkg/fs/fs-errors.go b/fs-errors.go similarity index 99% rename from pkg/fs/fs-errors.go rename to fs-errors.go index d3c7313a1..03d7ed88e 100644 --- a/pkg/fs/fs-errors.go +++ b/fs-errors.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package fs +package main import "fmt" diff --git a/pkg/fs/fs-multipart.go b/fs-multipart.go similarity index 94% rename from pkg/fs/fs-multipart.go rename to fs-multipart.go index 8ccb2d721..e7489728a 100644 --- a/pkg/fs/fs-multipart.go +++ b/fs-multipart.go @@ -14,13 +14,12 @@ * limitations under the License. */ -package fs +package main import ( "crypto/md5" "encoding/base64" "encoding/hex" - "encoding/xml" "errors" "fmt" "io" @@ -63,7 +62,7 @@ func (fs Filesystem) ListMultipartUploads(bucket string, resources BucketMultipa if !IsValidBucketName(bucket) { return BucketMultipartResourcesMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) } - bucket = fs.denormalizeBucket(bucket) + bucket = getActualBucketname(fs.path, bucket) bucketPath := filepath.Join(fs.path, bucket) if _, e := os.Stat(bucketPath); e != nil { // Check bucket exists. @@ -244,7 +243,7 @@ func (fs Filesystem) NewMultipartUpload(bucket, object string) (string, *probe.E return "", probe.NewError(ObjectNameInvalid{Object: object}) } - bucket = fs.denormalizeBucket(bucket) + bucket = getActualBucketname(fs.path, bucket) bucketPath := filepath.Join(fs.path, bucket) if _, e = os.Stat(bucketPath); e != nil { // Check bucket exists. @@ -318,8 +317,8 @@ func (a partNumber) Len() int { return len(a) } func (a partNumber) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a partNumber) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber } -// CreateObjectPart - create a part in a multipart session -func (fs Filesystem) CreateObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Bytes []byte) (string, *probe.Error) { +// PutObjectPart - create a part in a multipart session +func (fs Filesystem) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, *probe.Error) { di, err := disk.GetInfo(fs.path) if err != nil { return "", probe.NewError(err) @@ -352,7 +351,7 @@ func (fs Filesystem) CreateObjectPart(bucket, object, uploadID string, partID in return "", probe.NewError(InvalidUploadID{UploadID: uploadID}) } - bucket = fs.denormalizeBucket(bucket) + bucket = getActualBucketname(fs.path, bucket) bucketPath := filepath.Join(fs.path, bucket) if _, e := os.Stat(bucketPath); e != nil { // Check bucket exists. @@ -362,12 +361,6 @@ func (fs Filesystem) CreateObjectPart(bucket, object, uploadID string, partID in return "", probe.NewError(e) } - // md5Hex representation. - var md5Hex string - if len(md5Bytes) != 0 { - md5Hex = hex.EncodeToString(md5Bytes) - } - objectPath := filepath.Join(bucketPath, object) partPathPrefix := objectPath + uploadID partPath := partPathPrefix + md5Hex + fmt.Sprintf("$%d-$multiparts", partID) @@ -390,7 +383,7 @@ func (fs Filesystem) CreateObjectPart(bucket, object, uploadID string, partID in // Finalize new md5. newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil)) - if len(md5Bytes) != 0 { + if md5Hex != "" { if newMD5Hex != md5Hex { return "", probe.NewError(BadDigest{md5Hex, newMD5Hex}) } @@ -438,7 +431,7 @@ func (fs Filesystem) CreateObjectPart(bucket, object, uploadID string, partID in } // CompleteMultipartUpload - complete a multipart upload and persist the data -func (fs Filesystem) CompleteMultipartUpload(bucket string, object string, uploadID string, completeMultipartBytes []byte) (ObjectInfo, *probe.Error) { +func (fs Filesystem) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []CompletePart) (ObjectInfo, *probe.Error) { // Check bucket name is valid. if !IsValidBucketName(bucket) { return ObjectInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) @@ -454,7 +447,7 @@ func (fs Filesystem) CompleteMultipartUpload(bucket string, object string, uploa return ObjectInfo{}, probe.NewError(InvalidUploadID{UploadID: uploadID}) } - bucket = fs.denormalizeBucket(bucket) + bucket = getActualBucketname(fs.path, bucket) bucketPath := filepath.Join(fs.path, bucket) if _, e := os.Stat(bucketPath); e != nil { // Check bucket exists. @@ -470,19 +463,6 @@ func (fs Filesystem) CompleteMultipartUpload(bucket string, object string, uploa return ObjectInfo{}, probe.NewError(e) } - completeMultipartUpload := &CompleteMultipartUpload{} - if e = xml.Unmarshal(completeMultipartBytes, completeMultipartUpload); e != nil { - objectWriter.CloseAndPurge() - return ObjectInfo{}, probe.NewError(MalformedXML{}) - } - if !sort.IsSorted(completedParts(completeMultipartUpload.Part)) { - objectWriter.CloseAndPurge() - return ObjectInfo{}, probe.NewError(InvalidPartOrder{}) - } - - // Save parts for verification. - parts := completeMultipartUpload.Part - // Critical region requiring read lock. fs.rwLock.RLock() savedParts := fs.multiparts.ActiveSession[uploadID].Parts @@ -582,7 +562,7 @@ func (fs Filesystem) ListObjectParts(bucket, object string, resources ObjectReso startPartNumber = objectResourcesMetadata.PartNumberMarker } - bucket = fs.denormalizeBucket(bucket) + bucket = getActualBucketname(fs.path, bucket) bucketPath := filepath.Join(fs.path, bucket) if _, e := os.Stat(bucketPath); e != nil { // Check bucket exists. @@ -631,7 +611,7 @@ func (fs Filesystem) AbortMultipartUpload(bucket, object, uploadID string) *prob return probe.NewError(InvalidUploadID{UploadID: uploadID}) } - bucket = fs.denormalizeBucket(bucket) + bucket = getActualBucketname(fs.path, bucket) bucketPath := filepath.Join(fs.path, bucket) if _, e := os.Stat(bucketPath); e != nil { // Check bucket exists. diff --git a/pkg/fs/fs-object.go b/fs-object.go similarity index 85% rename from pkg/fs/fs-object.go rename to fs-object.go index 57ef1599c..18ac424d4 100644 --- a/pkg/fs/fs-object.go +++ b/fs-object.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package fs +package main import ( "bytes" @@ -36,21 +36,17 @@ import ( /// Object Operations // GetObject - GET object -func (fs Filesystem) GetObject(w io.Writer, bucket, object string, start, length int64) (int64, *probe.Error) { - // Critical region requiring read lock. - fs.rwLock.RLock() - defer fs.rwLock.RUnlock() - +func (fs Filesystem) GetObject(bucket, object string, startOffset int64) (io.ReadCloser, *probe.Error) { // Input validation. if !IsValidBucketName(bucket) { - return 0, probe.NewError(BucketNameInvalid{Bucket: bucket}) + return nil, probe.NewError(BucketNameInvalid{Bucket: bucket}) } if !IsValidObjectName(object) { - return 0, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) + return nil, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) } // normalize buckets. - bucket = fs.denormalizeBucket(bucket) + bucket = getActualBucketname(fs.path, bucket) objectPath := filepath.Join(fs.path, bucket, object) file, e := os.Open(objectPath) @@ -60,45 +56,35 @@ func (fs Filesystem) GetObject(w io.Writer, bucket, object string, start, length if os.IsNotExist(e) { _, e = os.Stat(filepath.Join(fs.path, bucket)) if os.IsNotExist(e) { - return 0, probe.NewError(BucketNotFound{Bucket: bucket}) + return nil, probe.NewError(BucketNotFound{Bucket: bucket}) } - return 0, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object}) + return nil, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object}) } - return 0, probe.NewError(e) + return nil, probe.NewError(e) + } + // Initiate a cached stat operation on the file handler. + st, e := file.Stat() + if e != nil { + return nil, probe.NewError(e) + } + // Object path is a directory prefix, return object not found error. + if st.IsDir() { + return nil, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object}) } - defer file.Close() - _, e = file.Seek(start, os.SEEK_SET) + // Seet to a starting offset. + _, e = file.Seek(startOffset, os.SEEK_SET) if e != nil { // When the "handle is invalid", the file might be a directory on Windows. if runtime.GOOS == "windows" && strings.Contains(e.Error(), "handle is invalid") { - return 0, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object}) + return nil, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object}) } - - return 0, probe.NewError(e) + return nil, probe.NewError(e) } - var count int64 - // Copy over the whole file if the length is non-positive. - if length > 0 { - count, e = io.CopyN(w, file, length) - } else { - count, e = io.Copy(w, file) - } - - if e != nil { - // This call will fail if the object is a directory. Stat the file to see if - // this is true, if so, return an ObjectNotFound error. - stat, e := os.Stat(objectPath) - if e == nil && stat.IsDir() { - return count, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object}) - } - - return count, probe.NewError(e) - } - - return count, nil + // Return successfully seeked file handler. + return file, nil } // GetObjectInfo - get object info. @@ -113,7 +99,7 @@ func (fs Filesystem) GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Er } // Normalize buckets. - bucket = fs.denormalizeBucket(bucket) + bucket = getActualBucketname(fs.path, bucket) bucketPath := filepath.Join(fs.path, bucket) if _, e := os.Stat(bucketPath); e != nil { if os.IsNotExist(e) { @@ -196,8 +182,8 @@ func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) bool { return false } -// CreateObject - create an object. -func (fs Filesystem) CreateObject(bucket string, object string, size int64, data io.Reader, md5Bytes []byte) (ObjectInfo, *probe.Error) { +// PutObject - create an object. +func (fs Filesystem) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (ObjectInfo, *probe.Error) { di, e := disk.GetInfo(fs.path) if e != nil { return ObjectInfo{}, probe.NewError(e) @@ -215,7 +201,7 @@ func (fs Filesystem) CreateObject(bucket string, object string, size int64, data return ObjectInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) } - bucket = fs.denormalizeBucket(bucket) + bucket = getActualBucketname(fs.path, bucket) bucketPath := filepath.Join(fs.path, bucket) if _, e = os.Stat(bucketPath); e != nil { if os.IsNotExist(e) { @@ -234,8 +220,8 @@ func (fs Filesystem) CreateObject(bucket string, object string, size int64, data // md5Hex representation. var md5Hex string - if len(md5Bytes) != 0 { - md5Hex = hex.EncodeToString(md5Bytes) + if len(metadata) != 0 { + md5Hex = metadata["md5Sum"] } // Write object. @@ -275,7 +261,7 @@ func (fs Filesystem) CreateObject(bucket string, object string, size int64, data } newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil)) - if len(md5Bytes) != 0 { + if md5Hex != "" { if newMD5Hex != md5Hex { return ObjectInfo{}, probe.NewError(BadDigest{md5Hex, newMD5Hex}) } @@ -346,7 +332,7 @@ func (fs Filesystem) DeleteObject(bucket, object string) *probe.Error { return probe.NewError(BucketNameInvalid{Bucket: bucket}) } - bucket = fs.denormalizeBucket(bucket) + bucket = getActualBucketname(fs.path, bucket) bucketPath := filepath.Join(fs.path, bucket) // Check bucket exists if _, e := os.Stat(bucketPath); e != nil { diff --git a/pkg/fs/fs-object_test.go b/fs-object_test.go similarity index 75% rename from pkg/fs/fs-object_test.go rename to fs-object_test.go index 27c6a44ce..eca9e42ba 100644 --- a/pkg/fs/fs-object_test.go +++ b/fs-object_test.go @@ -14,12 +14,14 @@ * limitations under the License. */ -package fs +package main import ( "bytes" "crypto/md5" + "encoding/hex" "fmt" + "io" "io/ioutil" "os" "path/filepath" @@ -36,8 +38,8 @@ func TestGetObjectInfo(t *testing.T) { } defer os.RemoveAll(directory) - // Create the filesystem. - fs, err := New(directory) + // Create the fs. + fs, err := newFS(directory) if err != nil { t.Fatal(err) } @@ -46,7 +48,7 @@ func TestGetObjectInfo(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = fs.CreateObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil) + _, err = fs.PutObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil) if err != nil { t.Fatal(err) } @@ -56,7 +58,6 @@ func TestGetObjectInfo(t *testing.T) { {Bucket: "test-getobjectinfo", Name: "Asia/asiapics.jpg", ContentType: "image/jpeg", IsDir: false}, } testCases := []struct { - rootPath string bucketName string objectName string @@ -67,24 +68,24 @@ func TestGetObjectInfo(t *testing.T) { shouldPass bool }{ // Test cases with invalid bucket names ( Test number 1-4 ). - {fs.path, ".test", "", ObjectInfo{}, BucketNameInvalid{Bucket: ".test"}, false}, - {fs.path, "Test", "", ObjectInfo{}, BucketNameInvalid{Bucket: "Test"}, false}, - {fs.path, "---", "", ObjectInfo{}, BucketNameInvalid{Bucket: "---"}, false}, - {fs.path, "ad", "", ObjectInfo{}, BucketNameInvalid{Bucket: "ad"}, false}, + {".test", "", ObjectInfo{}, BucketNameInvalid{Bucket: ".test"}, false}, + {"Test", "", ObjectInfo{}, BucketNameInvalid{Bucket: "Test"}, false}, + {"---", "", ObjectInfo{}, BucketNameInvalid{Bucket: "---"}, false}, + {"ad", "", ObjectInfo{}, BucketNameInvalid{Bucket: "ad"}, false}, // Test cases with valid but non-existing bucket names (Test number 5-7). - {fs.path, "abcdefgh", "abc", ObjectInfo{}, BucketNotFound{Bucket: "abcdefgh"}, false}, - {fs.path, "ijklmnop", "efg", ObjectInfo{}, BucketNotFound{Bucket: "ijklmnop"}, false}, + {"abcdefgh", "abc", ObjectInfo{}, BucketNotFound{Bucket: "abcdefgh"}, false}, + {"ijklmnop", "efg", ObjectInfo{}, BucketNotFound{Bucket: "ijklmnop"}, false}, // Test cases with valid but non-existing bucket names and invalid object name (Test number 8-9). - {fs.path, "abcdefgh", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "abcdefgh", Object: ""}, false}, - {fs.path, "ijklmnop", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "ijklmnop", Object: ""}, false}, + {"abcdefgh", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "abcdefgh", Object: ""}, false}, + {"ijklmnop", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "ijklmnop", Object: ""}, false}, // Test cases with non-existing object name with existing bucket (Test number 10-12). - {fs.path, "test-getobjectinfo", "Africa", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Africa"}, false}, - {fs.path, "test-getobjectinfo", "Antartica", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Antartica"}, false}, - {fs.path, "test-getobjectinfo", "Asia/myfile", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia/myfile"}, false}, + {"test-getobjectinfo", "Africa", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Africa"}, false}, + {"test-getobjectinfo", "Antartica", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Antartica"}, false}, + {"test-getobjectinfo", "Asia/myfile", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia/myfile"}, false}, // Test case with existing bucket but object name set to a directory (Test number 13). - {fs.path, "test-getobjectinfo", "Asia", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia"}, false}, + {"test-getobjectinfo", "Asia", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia"}, false}, // Valid case with existing object (Test number 14). - {fs.path, "test-getobjectinfo", "Asia/asiapics.jpg", resultCases[0], nil, true}, + {"test-getobjectinfo", "Asia/asiapics.jpg", resultCases[0], nil, true}, } for i, testCase := range testCases { result, err := fs.GetObjectInfo(testCase.bucketName, testCase.objectName) @@ -127,8 +128,8 @@ func TestGetObjectInfoCore(t *testing.T) { } defer os.RemoveAll(directory) - // Create the filesystem. - fs, err := New(directory) + // Create the fs. + fs, err := newFS(directory) if err != nil { t.Fatal(err) } @@ -137,7 +138,7 @@ func TestGetObjectInfoCore(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = fs.CreateObject("test-getobjinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil) + _, err = fs.PutObject("test-getobjinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil) if err != nil { t.Fatal(err) } @@ -153,7 +154,6 @@ func TestGetObjectInfoCore(t *testing.T) { {Bucket: "test-getobjinfo", Name: "Africa", Size: 0, ContentType: "image/jpeg", IsDir: false}, } testCases := []struct { - rootPath string bucketName string objectName string @@ -165,14 +165,15 @@ func TestGetObjectInfoCore(t *testing.T) { shouldPass bool }{ // Testcase with object name set to a existing directory ( Test number 1). - {fs.path, "test-getobjinfo", "Asia", resultCases[0], nil, true}, + {"test-getobjinfo", "Asia", resultCases[0], nil, true}, // ObjectName set to a existing object ( Test number 2). - {fs.path, "test-getobjinfo", "Asia/asiapics.jpg", resultCases[1], nil, true}, + {"test-getobjinfo", "Asia/asiapics.jpg", resultCases[1], nil, true}, // Object name set to a non-existing object. (Test number 3). - {fs.path, "test-getobjinfo", "Africa", resultCases[2], fmt.Errorf("%s", filepath.FromSlash("test-getobjinfo/Africa")), false}, + {"test-getobjinfo", "Africa", resultCases[2], fmt.Errorf("%s", filepath.FromSlash("test-getobjinfo/Africa")), false}, } + rootPath := fs.(*Filesystem).GetRootPath() for i, testCase := range testCases { - result, err := getObjectInfo(testCase.rootPath, testCase.bucketName, testCase.objectName) + result, err := getObjectInfo(rootPath, testCase.bucketName, testCase.objectName) if err != nil && testCase.shouldPass { t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Cause.Error()) } @@ -205,21 +206,21 @@ func TestGetObjectInfoCore(t *testing.T) { } func BenchmarkGetObject(b *testing.B) { - // Make a temporary directory to use as the filesystem. + // Make a temporary directory to use as the fs. directory, e := ioutil.TempDir("", "minio-benchmark-getobject") if e != nil { b.Fatal(e) } defer os.RemoveAll(directory) - // Create the filesystem. - filesystem, err := New(directory) + // Create the fs. + fs, err := newFS(directory) if err != nil { b.Fatal(err) } // Make a bucket and put in a few objects. - err = filesystem.MakeBucket("bucket") + err = fs.MakeBucket("bucket") if err != nil { b.Fatal(err) } @@ -227,23 +228,30 @@ func BenchmarkGetObject(b *testing.B) { text := "Jack and Jill went up the hill / To fetch a pail of water." hasher := md5.New() hasher.Write([]byte(text)) + metadata := make(map[string]string) for i := 0; i < 10; i++ { - _, err = filesystem.CreateObject("bucket", "object"+strconv.Itoa(i), int64(len(text)), bytes.NewBufferString(text), hasher.Sum(nil)) + metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil)) + _, err = fs.PutObject("bucket", "object"+strconv.Itoa(i), int64(len(text)), bytes.NewBufferString(text), metadata) if err != nil { b.Fatal(err) } } - var w bytes.Buffer b.ResetTimer() for i := 0; i < b.N; i++ { - n, err := filesystem.GetObject(&w, "bucket", "object"+strconv.Itoa(i%10), 0, 0) + var w bytes.Buffer + r, err := fs.GetObject("bucket", "object"+strconv.Itoa(i%10), 0) if err != nil { b.Error(err) } + n, e := io.Copy(&w, r) + if e != nil { + b.Error(e) + } if n != int64(len(text)) { b.Errorf("GetObject returned incorrect length %d (should be %d)\n", n, int64(len(text))) } + r.Close() } } diff --git a/pkg/fs/fs-utils.go b/fs-utils.go similarity index 99% rename from pkg/fs/fs-utils.go rename to fs-utils.go index a759d9810..d1df78dd7 100644 --- a/pkg/fs/fs-utils.go +++ b/fs-utils.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package fs +package main import ( "regexp" diff --git a/pkg/fs/fs-utils_test.go b/fs-utils_test.go similarity index 99% rename from pkg/fs/fs-utils_test.go rename to fs-utils_test.go index 25e12bce8..24f2c2ad0 100644 --- a/pkg/fs/fs-utils_test.go +++ b/fs-utils_test.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package fs +package main import ( "testing" diff --git a/pkg/fs/fs.go b/fs.go similarity index 94% rename from pkg/fs/fs.go rename to fs.go index 48c08f0d9..ebfccdca6 100644 --- a/pkg/fs/fs.go +++ b/fs.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package fs +package main import ( "os" @@ -95,8 +95,8 @@ type Multiparts struct { ActiveSession map[string]*MultipartSession `json:"activeSessions"` } -// New instantiate a new donut -func New(rootPath string) (Filesystem, *probe.Error) { +// newFS instantiate a new filesystem. +func newFS(rootPath string) (ObjectAPI, *probe.Error) { setFSMultipartsMetadataPath(filepath.Join(rootPath, "$multiparts-session.json")) var err *probe.Error @@ -117,7 +117,7 @@ func New(rootPath string) (Filesystem, *probe.Error) { } } - fs := Filesystem{ + fs := &Filesystem{ rwLock: &sync.RWMutex{}, } fs.path = rootPath @@ -125,7 +125,7 @@ func New(rootPath string) (Filesystem, *probe.Error) { /// Defaults - // minium free disk required for i/o operations to succeed. + // Minium free disk required for i/o operations to succeed. fs.minFreeDisk = 5 fs.listObjectMap = make(map[ListObjectParams][]ObjectInfoChannel) diff --git a/pkg/fs/api_suite_test.go b/fs_api_suite_test.go similarity index 72% rename from pkg/fs/api_suite_test.go rename to fs_api_suite_test.go index aa4156c56..f6099f056 100644 --- a/pkg/fs/api_suite_test.go +++ b/fs_api_suite_test.go @@ -14,13 +14,13 @@ * limitations under the License. */ -package fs +package main import ( "bytes" "crypto/md5" "encoding/hex" - "encoding/xml" + "io" "math/rand" "strconv" @@ -28,7 +28,7 @@ import ( ) // APITestSuite - collection of API tests -func APITestSuite(c *check.C, create func() Filesystem) { +func APITestSuite(c *check.C, create func() ObjectAPI) { testMakeBucket(c, create) testMultipleObjectCreation(c, create) testPaging(c, create) @@ -46,13 +46,13 @@ func APITestSuite(c *check.C, create func() Filesystem) { testMultipartObjectAbort(c, create) } -func testMakeBucket(c *check.C, create func() Filesystem) { +func testMakeBucket(c *check.C, create func() ObjectAPI) { fs := create() err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) } -func testMultipartObjectCreation(c *check.C, create func() Filesystem) { +func testMultipartObjectCreation(c *check.C, create func() ObjectAPI) { fs := create() err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) @@ -73,19 +73,17 @@ func testMultipartObjectCreation(c *check.C, create func() Filesystem) { expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil)) var calculatedMD5sum string - calculatedMD5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), hasher.Sum(nil)) + calculatedMD5sum, err = fs.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex) c.Assert(err, check.IsNil) c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex) - completedParts.Part = append(completedParts.Part, CompletePart{PartNumber: i, ETag: calculatedMD5sum}) + completedParts.Parts = append(completedParts.Parts, CompletePart{PartNumber: i, ETag: calculatedMD5sum}) } - completedPartsBytes, e := xml.Marshal(completedParts) - c.Assert(e, check.IsNil) - objectInfo, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, completedPartsBytes) + objectInfo, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, completedParts.Parts) c.Assert(err, check.IsNil) c.Assert(objectInfo.MD5Sum, check.Equals, "9b7d6f13ba00e24d0b02de92e814891b-10") } -func testMultipartObjectAbort(c *check.C, create func() Filesystem) { +func testMultipartObjectAbort(c *check.C, create func() ObjectAPI) { fs := create() err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) @@ -105,7 +103,7 @@ func testMultipartObjectAbort(c *check.C, create func() Filesystem) { expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil)) var calculatedMD5sum string - calculatedMD5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), hasher.Sum(nil)) + calculatedMD5sum, err = fs.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex) c.Assert(err, check.IsNil) c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex) parts[i] = expectedMD5Sumhex @@ -114,7 +112,7 @@ func testMultipartObjectAbort(c *check.C, create func() Filesystem) { c.Assert(err, check.IsNil) } -func testMultipleObjectCreation(c *check.C, create func() Filesystem) { +func testMultipleObjectCreation(c *check.C, create func() ObjectAPI) { objects := make(map[string][]byte) fs := create() err := fs.MakeBucket("bucket") @@ -133,24 +131,28 @@ func testMultipleObjectCreation(c *check.C, create func() Filesystem) { key := "obj" + strconv.Itoa(i) objects[key] = []byte(randomString) var objectInfo ObjectInfo - objectInfo, err = fs.CreateObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), hasher.Sum(nil)) + metadata := make(map[string]string) + metadata["md5Sum"] = expectedMD5Sumhex + objectInfo, err = fs.PutObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), metadata) c.Assert(err, check.IsNil) c.Assert(objectInfo.MD5Sum, check.Equals, expectedMD5Sumhex) } for key, value := range objects { var byteBuffer bytes.Buffer - _, err := fs.GetObject(&byteBuffer, "bucket", key, 0, 0) + r, err := fs.GetObject("bucket", key, 0) c.Assert(err, check.IsNil) + io.Copy(&byteBuffer, r) c.Assert(byteBuffer.Bytes(), check.DeepEquals, value) metadata, err := fs.GetObjectInfo("bucket", key) c.Assert(err, check.IsNil) c.Assert(metadata.Size, check.Equals, int64(len(value))) + r.Close() } } -func testPaging(c *check.C, create func() Filesystem) { +func testPaging(c *check.C, create func() ObjectAPI) { fs := create() fs.MakeBucket("bucket") result, err := fs.ListObjects("bucket", "", "", "", 0) @@ -160,7 +162,7 @@ func testPaging(c *check.C, create func() Filesystem) { // check before paging occurs for i := 0; i < 5; i++ { key := "obj" + strconv.Itoa(i) - _, err = fs.CreateObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) + _, err = fs.PutObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) c.Assert(err, check.IsNil) result, err = fs.ListObjects("bucket", "", "", "", 5) c.Assert(err, check.IsNil) @@ -170,7 +172,7 @@ func testPaging(c *check.C, create func() Filesystem) { // check after paging occurs pages work for i := 6; i <= 10; i++ { key := "obj" + strconv.Itoa(i) - _, err = fs.CreateObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) + _, err = fs.PutObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) c.Assert(err, check.IsNil) result, err = fs.ListObjects("bucket", "obj", "", "", 5) c.Assert(err, check.IsNil) @@ -179,9 +181,9 @@ func testPaging(c *check.C, create func() Filesystem) { } // check paging with prefix at end returns less objects { - _, err = fs.CreateObject("bucket", "newPrefix", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil) + _, err = fs.PutObject("bucket", "newPrefix", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil) c.Assert(err, check.IsNil) - _, err = fs.CreateObject("bucket", "newPrefix2", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) + _, err = fs.PutObject("bucket", "newPrefix2", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) c.Assert(err, check.IsNil) result, err = fs.ListObjects("bucket", "new", "", "", 5) c.Assert(err, check.IsNil) @@ -201,9 +203,9 @@ func testPaging(c *check.C, create func() Filesystem) { // check delimited results with delimiter and prefix { - _, err = fs.CreateObject("bucket", "this/is/delimited", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil) + _, err = fs.PutObject("bucket", "this/is/delimited", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil) c.Assert(err, check.IsNil) - _, err = fs.CreateObject("bucket", "this/is/also/a/delimited/file", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) + _, err = fs.PutObject("bucket", "this/is/also/a/delimited/file", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) c.Assert(err, check.IsNil) result, err = fs.ListObjects("bucket", "this/is/", "", "/", 10) c.Assert(err, check.IsNil) @@ -250,32 +252,33 @@ func testPaging(c *check.C, create func() Filesystem) { } } -func testObjectOverwriteWorks(c *check.C, create func() Filesystem) { +func testObjectOverwriteWorks(c *check.C, create func() ObjectAPI) { fs := create() err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) - _, err = fs.CreateObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil) + _, err = fs.PutObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil) c.Assert(err, check.IsNil) // c.Assert(md5Sum1hex, check.Equals, objectInfo.MD5Sum) - _, err = fs.CreateObject("bucket", "object", int64(len("three")), bytes.NewBufferString("three"), nil) + _, err = fs.PutObject("bucket", "object", int64(len("three")), bytes.NewBufferString("three"), nil) c.Assert(err, check.IsNil) var bytesBuffer bytes.Buffer - length, err := fs.GetObject(&bytesBuffer, "bucket", "object", 0, 0) + r, err := fs.GetObject("bucket", "object", 0) c.Assert(err, check.IsNil) - c.Assert(length, check.Equals, int64(len("three"))) + io.Copy(&bytesBuffer, r) c.Assert(string(bytesBuffer.Bytes()), check.Equals, "three") + r.Close() } -func testNonExistantBucketOperations(c *check.C, create func() Filesystem) { +func testNonExistantBucketOperations(c *check.C, create func() ObjectAPI) { fs := create() - _, err := fs.CreateObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil) + _, err := fs.PutObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil) c.Assert(err, check.Not(check.IsNil)) } -func testBucketRecreateFails(c *check.C, create func() Filesystem) { +func testBucketRecreateFails(c *check.C, create func() ObjectAPI) { fs := create() err := fs.MakeBucket("string") c.Assert(err, check.IsNil) @@ -283,22 +286,23 @@ func testBucketRecreateFails(c *check.C, create func() Filesystem) { c.Assert(err, check.Not(check.IsNil)) } -func testPutObjectInSubdir(c *check.C, create func() Filesystem) { +func testPutObjectInSubdir(c *check.C, create func() ObjectAPI) { fs := create() err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) - _, err = fs.CreateObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil) + _, err = fs.PutObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil) c.Assert(err, check.IsNil) var bytesBuffer bytes.Buffer - length, err := fs.GetObject(&bytesBuffer, "bucket", "dir1/dir2/object", 0, 0) + r, err := fs.GetObject("bucket", "dir1/dir2/object", 0) c.Assert(err, check.IsNil) + io.Copy(&bytesBuffer, r) c.Assert(len(bytesBuffer.Bytes()), check.Equals, len("hello world")) - c.Assert(int64(len(bytesBuffer.Bytes())), check.Equals, length) + r.Close() } -func testListBuckets(c *check.C, create func() Filesystem) { +func testListBuckets(c *check.C, create func() ObjectAPI) { fs := create() // test empty list @@ -330,7 +334,7 @@ func testListBuckets(c *check.C, create func() Filesystem) { c.Assert(err, check.IsNil) } -func testListBucketsOrder(c *check.C, create func() Filesystem) { +func testListBucketsOrder(c *check.C, create func() ObjectAPI) { // if implementation contains a map, order of map keys will vary. // this ensures they return in the same order each time for i := 0; i < 10; i++ { @@ -348,7 +352,7 @@ func testListBucketsOrder(c *check.C, create func() Filesystem) { } } -func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Filesystem) { +func testListObjectsTestsForNonExistantBucket(c *check.C, create func() ObjectAPI) { fs := create() result, err := fs.ListObjects("bucket", "", "", "", 1000) c.Assert(err, check.Not(check.IsNil)) @@ -356,16 +360,13 @@ func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Filesyst c.Assert(len(result.Objects), check.Equals, 0) } -func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) { +func testNonExistantObjectInBucket(c *check.C, create func() ObjectAPI) { fs := create() err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) - var byteBuffer bytes.Buffer - length, err := fs.GetObject(&byteBuffer, "bucket", "dir1", 0, 0) - c.Assert(length, check.Equals, int64(0)) + _, err = fs.GetObject("bucket", "dir1", 0) c.Assert(err, check.Not(check.IsNil)) - c.Assert(len(byteBuffer.Bytes()), check.Equals, 0) switch err := err.ToGoError().(type) { case ObjectNotFound: c.Assert(err, check.ErrorMatches, "Object not found: bucket#dir1") @@ -374,17 +375,15 @@ func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) { } } -func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem) { +func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() ObjectAPI) { fs := create() err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) - _, err = fs.CreateObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil) + _, err = fs.PutObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil) c.Assert(err, check.IsNil) - var byteBuffer bytes.Buffer - length, err := fs.GetObject(&byteBuffer, "bucket", "dir1", 0, 0) - c.Assert(length, check.Equals, int64(0)) + _, err = fs.GetObject("bucket", "dir1", 0) switch err := err.ToGoError().(type) { case ObjectNotFound: c.Assert(err.Bucket, check.Equals, "bucket") @@ -393,11 +392,8 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem) // force a failure with a line number c.Assert(err, check.Equals, "ObjectNotFound") } - c.Assert(len(byteBuffer.Bytes()), check.Equals, 0) - var byteBuffer2 bytes.Buffer - length, err = fs.GetObject(&byteBuffer, "bucket", "dir1/", 0, 0) - c.Assert(length, check.Equals, int64(0)) + _, err = fs.GetObject("bucket", "dir1/", 0) switch err := err.ToGoError().(type) { case ObjectNotFound: c.Assert(err.Bucket, check.Equals, "bucket") @@ -406,17 +402,16 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem) // force a failure with a line number c.Assert(err, check.Equals, "ObjectNotFound") } - c.Assert(len(byteBuffer2.Bytes()), check.Equals, 0) } -func testDefaultContentType(c *check.C, create func() Filesystem) { +func testDefaultContentType(c *check.C, create func() ObjectAPI) { fs := create() err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) // Test empty - _, err = fs.CreateObject("bucket", "one", int64(len("one")), bytes.NewBufferString("one"), nil) - metadata, err := fs.GetObjectInfo("bucket", "one") + _, err = fs.PutObject("bucket", "one", int64(len("one")), bytes.NewBufferString("one"), nil) + objInfo, err := fs.GetObjectInfo("bucket", "one") c.Assert(err, check.IsNil) - c.Assert(metadata.ContentType, check.Equals, "application/octet-stream") + c.Assert(objInfo.ContentType, check.Equals, "application/octet-stream") } diff --git a/pkg/fs/fs_test.go b/fs_test.go similarity index 79% rename from pkg/fs/fs_test.go rename to fs_test.go index 1451c73f7..811ceeb7a 100644 --- a/pkg/fs/fs_test.go +++ b/fs_test.go @@ -14,29 +14,22 @@ * limitations under the License. */ -package fs +package main import ( "io/ioutil" "os" - "testing" . "gopkg.in/check.v1" ) -func Test(t *testing.T) { TestingT(t) } - -type MySuite struct{} - -var _ = Suite(&MySuite{}) - -func (s *MySuite) TestAPISuite(c *C) { +func (s *MyAPISuite) TestAPISuite(c *C) { var storageList []string - create := func() Filesystem { + create := func() ObjectAPI { path, e := ioutil.TempDir(os.TempDir(), "minio-") c.Check(e, IsNil) storageList = append(storageList, path) - store, err := New(path) + store, err := newFS(path) c.Check(err, IsNil) return store } @@ -46,7 +39,6 @@ func (s *MySuite) TestAPISuite(c *C) { func removeRoots(c *C, roots []string) { for _, root := range roots { - err := os.RemoveAll(root) - c.Check(err, IsNil) + os.RemoveAll(root) } } diff --git a/httprange.go b/httprange.go index c08e11136..09389db45 100644 --- a/httprange.go +++ b/httprange.go @@ -22,7 +22,6 @@ import ( "strconv" "strings" - "github.com/minio/minio/pkg/fs" "github.com/minio/minio/pkg/probe" ) @@ -60,7 +59,7 @@ func getRequestedRange(hrange string, size int64) (*httpRange, *probe.Error) { func (r *httpRange) parse(ra string) *probe.Error { i := strings.Index(ra, "-") if i < 0 { - return probe.NewError(fs.InvalidRange{}) + return probe.NewError(InvalidRange{}) } start, end := strings.TrimSpace(ra[:i]), strings.TrimSpace(ra[i+1:]) if start == "" { @@ -68,7 +67,7 @@ func (r *httpRange) parse(ra string) *probe.Error { // range start relative to the end of the file. i, err := strconv.ParseInt(end, 10, 64) if err != nil { - return probe.NewError(fs.InvalidRange{}) + return probe.NewError(InvalidRange{}) } if i > r.size { i = r.size @@ -78,7 +77,7 @@ func (r *httpRange) parse(ra string) *probe.Error { } else { i, err := strconv.ParseInt(start, 10, 64) if err != nil || i > r.size || i < 0 { - return probe.NewError(fs.InvalidRange{}) + return probe.NewError(InvalidRange{}) } r.start = i if end == "" { @@ -87,7 +86,7 @@ func (r *httpRange) parse(ra string) *probe.Error { } else { i, err := strconv.ParseInt(end, 10, 64) if err != nil || r.start > i { - return probe.NewError(fs.InvalidRange{}) + return probe.NewError(InvalidRange{}) } if i >= r.size { i = r.size - 1 @@ -104,7 +103,7 @@ func (r *httpRange) parseRange(s string) *probe.Error { return probe.NewError(errors.New("header not present")) } if !strings.HasPrefix(s, b) { - return probe.NewError(fs.InvalidRange{}) + return probe.NewError(InvalidRange{}) } ras := strings.Split(s[len(b):], ",") @@ -118,7 +117,7 @@ func (r *httpRange) parseRange(s string) *probe.Error { ra := strings.TrimSpace(ras[0]) if ra == "" { - return probe.NewError(fs.InvalidRange{}) + return probe.NewError(InvalidRange{}) } return r.parse(ra) } diff --git a/minio-main.go b/minio-main.go index f69e2e492..908c85cb8 100644 --- a/minio-main.go +++ b/minio-main.go @@ -30,7 +30,6 @@ import ( "github.com/minio/cli" "github.com/minio/mc/pkg/console" - "github.com/minio/minio/pkg/fs" "github.com/minio/minio/pkg/minhttp" "github.com/minio/minio/pkg/probe" ) @@ -92,11 +91,11 @@ EXAMPLES: } // configureServer configure a new server instance -func configureServer(filesystem fs.Filesystem) *http.Server { +func configureServer(objectAPI ObjectAPI) *http.Server { // Minio server config apiServer := &http.Server{ Addr: serverConfig.GetAddr(), - Handler: configureServerHandler(filesystem), + Handler: configureServerHandler(objectAPI), MaxHeaderBytes: 1 << 20, } @@ -306,49 +305,52 @@ func serverMain(c *cli.Context) { cli.ShowCommandHelpAndExit(c, "server", 1) } + var objectAPI ObjectAPI + var err *probe.Error + // get backend. backend := serverConfig.GetBackend() if backend.Type == "fs" { - // Initialize file system. - filesystem, err := fs.New(backend.Disk) + // Initialize filesystem storage layer. + objectAPI, err = newFS(backend.Disk) fatalIf(err.Trace(backend.Type, backend.Disk), "Initializing filesystem failed.", nil) - - // Configure server. - apiServer := configureServer(filesystem) - - // Credential. - cred := serverConfig.GetCredential() - - // Region. - region := serverConfig.GetRegion() - - // Print credentials and region. - console.Println("\n" + cred.String() + " " + colorMagenta("Region: ") + colorWhite(region)) - - console.Println("\nMinio Object Storage:") - // Print api listen ips. - printListenIPs(apiServer) - - console.Println("\nMinio Browser:") - // Print browser listen ips. - printListenIPs(apiServer) - - console.Println("\nTo configure Minio Client:") - - // Download 'mc' links. - if runtime.GOOS == "windows" { - console.Println(" Download 'mc' from https://dl.minio.io/client/mc/release/" + runtime.GOOS + "-" + runtime.GOARCH + "/mc.exe") - console.Println(" $ mc.exe config host add myminio http://localhost:9000 " + cred.AccessKeyID + " " + cred.SecretAccessKey) - } else { - console.Println(" $ wget https://dl.minio.io/client/mc/release/" + runtime.GOOS + "-" + runtime.GOARCH + "/mc") - console.Println(" $ chmod 755 mc") - console.Println(" $ ./mc config host add myminio http://localhost:9000 " + cred.AccessKeyID + " " + cred.SecretAccessKey) - } - - // Start server. - err = minhttp.ListenAndServe(apiServer) - errorIf(err.Trace(), "Failed to start the minio server.", nil) - return + } else { // else if backend.Type == "xl" { here. + console.Fatalln("No known backends configured, please use ‘minio init --help’ to initialize a backend.") } - console.Println(colorGreen("No known backends configured, please use ‘minio init --help’ to initialize a backend.")) + + // Configure server. + apiServer := configureServer(objectAPI) + + // Credential. + cred := serverConfig.GetCredential() + + // Region. + region := serverConfig.GetRegion() + + // Print credentials and region. + console.Println("\n" + cred.String() + " " + colorMagenta("Region: ") + colorWhite(region)) + + console.Println("\nMinio Object Storage:") + // Print api listen ips. + printListenIPs(apiServer) + + console.Println("\nMinio Browser:") + // Print browser listen ips. + printListenIPs(apiServer) + + console.Println("\nTo configure Minio Client:") + + // Download 'mc' links. + if runtime.GOOS == "windows" { + console.Println(" Download 'mc' from https://dl.minio.io/client/mc/release/" + runtime.GOOS + "-" + runtime.GOARCH + "/mc.exe") + console.Println(" $ mc.exe config host add myminio http://localhost:9000 " + cred.AccessKeyID + " " + cred.SecretAccessKey) + } else { + console.Println(" $ wget https://dl.minio.io/client/mc/release/" + runtime.GOOS + "-" + runtime.GOARCH + "/mc") + console.Println(" $ chmod 755 mc") + console.Println(" $ ./mc config host add myminio http://localhost:9000 " + cred.AccessKeyID + " " + cred.SecretAccessKey) + } + + // Start server. + err = minhttp.ListenAndServe(apiServer) + errorIf(err.Trace(), "Failed to start the minio server.", nil) } diff --git a/object-api-interface.go b/object-api-interface.go new file mode 100644 index 000000000..fa04a3a41 --- /dev/null +++ b/object-api-interface.go @@ -0,0 +1,33 @@ +package main + +import ( + "io" + + "github.com/minio/minio/pkg/probe" +) + +// ObjectAPI interface. +type ObjectAPI interface { + // Bucket resource API. + DeleteBucket(bucket string) *probe.Error + ListBuckets() ([]BucketInfo, *probe.Error) + MakeBucket(bucket string) *probe.Error + GetBucketInfo(bucket string) (BucketInfo, *probe.Error) + + // Bucket query API. + ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsResult, *probe.Error) + ListMultipartUploads(bucket string, resources BucketMultipartResourcesMetadata) (BucketMultipartResourcesMetadata, *probe.Error) + + // Object resource API. + GetObject(bucket, object string, startOffset int64) (io.ReadCloser, *probe.Error) + GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Error) + PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (ObjectInfo, *probe.Error) + DeleteObject(bucket, object string) *probe.Error + + // Object query API. + NewMultipartUpload(bucket, object string) (string, *probe.Error) + PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, *probe.Error) + ListObjectParts(bucket, object string, resources ObjectResourcesMetadata) (ObjectResourcesMetadata, *probe.Error) + CompleteMultipartUpload(bucket string, object string, uploadID string, parts []CompletePart) (ObjectInfo, *probe.Error) + AbortMultipartUpload(bucket, object, uploadID string) *probe.Error +} diff --git a/object-handlers.go b/object-handlers.go index cc2199c86..51e2075c7 100644 --- a/object-handlers.go +++ b/object-handlers.go @@ -9,7 +9,7 @@ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implieapi.Filesystem. + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implieapi.ObjectAPI. * See the License for the specific language governing permissions and * limitations under the License. */ @@ -19,17 +19,18 @@ package main import ( "crypto/sha256" "encoding/hex" + "encoding/xml" "fmt" "io" "io/ioutil" "net/http" "net/url" + "sort" "strconv" "strings" "time" mux "github.com/gorilla/mux" - "github.com/minio/minio/pkg/fs" "github.com/minio/minio/pkg/probe" ) @@ -59,7 +60,7 @@ func setGetRespHeaders(w http.ResponseWriter, reqParams url.Values) { // ---------- // This implementation of the GET operation retrieves object. To use GET, // you must have READ access to the object. -func (api storageAPI) GetObjectHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) GetObjectHandler(w http.ResponseWriter, r *http.Request) { var object, bucket string vars := mux.Vars(r) bucket = vars["bucket"] @@ -83,17 +84,17 @@ func (api storageAPI) GetObjectHandler(w http.ResponseWriter, r *http.Request) { } } - objectInfo, err := api.Filesystem.GetObjectInfo(bucket, object) + objectInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object) if err != nil { errorIf(err.Trace(), "GetObject failed.", nil) switch err.ToGoError().(type) { - case fs.BucketNameInvalid: + case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) - case fs.BucketNotFound: + case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) - case fs.ObjectNotFound: + case ObjectNotFound: writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) - case fs.ObjectNameInvalid: + case ObjectNameInvalid: writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) @@ -125,10 +126,27 @@ func (api storageAPI) GetObjectHandler(w http.ResponseWriter, r *http.Request) { setGetRespHeaders(w, r.URL.Query()) // Get the object. - if _, err = api.Filesystem.GetObject(w, bucket, object, hrange.start, hrange.length); err != nil { + startOffset := hrange.start + readCloser, err := api.ObjectAPI.GetObject(bucket, object, startOffset) + if err != nil { errorIf(err.Trace(), "GetObject failed.", nil) + writeErrorResponse(w, r, ErrInternalError, r.URL.Path) return } + defer readCloser.Close() // Close after this handler returns. + if hrange.length > 0 { + if _, e := io.CopyN(w, readCloser, hrange.length); e != nil { + errorIf(probe.NewError(e), "Writing to client failed", nil) + // Do not send error response here, since client could have died. + return + } + } else { + if _, e := io.Copy(w, readCloser); e != nil { + errorIf(probe.NewError(e), "Writing to client failed", nil) + // Do not send error response here, since client could have died. + return + } + } } var unixEpochTime = time.Unix(0, 0) @@ -228,7 +246,7 @@ func checkETag(w http.ResponseWriter, r *http.Request) bool { // HeadObjectHandler - HEAD Object // ----------- // The HEAD operation retrieves metadata from an object without returning the object itself. -func (api storageAPI) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { var object, bucket string vars := mux.Vars(r) bucket = vars["bucket"] @@ -246,17 +264,17 @@ func (api storageAPI) HeadObjectHandler(w http.ResponseWriter, r *http.Request) } } - objectInfo, err := api.Filesystem.GetObjectInfo(bucket, object) + objectInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object) if err != nil { errorIf(err.Trace(bucket, object), "GetObjectInfo failed.", nil) switch err.ToGoError().(type) { - case fs.BucketNameInvalid: + case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) - case fs.BucketNotFound: + case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) - case fs.ObjectNotFound: + case ObjectNotFound: writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) - case fs.ObjectNameInvalid: + case ObjectNameInvalid: writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) @@ -286,7 +304,7 @@ func (api storageAPI) HeadObjectHandler(w http.ResponseWriter, r *http.Request) // ---------- // This implementation of the PUT operation adds an object to a bucket // while reading the object from another source. -func (api storageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] @@ -339,17 +357,17 @@ func (api storageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Request) return } - objectInfo, err := api.Filesystem.GetObjectInfo(sourceBucket, sourceObject) + objectInfo, err := api.ObjectAPI.GetObjectInfo(sourceBucket, sourceObject) if err != nil { errorIf(err.Trace(), "GetObjectInfo failed.", nil) switch err.ToGoError().(type) { - case fs.BucketNameInvalid: + case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, objectSource) - case fs.BucketNotFound: + case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, objectSource) - case fs.ObjectNotFound: + case ObjectNotFound: writeErrorResponse(w, r, ErrNoSuchKey, objectSource) - case fs.ObjectNameInvalid: + case ObjectNameInvalid: writeErrorResponse(w, r, ErrNoSuchKey, objectSource) default: writeErrorResponse(w, r, ErrInternalError, objectSource) @@ -388,37 +406,45 @@ func (api storageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Request) } } - // Initialize a pipe for data pipe line. - reader, writer := io.Pipe() - - // Start writing in a routine. - go func() { - defer writer.Close() - if _, getErr := api.Filesystem.GetObject(writer, sourceBucket, sourceObject, 0, 0); getErr != nil { - writer.CloseWithError(probe.WrapError(getErr)) - return + startOffset := int64(0) // Read the whole file. + // Get the object. + readCloser, getErr := api.ObjectAPI.GetObject(sourceBucket, sourceObject, startOffset) + if getErr != nil { + errorIf(getErr.Trace(sourceBucket, sourceObject), "Reading "+objectSource+" failed.", nil) + switch err.ToGoError().(type) { + case BucketNotFound: + writeErrorResponse(w, r, ErrNoSuchBucket, objectSource) + case ObjectNotFound: + writeErrorResponse(w, r, ErrNoSuchKey, objectSource) + default: + writeErrorResponse(w, r, ErrInternalError, objectSource) } - }() + return + } // Size of object. size := objectInfo.Size + // Save metadata. + metadata := make(map[string]string) + metadata["md5Sum"] = hex.EncodeToString(md5Bytes) + // Create the object. - objectInfo, err = api.Filesystem.CreateObject(bucket, object, size, reader, md5Bytes) + objectInfo, err = api.ObjectAPI.PutObject(bucket, object, size, readCloser, metadata) if err != nil { - errorIf(err.Trace(), "CreateObject failed.", nil) + errorIf(err.Trace(), "PutObject failed.", nil) switch err.ToGoError().(type) { - case fs.RootPathFull: + case RootPathFull: writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path) - case fs.BucketNotFound: + case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) - case fs.BucketNameInvalid: + case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) - case fs.BadDigest: + case BadDigest: writeErrorResponse(w, r, ErrBadDigest, r.URL.Path) - case fs.IncompleteBody: + case IncompleteBody: writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path) - case fs.ObjectExistsAsPrefix: + case ObjectExistsAsPrefix: writeErrorResponse(w, r, ErrObjectExistsAsPrefix, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) @@ -431,6 +457,8 @@ func (api storageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Request) setCommonHeaders(w) // write success response. writeSuccessResponse(w, encodedSuccessResponse) + // Explicitly close the reader, to avoid fd leaks. + readCloser.Close() } // checkCopySource implements x-amz-copy-source-if-modified-since and @@ -528,7 +556,7 @@ func checkCopySourceETag(w http.ResponseWriter, r *http.Request) bool { // PutObjectHandler - PUT Object // ---------- // This implementation of the PUT operation adds an object to a bucket. -func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) { // If the matching failed, it means that the X-Amz-Copy-Source was // wrong, fail right here. if _, ok := r.Header["X-Amz-Copy-Source"]; ok { @@ -558,7 +586,7 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) { return } - var objectInfo fs.ObjectInfo + var objectInfo ObjectInfo switch getRequestAuthType(r) { default: // For all unknown auth types return error. @@ -571,7 +599,7 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) { return } // Create anonymous object. - objectInfo, err = api.Filesystem.CreateObject(bucket, object, size, r.Body, nil) + objectInfo, err = api.ObjectAPI.PutObject(bucket, object, size, r.Body, nil) case authTypePresigned: // For presigned requests verify them right here. if apiErr := doesPresignedSignatureMatch(r); apiErr != ErrNone { @@ -579,7 +607,7 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) { return } // Create presigned object. - objectInfo, err = api.Filesystem.CreateObject(bucket, object, size, r.Body, nil) + objectInfo, err = api.ObjectAPI.PutObject(bucket, object, size, r.Body, nil) case authTypeSigned: // Initialize a pipe for data pipe line. reader, writer := io.Pipe() @@ -605,11 +633,15 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) { writer.Close() }() + // Save metadata. + metadata := make(map[string]string) + metadata["md5Sum"] = hex.EncodeToString(md5Bytes) + // Create object. - objectInfo, err = api.Filesystem.CreateObject(bucket, object, size, reader, md5Bytes) + objectInfo, err = api.ObjectAPI.PutObject(bucket, object, size, reader, metadata) } if err != nil { - errorIf(err.Trace(), "CreateObject failed.", nil) + errorIf(err.Trace(), "PutObject failed.", nil) e := err.ToGoError() // Verify if the underlying error is signature mismatch. if e == errSignatureMismatch { @@ -617,17 +649,17 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) { return } switch e.(type) { - case fs.RootPathFull: + case RootPathFull: writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path) - case fs.BucketNotFound: + case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) - case fs.BucketNameInvalid: + case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) - case fs.BadDigest: + case BadDigest: writeErrorResponse(w, r, ErrBadDigest, r.URL.Path) - case fs.IncompleteBody: + case IncompleteBody: writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path) - case fs.ObjectExistsAsPrefix: + case ObjectExistsAsPrefix: writeErrorResponse(w, r, ErrObjectExistsAsPrefix, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) @@ -640,10 +672,10 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) { writeSuccessResponse(w, nil) } -/// Multipart storageAPI +/// Multipart objectStorageAPI // NewMultipartUploadHandler - New multipart upload -func (api storageAPI) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { var object, bucket string vars := mux.Vars(r) bucket = vars["bucket"] @@ -667,19 +699,19 @@ func (api storageAPI) NewMultipartUploadHandler(w http.ResponseWriter, r *http.R } } - uploadID, err := api.Filesystem.NewMultipartUpload(bucket, object) + uploadID, err := api.ObjectAPI.NewMultipartUpload(bucket, object) if err != nil { errorIf(err.Trace(), "NewMultipartUpload failed.", nil) switch err.ToGoError().(type) { - case fs.RootPathFull: + case RootPathFull: writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path) - case fs.BucketNameInvalid: + case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) - case fs.BucketNotFound: + case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) - case fs.ObjectNotFound: + case ObjectNotFound: writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) - case fs.ObjectNameInvalid: + case ObjectNameInvalid: writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) @@ -696,7 +728,7 @@ func (api storageAPI) NewMultipartUploadHandler(w http.ResponseWriter, r *http.R } // PutObjectPartHandler - Upload part -func (api storageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] @@ -745,7 +777,7 @@ func (api storageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Reques } // No need to verify signature, anonymous request access is // already allowed. - partMD5, err = api.Filesystem.CreateObjectPart(bucket, object, uploadID, partID, size, r.Body, nil) + partMD5, err = api.ObjectAPI.PutObjectPart(bucket, object, uploadID, partID, size, r.Body, hex.EncodeToString(md5Bytes)) case authTypePresigned: // For presigned requests verify right here. apiErr := doesPresignedSignatureMatch(r) @@ -753,7 +785,7 @@ func (api storageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Reques writeErrorResponse(w, r, apiErr, r.URL.Path) return } - partMD5, err = api.Filesystem.CreateObjectPart(bucket, object, uploadID, partID, size, r.Body, nil) + partMD5, err = api.ObjectAPI.PutObjectPart(bucket, object, uploadID, partID, size, r.Body, hex.EncodeToString(md5Bytes)) case authTypeSigned: // Initialize a pipe for data pipe line. reader, writer := io.Pipe() @@ -778,10 +810,10 @@ func (api storageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Reques } writer.Close() }() - partMD5, err = api.Filesystem.CreateObjectPart(bucket, object, uploadID, partID, size, reader, md5Bytes) + partMD5, err = api.ObjectAPI.PutObjectPart(bucket, object, uploadID, partID, size, reader, hex.EncodeToString(md5Bytes)) } if err != nil { - errorIf(err.Trace(), "CreateObjectPart failed.", nil) + errorIf(err.Trace(), "PutObjectPart failed.", nil) e := err.ToGoError() // Verify if the underlying error is signature mismatch. if e == errSignatureMismatch { @@ -789,13 +821,13 @@ func (api storageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Reques return } switch e.(type) { - case fs.RootPathFull: + case RootPathFull: writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path) - case fs.InvalidUploadID: + case InvalidUploadID: writeErrorResponse(w, r, ErrNoSuchUpload, r.URL.Path) - case fs.BadDigest: + case BadDigest: writeErrorResponse(w, r, ErrBadDigest, r.URL.Path) - case fs.IncompleteBody: + case IncompleteBody: writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) @@ -809,7 +841,7 @@ func (api storageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Reques } // AbortMultipartUploadHandler - Abort multipart upload -func (api storageAPI) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] @@ -832,20 +864,20 @@ func (api storageAPI) AbortMultipartUploadHandler(w http.ResponseWriter, r *http } } - objectResourcesMetadata := getObjectResources(r.URL.Query()) - err := api.Filesystem.AbortMultipartUpload(bucket, object, objectResourcesMetadata.UploadID) + uploadID := getUploadID(r.URL.Query()) // Get upload id. + err := api.ObjectAPI.AbortMultipartUpload(bucket, object, uploadID) if err != nil { errorIf(err.Trace(), "AbortMutlipartUpload failed.", nil) switch err.ToGoError().(type) { - case fs.BucketNameInvalid: + case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) - case fs.BucketNotFound: + case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) - case fs.ObjectNotFound: + case ObjectNotFound: writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) - case fs.ObjectNameInvalid: + case ObjectNameInvalid: writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) - case fs.InvalidUploadID: + case InvalidUploadID: writeErrorResponse(w, r, ErrNoSuchUpload, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) @@ -856,7 +888,7 @@ func (api storageAPI) AbortMultipartUploadHandler(w http.ResponseWriter, r *http } // ListObjectPartsHandler - List object parts -func (api storageAPI) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] @@ -892,19 +924,19 @@ func (api storageAPI) ListObjectPartsHandler(w http.ResponseWriter, r *http.Requ objectResourcesMetadata.MaxParts = maxPartsList } - objectResourcesMetadata, err := api.Filesystem.ListObjectParts(bucket, object, objectResourcesMetadata) + objectResourcesMetadata, err := api.ObjectAPI.ListObjectParts(bucket, object, objectResourcesMetadata) if err != nil { errorIf(err.Trace(), "ListObjectParts failed.", nil) switch err.ToGoError().(type) { - case fs.BucketNameInvalid: + case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) - case fs.BucketNotFound: + case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) - case fs.ObjectNotFound: + case ObjectNotFound: writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) - case fs.ObjectNameInvalid: + case ObjectNameInvalid: writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) - case fs.InvalidUploadID: + case InvalidUploadID: writeErrorResponse(w, r, ErrNoSuchUpload, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) @@ -920,16 +952,14 @@ func (api storageAPI) ListObjectPartsHandler(w http.ResponseWriter, r *http.Requ } // CompleteMultipartUploadHandler - Complete multipart upload -func (api storageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] - // Extract object resources. - objectResourcesMetadata := getObjectResources(r.URL.Query()) + // Get upload id. + uploadID := getUploadID(r.URL.Query()) // Get upload id. - var objectInfo fs.ObjectInfo - var err *probe.Error switch getRequestAuthType(r) { default: // For all unknown auth types return error. @@ -941,48 +971,52 @@ func (api storageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter, r *h writeErrorResponse(w, r, s3Error, r.URL.Path) return } - completePartBytes, e := ioutil.ReadAll(r.Body) - if e != nil { - errorIf(probe.NewError(e), "CompleteMultipartUpload failed.", nil) - writeErrorResponse(w, r, ErrInternalError, r.URL.Path) - return - } - // Complete multipart upload anonymous. - objectInfo, err = api.Filesystem.CompleteMultipartUpload(bucket, object, objectResourcesMetadata.UploadID, completePartBytes) case authTypePresigned, authTypeSigned: if s3Error := isReqAuthenticated(r); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } - completePartBytes, e := ioutil.ReadAll(r.Body) - if e != nil { - errorIf(probe.NewError(e), "CompleteMultipartUpload failed.", nil) - writeErrorResponse(w, r, ErrInternalError, r.URL.Path) - return - } - // Complete multipart upload presigned. - objectInfo, err = api.Filesystem.CompleteMultipartUpload(bucket, object, objectResourcesMetadata.UploadID, completePartBytes) } + completeMultipartBytes, e := ioutil.ReadAll(r.Body) + if e != nil { + errorIf(probe.NewError(e), "CompleteMultipartUpload failed.", nil) + writeErrorResponse(w, r, ErrInternalError, r.URL.Path) + return + } + completeMultipartUpload := &CompleteMultipartUpload{} + if e = xml.Unmarshal(completeMultipartBytes, completeMultipartUpload); e != nil { + writeErrorResponse(w, r, ErrMalformedXML, r.URL.Path) + return + } + if !sort.IsSorted(completedParts(completeMultipartUpload.Parts)) { + writeErrorResponse(w, r, ErrInvalidPartOrder, r.URL.Path) + return + } + // Complete parts. + completeParts := completeMultipartUpload.Parts + + // Complete multipart upload. + objectInfo, err := api.ObjectAPI.CompleteMultipartUpload(bucket, object, uploadID, completeParts) if err != nil { errorIf(err.Trace(), "CompleteMultipartUpload failed.", nil) switch err.ToGoError().(type) { - case fs.BucketNameInvalid: + case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) - case fs.BucketNotFound: + case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) - case fs.ObjectNotFound: + case ObjectNotFound: writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) - case fs.ObjectNameInvalid: + case ObjectNameInvalid: writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) - case fs.InvalidUploadID: + case InvalidUploadID: writeErrorResponse(w, r, ErrNoSuchUpload, r.URL.Path) - case fs.InvalidPart: + case InvalidPart: writeErrorResponse(w, r, ErrInvalidPart, r.URL.Path) - case fs.InvalidPartOrder: + case InvalidPartOrder: writeErrorResponse(w, r, ErrInvalidPartOrder, r.URL.Path) - case fs.IncompleteBody: + case IncompleteBody: writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path) - case fs.MalformedXML: + case MalformedXML: writeErrorResponse(w, r, ErrMalformedXML, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) @@ -1000,10 +1034,10 @@ func (api storageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter, r *h writeSuccessResponse(w, encodedSuccessResponse) } -/// Delete storageAPI +/// Delete objectStorageAPI // DeleteObjectHandler - delete an object -func (api storageAPI) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { +func (api objectStorageAPI) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] @@ -1025,17 +1059,17 @@ func (api storageAPI) DeleteObjectHandler(w http.ResponseWriter, r *http.Request return } } - err := api.Filesystem.DeleteObject(bucket, object) + err := api.ObjectAPI.DeleteObject(bucket, object) if err != nil { errorIf(err.Trace(), "DeleteObject failed.", nil) switch err.ToGoError().(type) { - case fs.BucketNameInvalid: + case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) - case fs.BucketNotFound: + case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) - case fs.ObjectNotFound: + case ObjectNotFound: writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) - case fs.ObjectNameInvalid: + case ObjectNameInvalid: writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) diff --git a/object-interface.go b/object-interface.go new file mode 100644 index 000000000..06ab7d0f9 --- /dev/null +++ b/object-interface.go @@ -0,0 +1 @@ +package main diff --git a/routers.go b/routers.go index f724e18d8..cc363424e 100644 --- a/routers.go +++ b/routers.go @@ -20,18 +20,18 @@ import ( "net/http" router "github.com/gorilla/mux" - "github.com/minio/minio/pkg/fs" ) // configureServer handler returns final handler for the http server. -func configureServerHandler(filesystem fs.Filesystem) http.Handler { +func configureServerHandler(objectAPI ObjectAPI) http.Handler { // Initialize API. - api := storageAPI{ - Filesystem: filesystem, + api := objectStorageAPI{ + ObjectAPI: objectAPI, } + // Initialize Web. web := &webAPI{ - Filesystem: filesystem, + ObjectAPI: objectAPI, } // Initialize router. diff --git a/server_fs_test.go b/server_fs_test.go index 010da0ded..b5b3b347c 100644 --- a/server_fs_test.go +++ b/server_fs_test.go @@ -35,22 +35,23 @@ import ( "net/http" "net/http/httptest" - "github.com/minio/minio/pkg/fs" . "gopkg.in/check.v1" ) +// Concurreny level. const ( ConcurrencyLevel = 10 ) -type MyAPIFSCacheSuite struct { +// API suite container. +type MyAPISuite struct { root string req *http.Request body io.ReadSeeker credential credential } -var _ = Suite(&MyAPIFSCacheSuite{}) +var _ = Suite(&MyAPISuite{}) var testAPIFSCacheServer *httptest.Server @@ -69,7 +70,7 @@ func getFreePort() int { return l.Addr().(*net.TCPAddr).Port } -func (s *MyAPIFSCacheSuite) SetUpSuite(c *C) { +func (s *MyAPISuite) SetUpSuite(c *C) { root, e := ioutil.TempDir(os.TempDir(), "api-") c.Assert(e, IsNil) s.root = root @@ -95,14 +96,14 @@ func (s *MyAPIFSCacheSuite) SetUpSuite(c *C) { // Save config. c.Assert(serverConfig.Save(), IsNil) - fs, err := fs.New(fsroot) + fs, err := newFS(fsroot) c.Assert(err, IsNil) httpHandler := configureServerHandler(fs) testAPIFSCacheServer = httptest.NewServer(httpHandler) } -func (s *MyAPIFSCacheSuite) TearDownSuite(c *C) { +func (s *MyAPISuite) TearDownSuite(c *C) { os.RemoveAll(s.root) testAPIFSCacheServer.Close() } @@ -142,7 +143,7 @@ var ignoredHeaders = map[string]bool{ "User-Agent": true, } -func (s *MyAPIFSCacheSuite) newRequest(method, urlStr string, contentLength int64, body io.ReadSeeker) (*http.Request, error) { +func (s *MyAPISuite) newRequest(method, urlStr string, contentLength int64, body io.ReadSeeker) (*http.Request, error) { if method == "" { method = "POST" } @@ -267,7 +268,7 @@ func (s *MyAPIFSCacheSuite) newRequest(method, urlStr string, contentLength int6 return req, nil } -func (s *MyAPIFSCacheSuite) TestAuth(c *C) { +func (s *MyAPISuite) TestAuth(c *C) { secretID, err := genSecretAccessKey() c.Assert(err, IsNil) @@ -278,7 +279,7 @@ func (s *MyAPIFSCacheSuite) TestAuth(c *C) { c.Assert(len(accessID), Equals, minioAccessID) } -func (s *MyAPIFSCacheSuite) TestBucketPolicy(c *C) { +func (s *MyAPISuite) TestBucketPolicy(c *C) { // Sample bucket policy. bucketPolicyBuf := `{ "Version": "2012-10-17", @@ -348,7 +349,7 @@ func (s *MyAPIFSCacheSuite) TestBucketPolicy(c *C) { c.Assert(response.StatusCode, Equals, http.StatusNoContent) } -func (s *MyAPIFSCacheSuite) TestDeleteBucket(c *C) { +func (s *MyAPISuite) TestDeleteBucket(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/deletebucket", 0, nil) c.Assert(err, IsNil) @@ -366,7 +367,7 @@ func (s *MyAPIFSCacheSuite) TestDeleteBucket(c *C) { c.Assert(response.StatusCode, Equals, http.StatusNoContent) } -func (s *MyAPIFSCacheSuite) TestDeleteObject(c *C) { +func (s *MyAPISuite) TestDeleteObject(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/deletebucketobject", 0, nil) c.Assert(err, IsNil) @@ -391,7 +392,7 @@ func (s *MyAPIFSCacheSuite) TestDeleteObject(c *C) { c.Assert(response.StatusCode, Equals, http.StatusNoContent) } -func (s *MyAPIFSCacheSuite) TestNonExistantBucket(c *C) { +func (s *MyAPISuite) TestNonExistantBucket(c *C) { request, err := s.newRequest("HEAD", testAPIFSCacheServer.URL+"/nonexistantbucket", 0, nil) c.Assert(err, IsNil) @@ -401,7 +402,7 @@ func (s *MyAPIFSCacheSuite) TestNonExistantBucket(c *C) { c.Assert(response.StatusCode, Equals, http.StatusNotFound) } -func (s *MyAPIFSCacheSuite) TestEmptyObject(c *C) { +func (s *MyAPISuite) TestEmptyObject(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/emptyobject", 0, nil) c.Assert(err, IsNil) @@ -432,7 +433,7 @@ func (s *MyAPIFSCacheSuite) TestEmptyObject(c *C) { c.Assert(true, Equals, bytes.Equal(responseBody, buffer.Bytes())) } -func (s *MyAPIFSCacheSuite) TestBucket(c *C) { +func (s *MyAPISuite) TestBucket(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/bucket", 0, nil) c.Assert(err, IsNil) @@ -450,7 +451,7 @@ func (s *MyAPIFSCacheSuite) TestBucket(c *C) { c.Assert(response.StatusCode, Equals, http.StatusOK) } -func (s *MyAPIFSCacheSuite) TestObject(c *C) { +func (s *MyAPISuite) TestObject(c *C) { buffer := bytes.NewReader([]byte("hello world")) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/testobject", 0, nil) c.Assert(err, IsNil) @@ -482,7 +483,7 @@ func (s *MyAPIFSCacheSuite) TestObject(c *C) { } -func (s *MyAPIFSCacheSuite) TestMultipleObjects(c *C) { +func (s *MyAPISuite) TestMultipleObjects(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/multipleobjects", 0, nil) c.Assert(err, IsNil) @@ -569,7 +570,7 @@ func (s *MyAPIFSCacheSuite) TestMultipleObjects(c *C) { c.Assert(true, Equals, bytes.Equal(responseBody, []byte("hello three"))) } -func (s *MyAPIFSCacheSuite) TestNotImplemented(c *C) { +func (s *MyAPISuite) TestNotImplemented(c *C) { request, err := s.newRequest("GET", testAPIFSCacheServer.URL+"/bucket/object?policy", 0, nil) c.Assert(err, IsNil) @@ -579,7 +580,7 @@ func (s *MyAPIFSCacheSuite) TestNotImplemented(c *C) { c.Assert(response.StatusCode, Equals, http.StatusNotImplemented) } -func (s *MyAPIFSCacheSuite) TestHeader(c *C) { +func (s *MyAPISuite) TestHeader(c *C) { request, err := s.newRequest("GET", testAPIFSCacheServer.URL+"/bucket/object", 0, nil) c.Assert(err, IsNil) @@ -590,7 +591,7 @@ func (s *MyAPIFSCacheSuite) TestHeader(c *C) { verifyError(c, response, "NoSuchKey", "The specified key does not exist.", http.StatusNotFound) } -func (s *MyAPIFSCacheSuite) TestPutBucket(c *C) { +func (s *MyAPISuite) TestPutBucket(c *C) { // Block 1: Testing for racey access // The assertion is removed from this block since the purpose of this block is to find races // The purpose this block is not to check for correctness of functionality @@ -602,7 +603,6 @@ func (s *MyAPIFSCacheSuite) TestPutBucket(c *C) { defer wg.Done() request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/put-bucket", 0, nil) c.Assert(err, IsNil) - request.Header.Add("x-amz-acl", "private") client := http.Client{} response, err := client.Do(request) @@ -614,7 +614,6 @@ func (s *MyAPIFSCacheSuite) TestPutBucket(c *C) { //Block 2: testing for correctness of the functionality request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/put-bucket-slash/", 0, nil) c.Assert(err, IsNil) - request.Header.Add("x-amz-acl", "private") client := http.Client{} response, err := client.Do(request) @@ -624,10 +623,9 @@ func (s *MyAPIFSCacheSuite) TestPutBucket(c *C) { } -func (s *MyAPIFSCacheSuite) TestCopyObject(c *C) { +func (s *MyAPISuite) TestCopyObject(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/put-object-copy", 0, nil) c.Assert(err, IsNil) - request.Header.Add("x-amz-acl", "private") client := http.Client{} response, err := client.Do(request) @@ -662,7 +660,7 @@ func (s *MyAPIFSCacheSuite) TestCopyObject(c *C) { c.Assert(string(object), Equals, "hello world") } -func (s *MyAPIFSCacheSuite) TestPutObject(c *C) { +func (s *MyAPISuite) TestPutObject(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/put-object", 0, nil) c.Assert(err, IsNil) @@ -680,7 +678,7 @@ func (s *MyAPIFSCacheSuite) TestPutObject(c *C) { c.Assert(response.StatusCode, Equals, http.StatusOK) } -func (s *MyAPIFSCacheSuite) TestListBuckets(c *C) { +func (s *MyAPISuite) TestListBuckets(c *C) { request, err := s.newRequest("GET", testAPIFSCacheServer.URL+"/", 0, nil) c.Assert(err, IsNil) @@ -695,7 +693,7 @@ func (s *MyAPIFSCacheSuite) TestListBuckets(c *C) { c.Assert(err, IsNil) } -func (s *MyAPIFSCacheSuite) TestNotBeAbleToCreateObjectInNonexistantBucket(c *C) { +func (s *MyAPISuite) TestNotBeAbleToCreateObjectInNonexistantBucket(c *C) { buffer1 := bytes.NewReader([]byte("hello world")) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/innonexistantbucket/object", int64(buffer1.Len()), buffer1) c.Assert(err, IsNil) @@ -706,7 +704,7 @@ func (s *MyAPIFSCacheSuite) TestNotBeAbleToCreateObjectInNonexistantBucket(c *C) verifyError(c, response, "NoSuchBucket", "The specified bucket does not exist.", http.StatusNotFound) } -func (s *MyAPIFSCacheSuite) TestHeadOnObject(c *C) { +func (s *MyAPISuite) TestHeadOnObject(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/headonobject", 0, nil) c.Assert(err, IsNil) @@ -749,7 +747,7 @@ func (s *MyAPIFSCacheSuite) TestHeadOnObject(c *C) { c.Assert(response.StatusCode, Equals, http.StatusPreconditionFailed) } -func (s *MyAPIFSCacheSuite) TestHeadOnBucket(c *C) { +func (s *MyAPISuite) TestHeadOnBucket(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/headonbucket", 0, nil) c.Assert(err, IsNil) @@ -766,7 +764,7 @@ func (s *MyAPIFSCacheSuite) TestHeadOnBucket(c *C) { c.Assert(response.StatusCode, Equals, http.StatusOK) } -func (s *MyAPIFSCacheSuite) TestXMLNameNotInBucketListJson(c *C) { +func (s *MyAPISuite) TestXMLNameNotInBucketListJson(c *C) { request, err := s.newRequest("GET", testAPIFSCacheServer.URL+"/", 0, nil) c.Assert(err, IsNil) request.Header.Add("Accept", "application/json") @@ -781,7 +779,7 @@ func (s *MyAPIFSCacheSuite) TestXMLNameNotInBucketListJson(c *C) { c.Assert(strings.Contains(string(byteResults), "XML"), Equals, false) } -func (s *MyAPIFSCacheSuite) TestXMLNameNotInObjectListJson(c *C) { +func (s *MyAPISuite) TestXMLNameNotInObjectListJson(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/xmlnamenotinobjectlistjson", 0, nil) c.Assert(err, IsNil) request.Header.Add("Accept", "application/json") @@ -805,7 +803,7 @@ func (s *MyAPIFSCacheSuite) TestXMLNameNotInObjectListJson(c *C) { c.Assert(strings.Contains(string(byteResults), "XML"), Equals, false) } -func (s *MyAPIFSCacheSuite) TestContentTypePersists(c *C) { +func (s *MyAPISuite) TestContentTypePersists(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/contenttype-persists", 0, nil) c.Assert(err, IsNil) @@ -865,7 +863,7 @@ func (s *MyAPIFSCacheSuite) TestContentTypePersists(c *C) { c.Assert(response.Header.Get("Content-Type"), Equals, "application/octet-stream") } -func (s *MyAPIFSCacheSuite) TestPartialContent(c *C) { +func (s *MyAPISuite) TestPartialContent(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/partial-content", 0, nil) c.Assert(err, IsNil) @@ -898,7 +896,7 @@ func (s *MyAPIFSCacheSuite) TestPartialContent(c *C) { c.Assert(string(partialObject), Equals, "Wo") } -func (s *MyAPIFSCacheSuite) TestListObjectsHandlerErrors(c *C) { +func (s *MyAPISuite) TestListObjectsHandlerErrors(c *C) { request, err := s.newRequest("GET", testAPIFSCacheServer.URL+"/objecthandlererrors-.", 0, nil) c.Assert(err, IsNil) @@ -931,7 +929,7 @@ func (s *MyAPIFSCacheSuite) TestListObjectsHandlerErrors(c *C) { verifyError(c, response, "InvalidArgument", "Argument maxKeys must be an integer between 0 and 2147483647.", http.StatusBadRequest) } -func (s *MyAPIFSCacheSuite) TestPutBucketErrors(c *C) { +func (s *MyAPISuite) TestPutBucketErrors(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/putbucket-.", 0, nil) c.Assert(err, IsNil) @@ -963,7 +961,7 @@ func (s *MyAPIFSCacheSuite) TestPutBucketErrors(c *C) { verifyError(c, response, "NotImplemented", "A header you provided implies functionality that is not implemented.", http.StatusNotImplemented) } -func (s *MyAPIFSCacheSuite) TestGetObjectErrors(c *C) { +func (s *MyAPISuite) TestGetObjectErrors(c *C) { request, err := s.newRequest("GET", testAPIFSCacheServer.URL+"/getobjecterrors", 0, nil) c.Assert(err, IsNil) @@ -997,7 +995,7 @@ func (s *MyAPIFSCacheSuite) TestGetObjectErrors(c *C) { } -func (s *MyAPIFSCacheSuite) TestGetObjectRangeErrors(c *C) { +func (s *MyAPISuite) TestGetObjectRangeErrors(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/getobjectrangeerrors", 0, nil) c.Assert(err, IsNil) @@ -1025,7 +1023,7 @@ func (s *MyAPIFSCacheSuite) TestGetObjectRangeErrors(c *C) { verifyError(c, response, "InvalidRange", "The requested range cannot be satisfied.", http.StatusRequestedRangeNotSatisfiable) } -func (s *MyAPIFSCacheSuite) TestObjectMultipartAbort(c *C) { +func (s *MyAPISuite) TestObjectMultipartAbort(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultipartabort", 0, nil) c.Assert(err, IsNil) @@ -1072,7 +1070,8 @@ func (s *MyAPIFSCacheSuite) TestObjectMultipartAbort(c *C) { c.Assert(response3.StatusCode, Equals, http.StatusNoContent) } -func (s *MyAPIFSCacheSuite) TestBucketMultipartList(c *C) { +/* +func (s *MyAPISuite) TestBucketMultipartList(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/bucketmultipartlist", 0, nil) c.Assert(err, IsNil) @@ -1085,6 +1084,7 @@ func (s *MyAPIFSCacheSuite) TestBucketMultipartList(c *C) { c.Assert(err, IsNil) response, err = client.Do(request) + c.Assert(err, IsNil) c.Assert(response.StatusCode, Equals, http.StatusOK) decoder := xml.NewDecoder(response.Body) @@ -1159,8 +1159,9 @@ func (s *MyAPIFSCacheSuite) TestBucketMultipartList(c *C) { c.Assert(err, IsNil) c.Assert(newResponse3.Bucket, Equals, "bucketmultipartlist") } +*/ -func (s *MyAPIFSCacheSuite) TestValidateObjectMultipartUploadID(c *C) { +func (s *MyAPISuite) TestValidateObjectMultipartUploadID(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultipartlist-uploadid", 0, nil) c.Assert(err, IsNil) @@ -1183,7 +1184,7 @@ func (s *MyAPIFSCacheSuite) TestValidateObjectMultipartUploadID(c *C) { c.Assert(len(newResponse.UploadID) > 0, Equals, true) } -func (s *MyAPIFSCacheSuite) TestObjectMultipartList(c *C) { +func (s *MyAPISuite) TestObjectMultipartList(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultipartlist", 0, nil) c.Assert(err, IsNil) @@ -1237,7 +1238,7 @@ func (s *MyAPIFSCacheSuite) TestObjectMultipartList(c *C) { verifyError(c, response4, "InvalidArgument", "Argument maxParts must be an integer between 1 and 10000.", http.StatusBadRequest) } -func (s *MyAPIFSCacheSuite) TestObjectMultipart(c *C) { +func (s *MyAPISuite) TestObjectMultipart(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultiparts", 0, nil) c.Assert(err, IsNil) @@ -1287,8 +1288,8 @@ func (s *MyAPIFSCacheSuite) TestObjectMultipart(c *C) { c.Assert(response2.StatusCode, Equals, http.StatusOK) // Complete multipart upload - completeUploads := &fs.CompleteMultipartUpload{ - Part: []fs.CompletePart{ + completeUploads := &CompleteMultipartUpload{ + Parts: []CompletePart{ { PartNumber: 1, ETag: response1.Header.Get("ETag"), diff --git a/web-handlers.go b/web-handlers.go index e04aeb38e..ce6f28d57 100644 --- a/web-handlers.go +++ b/web-handlers.go @@ -18,6 +18,7 @@ package main import ( "fmt" + "io" "net/http" "os" "path" @@ -31,7 +32,6 @@ import ( "github.com/gorilla/mux" "github.com/gorilla/rpc/v2/json2" "github.com/minio/minio/pkg/disk" - "github.com/minio/minio/pkg/fs" "github.com/minio/miniobrowser" ) @@ -110,7 +110,7 @@ func (web *webAPI) DiskInfo(r *http.Request, args *GenericArgs, reply *DiskInfoR if !isJWTReqAuthenticated(r) { return &json2.Error{Message: "Unauthorized request"} } - info, e := disk.GetInfo(web.Filesystem.GetRootPath()) + info, e := disk.GetInfo(web.ObjectAPI.(*Filesystem).GetRootPath()) if e != nil { return &json2.Error{Message: e.Error()} } @@ -130,7 +130,7 @@ func (web *webAPI) MakeBucket(r *http.Request, args *MakeBucketArgs, reply *Gene return &json2.Error{Message: "Unauthorized request"} } reply.UIVersion = miniobrowser.UIVersion - e := web.Filesystem.MakeBucket(args.BucketName) + e := web.ObjectAPI.MakeBucket(args.BucketName) if e != nil { return &json2.Error{Message: e.Cause.Error()} } @@ -139,12 +139,12 @@ func (web *webAPI) MakeBucket(r *http.Request, args *MakeBucketArgs, reply *Gene // ListBucketsRep - list buckets response type ListBucketsRep struct { - Buckets []BucketInfo `json:"buckets"` - UIVersion string `json:"uiVersion"` + Buckets []BketInfo `json:"buckets"` + UIVersion string `json:"uiVersion"` } -// BucketInfo container for list buckets metadata. -type BucketInfo struct { +// BketInfo container for list buckets. +type BketInfo struct { // The name of the bucket. Name string `json:"name"` // Date the bucket was created. @@ -156,14 +156,14 @@ func (web *webAPI) ListBuckets(r *http.Request, args *GenericArgs, reply *ListBu if !isJWTReqAuthenticated(r) { return &json2.Error{Message: "Unauthorized request"} } - buckets, e := web.Filesystem.ListBuckets() + buckets, e := web.ObjectAPI.ListBuckets() if e != nil { return &json2.Error{Message: e.Cause.Error()} } for _, bucket := range buckets { // List all buckets which are not private. if bucket.Name != path.Base(reservedBucket) { - reply.Buckets = append(reply.Buckets, BucketInfo{ + reply.Buckets = append(reply.Buckets, BketInfo{ Name: bucket.Name, CreationDate: bucket.Created, }) @@ -181,12 +181,12 @@ type ListObjectsArgs struct { // ListObjectsRep - list objects response. type ListObjectsRep struct { - Objects []ObjectInfo `json:"objects"` - UIVersion string `json:"uiVersion"` + Objects []ObjInfo `json:"objects"` + UIVersion string `json:"uiVersion"` } -// ObjectInfo container for list objects metadata. -type ObjectInfo struct { +// ObjInfo container for list objects. +type ObjInfo struct { // Name of the object Key string `json:"name"` // Date and time the object was last modified. @@ -204,20 +204,20 @@ func (web *webAPI) ListObjects(r *http.Request, args *ListObjectsArgs, reply *Li return &json2.Error{Message: "Unauthorized request"} } for { - lo, err := web.Filesystem.ListObjects(args.BucketName, args.Prefix, marker, "/", 1000) + lo, err := web.ObjectAPI.ListObjects(args.BucketName, args.Prefix, marker, "/", 1000) if err != nil { return &json2.Error{Message: err.Cause.Error()} } marker = lo.NextMarker for _, obj := range lo.Objects { - reply.Objects = append(reply.Objects, ObjectInfo{ + reply.Objects = append(reply.Objects, ObjInfo{ Key: obj.Name, LastModified: obj.ModifiedTime, Size: obj.Size, }) } for _, prefix := range lo.Prefixes { - reply.Objects = append(reply.Objects, ObjectInfo{ + reply.Objects = append(reply.Objects, ObjInfo{ Key: prefix, }) } @@ -242,7 +242,7 @@ func (web *webAPI) RemoveObject(r *http.Request, args *RemoveObjectArgs, reply * return &json2.Error{Message: "Unauthorized request"} } reply.UIVersion = miniobrowser.UIVersion - e := web.Filesystem.DeleteObject(args.BucketName, args.ObjectName) + e := web.ObjectAPI.DeleteObject(args.BucketName, args.ObjectName) if e != nil { return &json2.Error{Message: e.Cause.Error()} } @@ -364,7 +364,7 @@ func (web *webAPI) Upload(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] - if _, err := web.Filesystem.CreateObject(bucket, object, -1, r.Body, nil); err != nil { + if _, err := web.ObjectAPI.PutObject(bucket, object, -1, r.Body, nil); err != nil { writeWebErrorResponse(w, err.ToGoError()) } } @@ -389,8 +389,14 @@ func (web *webAPI) Download(w http.ResponseWriter, r *http.Request) { } w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filepath.Base(object))) - if _, err := web.Filesystem.GetObject(w, bucket, object, 0, 0); err != nil { + objReader, err := web.ObjectAPI.GetObject(bucket, object, 0) + if err != nil { writeWebErrorResponse(w, err.ToGoError()) + return + } + if _, e := io.Copy(w, objReader); e != nil { + /// No need to print error, response writer already written to. + return } } @@ -402,35 +408,35 @@ func writeWebErrorResponse(w http.ResponseWriter, err error) { return } switch err.(type) { - case fs.RootPathFull: + case RootPathFull: apiErr := getAPIError(ErrRootPathFull) w.WriteHeader(apiErr.HTTPStatusCode) w.Write([]byte(apiErr.Description)) - case fs.BucketNotFound: + case BucketNotFound: apiErr := getAPIError(ErrNoSuchBucket) w.WriteHeader(apiErr.HTTPStatusCode) w.Write([]byte(apiErr.Description)) - case fs.BucketNameInvalid: + case BucketNameInvalid: apiErr := getAPIError(ErrInvalidBucketName) w.WriteHeader(apiErr.HTTPStatusCode) w.Write([]byte(apiErr.Description)) - case fs.BadDigest: + case BadDigest: apiErr := getAPIError(ErrBadDigest) w.WriteHeader(apiErr.HTTPStatusCode) w.Write([]byte(apiErr.Description)) - case fs.IncompleteBody: + case IncompleteBody: apiErr := getAPIError(ErrIncompleteBody) w.WriteHeader(apiErr.HTTPStatusCode) w.Write([]byte(apiErr.Description)) - case fs.ObjectExistsAsPrefix: + case ObjectExistsAsPrefix: apiErr := getAPIError(ErrObjectExistsAsPrefix) w.WriteHeader(apiErr.HTTPStatusCode) w.Write([]byte(apiErr.Description)) - case fs.ObjectNotFound: + case ObjectNotFound: apiErr := getAPIError(ErrNoSuchKey) w.WriteHeader(apiErr.HTTPStatusCode) w.Write([]byte(apiErr.Description)) - case fs.ObjectNameInvalid: + case ObjectNameInvalid: apiErr := getAPIError(ErrNoSuchKey) w.WriteHeader(apiErr.HTTPStatusCode) w.Write([]byte(apiErr.Description)) diff --git a/web-router.go b/web-router.go index 97f76e23c..c476545fc 100644 --- a/web-router.go +++ b/web-router.go @@ -25,13 +25,12 @@ import ( router "github.com/gorilla/mux" jsonrpc "github.com/gorilla/rpc/v2" "github.com/gorilla/rpc/v2/json2" - "github.com/minio/minio/pkg/fs" "github.com/minio/miniobrowser" ) // webAPI container for Web API. type webAPI struct { - Filesystem fs.Filesystem + ObjectAPI ObjectAPI } // indexHandler - Handler to serve index.html