From bc52d911ef723e63e4bdc312840ded2e4ca64be6 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 3 Mar 2017 10:14:17 -0800 Subject: [PATCH] api: Increase the maximum object size limit from 5GiB to 16GiB. (#3834) The globalMaxObjectSize limit is instilled in S3 spec perhaps due to certain limitations on S3 infrastructure. For minio we don't have such limitations and we can stream a larger file instead. So we are going to bump this limit to 16GiB. Fixes #3825 --- cmd/admin-handlers_test.go | 8 ++++---- cmd/bucket-handlers.go | 2 +- cmd/generic-handlers.go | 6 +++--- cmd/object-handlers.go | 5 +++-- cmd/object-handlers_test.go | 8 ++++---- cmd/utils.go | 32 +++++++++++++++++++++++--------- cmd/utils_test.go | 12 ++++++------ 7 files changed, 44 insertions(+), 29 deletions(-) diff --git a/cmd/admin-handlers_test.go b/cmd/admin-handlers_test.go index a1018d7d0..7b1a20f99 100644 --- a/cmd/admin-handlers_test.go +++ b/cmd/admin-handlers_test.go @@ -1239,16 +1239,16 @@ func TestWriteSetConfigResponse(t *testing.T) { } testPeers := []adminPeer{ - adminPeer{ + { addr: "localhost:9001", }, - adminPeer{ + { addr: "localhost:9002", }, - adminPeer{ + { addr: "localhost:9003", }, - adminPeer{ + { addr: "localhost:9004", }, } diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index 0f8ef0151..1c6bc67c6 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -486,7 +486,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h return } - if fileSize > lengthRange.Max || fileSize > maxObjectSize { + if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) { errorIf(err, "Unable to create object.") writeErrorResponse(w, toAPIErrorCode(errDataTooLarge), r.URL) return diff --git a/cmd/generic-handlers.go b/cmd/generic-handlers.go index 3f94789f5..1f69e5930 100644 --- a/cmd/generic-handlers.go +++ b/cmd/generic-handlers.go @@ -46,9 +46,9 @@ func registerHandlers(mux *router.Router, handlerFns ...HandlerFunc) http.Handle // which is more than enough to accommodate any form data fields and headers. const requestFormDataSize = 64 * humanize.MiByte -// For any HTTP request, request body should be not more than 5GiB + requestFormDataSize -// where, 5GiB is the maximum allowed object size for object upload. -const requestMaxBodySize = 5*humanize.GiByte + requestFormDataSize +// For any HTTP request, request body should be not more than 16GiB + requestFormDataSize +// where, 16GiB is the maximum allowed object size for object upload. +const requestMaxBodySize = globalMaxObjectSize + requestFormDataSize type requestSizeLimitHandler struct { handler http.Handler diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index cb6d2b39c..2e2717688 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -628,7 +628,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt } /// maximum copy size for multipart objects in a single operation - if isMaxObjectSize(length) { + if isMaxAllowedPartSize(length) { writeErrorResponse(w, ErrEntityTooLarge, r.URL) return } @@ -637,6 +637,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt // object is same then only metadata is updated. partInfo, err := objectAPI.CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, uploadID, partID, startOffset, length) if err != nil { + errorIf(err, "Unable to perform CopyObjectPart %s/%s", srcBucket, srcObject) writeErrorResponse(w, toAPIErrorCode(err), r.URL) return } @@ -687,7 +688,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http } /// maximum Upload size for multipart objects in a single operation - if isMaxObjectSize(size) { + if isMaxAllowedPartSize(size) { writeErrorResponse(w, ErrEntityTooLarge, r.URL) return } diff --git a/cmd/object-handlers_test.go b/cmd/object-handlers_test.go index e68a80ae4..3cf72167c 100644 --- a/cmd/object-handlers_test.go +++ b/cmd/object-handlers_test.go @@ -840,7 +840,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a req.ContentLength = -1 req.TransferEncoding = []string{} case TooBigObject: - req.ContentLength = maxObjectSize + 1 + req.ContentLength = globalMaxObjectSize + 1 } // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. // Call the ServeHTTP to execute the handler,`func (api objectAPIHandlers) GetObjectHandler` handles the request. @@ -883,7 +883,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a reqV2.ContentLength = -1 reqV2.TransferEncoding = []string{} case TooBigObject: - reqV2.ContentLength = maxObjectSize + 1 + reqV2.ContentLength = globalMaxObjectSize + 1 } // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. @@ -2722,7 +2722,7 @@ func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName strin { objectName: testObject, reader: bytes.NewReader([]byte("hello")), - partNumber: strconv.Itoa(maxPartID + 1), + partNumber: strconv.Itoa(globalMaxPartID + 1), fault: None, accessKey: credentials.AccessKey, secretKey: credentials.SecretKey, @@ -2880,7 +2880,7 @@ func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName strin // Setting the content length to a value greater than the max allowed size of a part. // Used in test case 4. case TooBigObject: - req.ContentLength = maxObjectSize + 1 + req.ContentLength = globalMaxObjectSize + 1 // Malformed signature. // Used in test case 6. case BadSignature: diff --git a/cmd/utils.go b/cmd/utils.go index 49657cf13..3eeedae09 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -170,27 +170,41 @@ func checkValidMD5(md5 string) ([]byte, error) { /// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html const ( - // maximum object size per PUT request is 5GiB - maxObjectSize = 5 * humanize.GiByte - // minimum Part size for multipart upload is 5MiB - minPartSize = 5 * humanize.MiByte - // maximum Part ID for multipart upload is 10000 (Acceptable values range from 1 to 10000 inclusive) - maxPartID = 10000 + // Maximum object size per PUT request is 16GiB. + // This is a divergence from S3 limit on purpose to support + // use cases where users are going to upload large files + // using 'curl' and presigned URL. + globalMaxObjectSize = 16 * humanize.GiByte + + // Minimum Part size for multipart upload is 5MiB + globalMinPartSize = 5 * humanize.MiByte + + // Maximum Part size for multipart upload is 5GiB + globalMaxPartSize = 5 * humanize.GiByte + + // Maximum Part ID for multipart upload is 10000 + // (Acceptable values range from 1 to 10000 inclusive) + globalMaxPartID = 10000 ) // isMaxObjectSize - verify if max object size func isMaxObjectSize(size int64) bool { - return size > maxObjectSize + return size > globalMaxObjectSize +} + +// // Check if part size is more than maximum allowed size. +func isMaxAllowedPartSize(size int64) bool { + return size > globalMaxPartSize } // Check if part size is more than or equal to minimum allowed size. func isMinAllowedPartSize(size int64) bool { - return size >= minPartSize + return size >= globalMinPartSize } // isMaxPartNumber - Check if part ID is greater than the maximum allowed ID. func isMaxPartID(partID int) bool { - return partID > maxPartID + return partID > globalMaxPartID } func contains(stringList []string, element string) bool { diff --git a/cmd/utils_test.go b/cmd/utils_test.go index d2b2ba9f6..d0d11c3f4 100644 --- a/cmd/utils_test.go +++ b/cmd/utils_test.go @@ -110,12 +110,12 @@ func TestMaxObjectSize(t *testing.T) { // Test - 1 - maximum object size. { true, - maxObjectSize + 1, + globalMaxObjectSize + 1, }, // Test - 2 - not maximum object size. { false, - maxObjectSize - 1, + globalMaxObjectSize - 1, }, } for i, s := range sizes { @@ -135,12 +135,12 @@ func TestMinAllowedPartSize(t *testing.T) { // Test - 1 - within minimum part size. { true, - minPartSize + 1, + globalMinPartSize + 1, }, // Test - 2 - smaller than minimum part size. { false, - minPartSize - 1, + globalMinPartSize - 1, }, } @@ -161,12 +161,12 @@ func TestMaxPartID(t *testing.T) { // Test - 1 part number within max part number. { false, - maxPartID - 1, + globalMaxPartID - 1, }, // Test - 2 part number bigger than max part number. { true, - maxPartID + 1, + globalMaxPartID + 1, }, }