From 2743d4ca87554a5bab6abda3cd0b74c78cd24f8b Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 8 Jul 2020 17:36:56 -0700 Subject: [PATCH] fix: Add support for preserving mtime for replication (#9995) This PR is needed for bucket replication support --- cmd/admin-handlers.go | 2 +- cmd/api-errors.go | 6 + cmd/api-response.go | 8 ++ cmd/bucket-handlers.go | 44 +++---- cmd/encryption-v1.go | 202 +----------------------------- cmd/erasure-multipart.go | 11 +- cmd/erasure-object.go | 8 +- cmd/erasure-zones.go | 2 +- cmd/handler-utils.go | 23 +++- cmd/http/headers.go | 15 ++- cmd/object-api-errors.go | 11 ++ cmd/object-api-interface.go | 12 +- cmd/object-api-options.go | 242 ++++++++++++++++++++++++++++++++++++ cmd/object-handlers.go | 38 ++---- 14 files changed, 358 insertions(+), 266 deletions(-) create mode 100644 cmd/object-api-options.go diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index a5e9dcf46..d539a0e8f 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -1419,7 +1419,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque } } - mode := "safe" + mode := "safemode" if newObjectLayerFn() != nil { mode = "online" } diff --git a/cmd/api-errors.go b/cmd/api-errors.go index 961735be4..3af53abb3 100644 --- a/cmd/api-errors.go +++ b/cmd/api-errors.go @@ -1909,6 +1909,12 @@ func toAPIError(ctx context.Context, err error) APIError { // their internal error types. This code is only // useful with gateway implementations. switch e := err.(type) { + case InvalidArgument: + apiErr = APIError{ + Code: "InvalidArgument", + Description: e.Error(), + HTTPStatusCode: errorCodes[ErrInvalidRequest].HTTPStatusCode, + } case *xml.SyntaxError: apiErr = APIError{ Code: "MalformedXML", diff --git a/cmd/api-response.go b/cmd/api-response.go index d4e22782c..43e5ffb29 100644 --- a/cmd/api-response.go +++ b/cmd/api-response.go @@ -698,6 +698,10 @@ func generateMultiDeleteResponse(quiet bool, deletedObjects []DeletedObject, err } func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) { + if newObjectLayerFn() == nil { + // Server still in safe mode. + w.Header().Set(xhttp.MinIOServerStatus, "safemode") + } setCommonHeaders(w) if mType != mimeNone { w.Header().Set(xhttp.ContentType, string(mType)) @@ -760,6 +764,10 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError // The request is from browser and also if browser // is enabled we need to redirect. if browser && globalBrowserEnabled { + if newObjectLayerFn() == nil { + // server still in safe mode. + w.Header().Set(xhttp.MinIOServerStatus, "safemode") + } w.Header().Set(xhttp.Location, minioReservedBucketPath+reqURL.Path) w.WriteHeader(http.StatusTemporaryRedirect) return diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index e7492b988..472a056b1 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -25,6 +25,7 @@ import ( "net/url" "path" "path/filepath" + "strconv" "strings" "github.com/gorilla/mux" @@ -918,34 +919,35 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http. return } - forceDelete := false - if value := r.Header.Get(xhttp.MinIOForceDelete); value != "" { - switch value { - case "true": - forceDelete = true - case "false": - default: - writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL, guessIsBrowserReq(r)) - return - } + // Verify if the caller has sufficient permissions. + if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) + return } - if forceDelete { + forceDelete := false + if value := r.Header.Get(xhttp.MinIOForceDelete); value != "" { + var err error + forceDelete, err = strconv.ParseBool(value) + if err != nil { + apiErr := errorCodes.ToAPIErr(ErrInvalidRequest) + apiErr.Description = err.Error() + writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r)) + return + } + + // if force delete header is set, we need to evaluate the policy anyways + // regardless of it being true or not. if s3Error := checkRequestAuthType(ctx, r, policy.ForceDeleteBucketAction, bucket, ""); s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) return } - } else { - if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone { - writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) - return - } - } - if forceDelete { - if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled { - writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r)) - return + if forceDelete { + if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r)) + return + } } } diff --git a/cmd/encryption-v1.go b/cmd/encryption-v1.go index 6e23d05e6..27ff56a9f 100644 --- a/cmd/encryption-v1.go +++ b/cmd/encryption-v1.go @@ -1,5 +1,5 @@ /* - * MinIO Cloud Storage, (C) 2017, 2018 MinIO, Inc. + * MinIO Cloud Storage, (C) 2017-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,7 +18,6 @@ package cmd import ( "bufio" - "context" "crypto/hmac" "crypto/rand" "crypto/subtle" @@ -31,8 +30,6 @@ import ( "strconv" "strings" - "github.com/google/uuid" - "github.com/minio/minio-go/v6/pkg/encrypt" "github.com/minio/minio/cmd/crypto" "github.com/minio/minio/cmd/logger" sha256 "github.com/minio/sha256-simd" @@ -854,200 +851,3 @@ func deriveClientKey(clientKey [32]byte, bucket, object string) [32]byte { mac.Sum(key[:0]) return key } - -// set encryption options for pass through to backend in the case of gateway and UserDefined metadata -func getDefaultOpts(header http.Header, copySource bool, metadata map[string]string) (opts ObjectOptions, err error) { - var clientKey [32]byte - var sse encrypt.ServerSide - - opts = ObjectOptions{UserDefined: metadata} - if copySource { - if crypto.SSECopy.IsRequested(header) { - clientKey, err = crypto.SSECopy.ParseHTTP(header) - if err != nil { - return - } - if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil { - return - } - opts.ServerSideEncryption = encrypt.SSECopy(sse) - return - } - return - } - - if crypto.SSEC.IsRequested(header) { - clientKey, err = crypto.SSEC.ParseHTTP(header) - if err != nil { - return - } - if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil { - return - } - opts.ServerSideEncryption = sse - return - } - if crypto.S3.IsRequested(header) || (metadata != nil && crypto.S3.IsEncrypted(metadata)) { - opts.ServerSideEncryption = encrypt.NewSSE() - } - return -} - -// get ObjectOptions for GET calls from encryption headers -func getOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) { - var ( - encryption encrypt.ServerSide - opts ObjectOptions - ) - - var partNumber int - var err error - if pn := r.URL.Query().Get("partNumber"); pn != "" { - partNumber, err = strconv.Atoi(pn) - if err != nil { - return opts, err - } - if partNumber <= 0 { - return opts, errInvalidArgument - } - } - - vid := strings.TrimSpace(r.URL.Query().Get("versionId")) - if vid != "" && vid != nullVersionID { - _, err := uuid.Parse(vid) - if err != nil { - logger.LogIf(ctx, err) - return opts, VersionNotFound{ - Bucket: bucket, - Object: object, - VersionID: vid, - } - } - } - - if GlobalGatewaySSE.SSEC() && crypto.SSEC.IsRequested(r.Header) { - key, err := crypto.SSEC.ParseHTTP(r.Header) - if err != nil { - return opts, err - } - derivedKey := deriveClientKey(key, bucket, object) - encryption, err = encrypt.NewSSEC(derivedKey[:]) - logger.CriticalIf(ctx, err) - return ObjectOptions{ - ServerSideEncryption: encryption, - VersionID: vid, - PartNumber: partNumber, - }, nil - } - - // default case of passing encryption headers to backend - opts, err = getDefaultOpts(r.Header, false, nil) - if err != nil { - return opts, err - } - opts.PartNumber = partNumber - opts.VersionID = vid - return opts, nil -} - -func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts ObjectOptions, err error) { - versioned := globalBucketVersioningSys.Enabled(bucket) - opts, err = getOpts(ctx, r, bucket, object) - if err != nil { - return opts, err - } - opts.Versioned = versioned - return opts, nil -} - -// get ObjectOptions for PUT calls from encryption headers and metadata -func putOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) { - versioned := globalBucketVersioningSys.Enabled(bucket) - vid := strings.TrimSpace(r.URL.Query().Get("versionId")) - if vid != "" && vid != nullVersionID { - _, err := uuid.Parse(vid) - if err != nil { - logger.LogIf(ctx, err) - return opts, VersionNotFound{ - Bucket: bucket, - Object: object, - VersionID: vid, - } - } - } - - // In the case of multipart custom format, the metadata needs to be checked in addition to header to see if it - // is SSE-S3 encrypted, primarily because S3 protocol does not require SSE-S3 headers in PutObjectPart calls - if GlobalGatewaySSE.SSES3() && (crypto.S3.IsRequested(r.Header) || crypto.S3.IsEncrypted(metadata)) { - return ObjectOptions{ - ServerSideEncryption: encrypt.NewSSE(), - UserDefined: metadata, - VersionID: vid, - Versioned: versioned, - }, nil - } - if GlobalGatewaySSE.SSEC() && crypto.SSEC.IsRequested(r.Header) { - opts, err = getOpts(ctx, r, bucket, object) - opts.VersionID = vid - opts.Versioned = versioned - opts.UserDefined = metadata - return - } - if crypto.S3KMS.IsRequested(r.Header) { - keyID, context, err := crypto.S3KMS.ParseHTTP(r.Header) - if err != nil { - return ObjectOptions{}, err - } - sseKms, err := encrypt.NewSSEKMS(keyID, context) - if err != nil { - return ObjectOptions{}, err - } - return ObjectOptions{ - ServerSideEncryption: sseKms, - UserDefined: metadata, - VersionID: vid, - Versioned: versioned, - }, nil - } - // default case of passing encryption headers and UserDefined metadata to backend - opts, err = getDefaultOpts(r.Header, false, metadata) - if err != nil { - return opts, err - } - opts.VersionID = vid - opts.Versioned = versioned - return opts, nil -} - -// get ObjectOptions for Copy calls with encryption headers provided on the target side and source side metadata -func copyDstOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) { - return putOpts(ctx, r, bucket, object, metadata) -} - -// get ObjectOptions for Copy calls with encryption headers provided on the source side -func copySrcOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) { - var ( - ssec encrypt.ServerSide - opts ObjectOptions - ) - - if GlobalGatewaySSE.SSEC() && crypto.SSECopy.IsRequested(r.Header) { - key, err := crypto.SSECopy.ParseHTTP(r.Header) - if err != nil { - return opts, err - } - derivedKey := deriveClientKey(key, bucket, object) - ssec, err = encrypt.NewSSEC(derivedKey[:]) - if err != nil { - return opts, err - } - return ObjectOptions{ServerSideEncryption: encrypt.SSECopy(ssec)}, nil - } - - // default case of passing encryption headers to backend - opts, err := getDefaultOpts(r.Header, false, nil) - if err != nil { - return opts, err - } - return opts, nil -} diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go index 461e875a9..a154e861a 100644 --- a/cmd/erasure-multipart.go +++ b/cmd/erasure-multipart.go @@ -215,7 +215,11 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string, fi.DataDir = mustGetUUID() fi.ModTime = UTCNow() - fi.Metadata = opts.UserDefined + if opts.UserDefined != nil { + fi.Metadata = opts.UserDefined + } else { + fi.Metadata = make(map[string]string) + } uploadID := mustGetUUID() uploadIDPath := er.getUploadIDDir(bucket, object, uploadID) @@ -691,7 +695,10 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str // Save the final object size and modtime. fi.Size = objectSize - fi.ModTime = UTCNow() + fi.ModTime = opts.MTime + if opts.MTime.IsZero() { + fi.ModTime = UTCNow() + } // Save successfully calculated md5sum. fi.Metadata["etag"] = s3MD5 diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 4cc1cca6c..0eb8ad5d9 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -698,9 +698,6 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st }) } - // Save additional erasureMetadata. - modTime := UTCNow() - opts.UserDefined["etag"] = r.MD5CurrentHexString() // Guess content-type from the extension if possible. @@ -708,6 +705,11 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st opts.UserDefined["content-type"] = mimedb.TypeByExtension(path.Ext(object)) } + modTime := opts.MTime + if opts.MTime.IsZero() { + modTime = UTCNow() + } + // Fill all the necessary metadata. // Update `xl.meta` content on each disks. for index := range partsMetadata { diff --git a/cmd/erasure-zones.go b/cmd/erasure-zones.go index afa741d18..09b0498a2 100644 --- a/cmd/erasure-zones.go +++ b/cmd/erasure-zones.go @@ -1,5 +1,5 @@ /* - * MinIO Cloud Storage, (C) 2019 MinIO, Inc. + * MinIO Cloud Storage, (C) 2019,2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cmd/handler-utils.go b/cmd/handler-utils.go index c2a7643fd..8ff9cd040 100644 --- a/cmd/handler-utils.go +++ b/cmd/handler-utils.go @@ -126,9 +126,28 @@ func extractMetadata(ctx context.Context, r *http.Request) (metadata map[string] } // Set content-type to default value if it is not set. - if _, ok := metadata["content-type"]; !ok { - metadata["content-type"] = "application/octet-stream" + if _, ok := metadata[strings.ToLower(xhttp.ContentType)]; !ok { + metadata[strings.ToLower(xhttp.ContentType)] = "application/octet-stream" } + + if contentEncoding, ok := metadata[strings.ToLower(xhttp.ContentEncoding)]; ok { + contentEncoding = trimAwsChunkedContentEncoding(contentEncoding) + if contentEncoding != "" { + // Make sure to trim and save the content-encoding + // parameter for a streaming signature which is set + // to a custom value for example: "aws-chunked,gzip". + metadata[strings.ToLower(xhttp.ContentEncoding)] = contentEncoding + } else { + // Trimmed content encoding is empty when the header + // value is set to "aws-chunked" only. + + // Make sure to delete the content-encoding parameter + // for a streaming signature which is set to value + // for example: "aws-chunked" + delete(metadata, strings.ToLower(xhttp.ContentEncoding)) + } + } + // Success. return metadata, nil } diff --git a/cmd/http/headers.go b/cmd/http/headers.go index a9c40185b..d0103a603 100644 --- a/cmd/http/headers.go +++ b/cmd/http/headers.go @@ -38,6 +38,7 @@ const ( ContentDisposition = "Content-Disposition" Authorization = "Authorization" Action = "Action" + Range = "Range" ) // Non standard S3 HTTP response constants @@ -114,6 +115,18 @@ const ( // Server-Status MinIOServerStatus = "x-minio-server-status" - // Delete special flag + // Delete special flag to force delete a bucket MinIOForceDelete = "x-minio-force-delete" + + // Header indicates if the mtime should be preserved by client + MinIOSourceMTime = "x-minio-source-mtime" +) + +// Common http query params S3 API +const ( + VersionID = "versionId" + + PartNumber = "partNumber" + + UploadID = "uploadId" ) diff --git a/cmd/object-api-errors.go b/cmd/object-api-errors.go index 0adbb725d..c9d73112c 100644 --- a/cmd/object-api-errors.go +++ b/cmd/object-api-errors.go @@ -175,6 +175,17 @@ type GenericError struct { Bucket string Object string VersionID string + Err error +} + +// InvalidArgument incorrect input argument +type InvalidArgument GenericError + +func (e InvalidArgument) Error() string { + if e.Err != nil { + return "Invalid arguments provided for " + e.Bucket + "/" + e.Object + ": (" + e.Err.Error() + ")" + } + return "Invalid arguments provided for " + e.Bucket + "/" + e.Object } // BucketNotFound bucket does not exist. diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index 52f79eca2..5f530a877 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -20,6 +20,7 @@ import ( "context" "io" "net/http" + "time" "github.com/minio/minio-go/v6/pkg/encrypt" "github.com/minio/minio-go/v6/pkg/tags" @@ -36,11 +37,12 @@ type GetObjectInfoFn func(ctx context.Context, bucket, object string, opts Objec // ObjectOptions represents object options for ObjectLayer object operations type ObjectOptions struct { ServerSideEncryption encrypt.ServerSide - Versioned bool - VersionID string - UserDefined map[string]string - PartNumber int - CheckCopyPrecondFn CheckCopyPreconditionFn + Versioned bool // indicates if the bucket is versioned + VersionID string // Specifies the versionID which needs to be overwritten or read + MTime time.Time // Is only set in POST/PUT operations + UserDefined map[string]string // only set in case of POST/PUT operations + PartNumber int // only useful in case of GetObject/HeadObject + CheckCopyPrecondFn CheckCopyPreconditionFn // only set during CopyObject preconditional valuation } // BucketOptions represents bucket options for ObjectLayer bucket operations diff --git a/cmd/object-api-options.go b/cmd/object-api-options.go new file mode 100644 index 000000000..a9ebab05d --- /dev/null +++ b/cmd/object-api-options.go @@ -0,0 +1,242 @@ +/* + * MinIO Cloud Storage, (C) 2017-2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "context" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/minio/minio-go/v6/pkg/encrypt" + "github.com/minio/minio/cmd/crypto" + xhttp "github.com/minio/minio/cmd/http" + "github.com/minio/minio/cmd/logger" +) + +// set encryption options for pass through to backend in the case of gateway and UserDefined metadata +func getDefaultOpts(header http.Header, copySource bool, metadata map[string]string) (opts ObjectOptions, err error) { + var clientKey [32]byte + var sse encrypt.ServerSide + + opts = ObjectOptions{UserDefined: metadata} + if copySource { + if crypto.SSECopy.IsRequested(header) { + clientKey, err = crypto.SSECopy.ParseHTTP(header) + if err != nil { + return + } + if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil { + return + } + opts.ServerSideEncryption = encrypt.SSECopy(sse) + return + } + return + } + + if crypto.SSEC.IsRequested(header) { + clientKey, err = crypto.SSEC.ParseHTTP(header) + if err != nil { + return + } + if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil { + return + } + opts.ServerSideEncryption = sse + return + } + if crypto.S3.IsRequested(header) || (metadata != nil && crypto.S3.IsEncrypted(metadata)) { + opts.ServerSideEncryption = encrypt.NewSSE() + } + return +} + +// get ObjectOptions for GET calls from encryption headers +func getOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) { + var ( + encryption encrypt.ServerSide + opts ObjectOptions + ) + + var partNumber int + var err error + if pn := r.URL.Query().Get(xhttp.PartNumber); pn != "" { + partNumber, err = strconv.Atoi(pn) + if err != nil { + return opts, err + } + if partNumber <= 0 { + return opts, errInvalidArgument + } + } + + vid := strings.TrimSpace(r.URL.Query().Get(xhttp.VersionID)) + if vid != "" && vid != nullVersionID { + _, err := uuid.Parse(vid) + if err != nil { + logger.LogIf(ctx, err) + return opts, VersionNotFound{ + Bucket: bucket, + Object: object, + VersionID: vid, + } + } + } + + if GlobalGatewaySSE.SSEC() && crypto.SSEC.IsRequested(r.Header) { + key, err := crypto.SSEC.ParseHTTP(r.Header) + if err != nil { + return opts, err + } + derivedKey := deriveClientKey(key, bucket, object) + encryption, err = encrypt.NewSSEC(derivedKey[:]) + logger.CriticalIf(ctx, err) + return ObjectOptions{ + ServerSideEncryption: encryption, + VersionID: vid, + PartNumber: partNumber, + }, nil + } + + // default case of passing encryption headers to backend + opts, err = getDefaultOpts(r.Header, false, nil) + if err != nil { + return opts, err + } + opts.PartNumber = partNumber + opts.VersionID = vid + return opts, nil +} + +func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts ObjectOptions, err error) { + versioned := globalBucketVersioningSys.Enabled(bucket) + opts, err = getOpts(ctx, r, bucket, object) + if err != nil { + return opts, err + } + opts.Versioned = versioned + return opts, nil +} + +// get ObjectOptions for PUT calls from encryption headers and metadata +func putOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) { + versioned := globalBucketVersioningSys.Enabled(bucket) + vid := strings.TrimSpace(r.URL.Query().Get(xhttp.VersionID)) + if vid != "" && vid != nullVersionID { + _, err := uuid.Parse(vid) + if err != nil { + logger.LogIf(ctx, err) + return opts, VersionNotFound{ + Bucket: bucket, + Object: object, + VersionID: vid, + } + } + } + mtime := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceMTime)) + if mtime != "" { + opts.MTime, err = time.Parse(time.RFC3339, mtime) + if err != nil { + return opts, InvalidArgument{ + Bucket: bucket, + Object: object, + Err: fmt.Errorf("Unable to parse %s, failed with %w", xhttp.MinIOSourceMTime, err), + } + } + } else { + opts.MTime = UTCNow() + } + + // In the case of multipart custom format, the metadata needs to be checked in addition to header to see if it + // is SSE-S3 encrypted, primarily because S3 protocol does not require SSE-S3 headers in PutObjectPart calls + if GlobalGatewaySSE.SSES3() && (crypto.S3.IsRequested(r.Header) || crypto.S3.IsEncrypted(metadata)) { + return ObjectOptions{ + ServerSideEncryption: encrypt.NewSSE(), + UserDefined: metadata, + VersionID: vid, + Versioned: versioned, + }, nil + } + if GlobalGatewaySSE.SSEC() && crypto.SSEC.IsRequested(r.Header) { + opts, err = getOpts(ctx, r, bucket, object) + opts.VersionID = vid + opts.Versioned = versioned + opts.UserDefined = metadata + return + } + if crypto.S3KMS.IsRequested(r.Header) { + keyID, context, err := crypto.S3KMS.ParseHTTP(r.Header) + if err != nil { + return ObjectOptions{}, err + } + sseKms, err := encrypt.NewSSEKMS(keyID, context) + if err != nil { + return ObjectOptions{}, err + } + return ObjectOptions{ + ServerSideEncryption: sseKms, + UserDefined: metadata, + VersionID: vid, + Versioned: versioned, + }, nil + } + // default case of passing encryption headers and UserDefined metadata to backend + opts, err = getDefaultOpts(r.Header, false, metadata) + if err != nil { + return opts, err + } + opts.VersionID = vid + opts.Versioned = versioned + return opts, nil +} + +// get ObjectOptions for Copy calls with encryption headers provided on the target side and source side metadata +func copyDstOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) { + return putOpts(ctx, r, bucket, object, metadata) +} + +// get ObjectOptions for Copy calls with encryption headers provided on the source side +func copySrcOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) { + var ( + ssec encrypt.ServerSide + opts ObjectOptions + ) + + if GlobalGatewaySSE.SSEC() && crypto.SSECopy.IsRequested(r.Header) { + key, err := crypto.SSECopy.ParseHTTP(r.Header) + if err != nil { + return opts, err + } + derivedKey := deriveClientKey(key, bucket, object) + ssec, err = encrypt.NewSSEC(derivedKey[:]) + if err != nil { + return opts, err + } + return ObjectOptions{ServerSideEncryption: encrypt.SSECopy(ssec)}, nil + } + + // default case of passing encryption headers to backend + opts, err := getDefaultOpts(r.Header, false, nil) + if err != nil { + return opts, err + } + return opts, nil +} diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index f02ca105b..4f3034764 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -159,7 +159,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r } // Get request range. - rangeHeader := r.Header.Get("Range") + rangeHeader := r.Header.Get(xhttp.Range) if rangeHeader != "" { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrUnsupportedRangeHeader), r.URL, guessIsBrowserReq(r)) return @@ -370,7 +370,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req // Get request range. var rs *HTTPRangeSpec - rangeHeader := r.Header.Get("Range") + rangeHeader := r.Header.Get(xhttp.Range) if rangeHeader != "" { if rs, err = parseRequestRangeSpec(rangeHeader); err != nil { // Handle only errInvalidRange. Ignore other @@ -552,7 +552,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re // Get request range. var rs *HTTPRangeSpec - rangeHeader := r.Header.Get("Range") + rangeHeader := r.Header.Get(xhttp.Range) if rangeHeader != "" { if rs, err = parseRequestRangeSpec(rangeHeader); err != nil { // Handle only errInvalidRange. Ignore other @@ -810,7 +810,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re cpSrcPath := r.Header.Get(xhttp.AmzCopySource) var vid string if u, err := url.Parse(cpSrcPath); err == nil { - vid = strings.TrimSpace(u.Query().Get("versionId")) + vid = strings.TrimSpace(u.Query().Get(xhttp.VersionID)) // Note that url.Parse does the unescaping cpSrcPath = u.Path } @@ -1355,26 +1355,6 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req metadata[xhttp.AmzObjectTagging] = objTags } - if rAuthType == authTypeStreamingSigned { - if contentEncoding, ok := metadata["content-encoding"]; ok { - contentEncoding = trimAwsChunkedContentEncoding(contentEncoding) - if contentEncoding != "" { - // Make sure to trim and save the content-encoding - // parameter for a streaming signature which is set - // to a custom value for example: "aws-chunked,gzip". - metadata["content-encoding"] = contentEncoding - } else { - // Trimmed content encoding is empty when the header - // value is set to "aws-chunked" only. - - // Make sure to delete the content-encoding parameter - // for a streaming signature which is set to value - // for example: "aws-chunked" - delete(metadata, "content-encoding") - } - } - } - var ( md5hex = hex.EncodeToString(md5Bytes) sha256hex = "" @@ -1729,7 +1709,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt cpSrcPath := r.Header.Get(xhttp.AmzCopySource) var vid string if u, err := url.Parse(cpSrcPath); err == nil { - vid = strings.TrimSpace(u.Query().Get("versionId")) + vid = strings.TrimSpace(u.Query().Get(xhttp.VersionID)) // Note that url.Parse does the unescaping cpSrcPath = u.Path } @@ -1761,8 +1741,8 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt return } - uploadID := r.URL.Query().Get("uploadId") - partIDString := r.URL.Query().Get("partNumber") + uploadID := r.URL.Query().Get(xhttp.UploadID) + partIDString := r.URL.Query().Get(xhttp.PartNumber) partID, err := strconv.Atoi(partIDString) if err != nil { @@ -2071,8 +2051,8 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http return } - uploadID := r.URL.Query().Get("uploadId") - partIDString := r.URL.Query().Get("partNumber") + uploadID := r.URL.Query().Get(xhttp.UploadID) + partIDString := r.URL.Query().Get(xhttp.PartNumber) partID, err := strconv.Atoi(partIDString) if err != nil {