From eb9172eecb985da13a814533490f274c5a5e83fa Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Tue, 5 Jan 2021 20:08:35 -0800 Subject: [PATCH] Allow Compression + encryption (#11103) --- cmd/config/compress/compress.go | 29 ++- cmd/encryption-v1.go | 4 +- cmd/object-api-getobject_test.go | 2 +- cmd/object-api-multipart_test.go | 6 +- cmd/object-api-putobject_test.go | 2 +- cmd/object-api-utils.go | 199 ++++++++++-------- cmd/object-api-utils_test.go | 14 +- cmd/object-handlers.go | 70 +++--- cmd/object-handlers_test.go | 95 +++++---- cmd/object_api_suite_test.go | 109 +++++++--- cmd/test-utils_test.go | 8 + docs/bucket/versioning/DESIGN.md | 2 +- .../{xl-meta-to-json.go => xl-meta.go} | 9 +- docs/compression/README.md | 84 ++++++-- 14 files changed, 399 insertions(+), 234 deletions(-) rename docs/bucket/versioning/{xl-meta-to-json.go => xl-meta.go} (94%) diff --git a/cmd/config/compress/compress.go b/cmd/config/compress/compress.go index a72dfc9a6..18a5ea4fb 100644 --- a/cmd/config/compress/compress.go +++ b/cmd/config/compress/compress.go @@ -26,19 +26,22 @@ import ( // Config represents the compression settings. type Config struct { - Enabled bool `json:"enabled"` - Extensions []string `json:"extensions"` - MimeTypes []string `json:"mime-types"` + Enabled bool `json:"enabled"` + AllowEncrypted bool `json:"allow_encryption"` + Extensions []string `json:"extensions"` + MimeTypes []string `json:"mime-types"` } // Compression environment variables const ( - Extensions = "extensions" - MimeTypes = "mime_types" + Extensions = "extensions" + AllowEncrypted = "allow_encryption" + MimeTypes = "mime_types" - EnvCompressState = "MINIO_COMPRESS_ENABLE" - EnvCompressExtensions = "MINIO_COMPRESS_EXTENSIONS" - EnvCompressMimeTypes = "MINIO_COMPRESS_MIME_TYPES" + EnvCompressState = "MINIO_COMPRESS_ENABLE" + EnvCompressAllowEncryption = "MINIO_COMPRESS_ALLOW_ENCRYPTION" + EnvCompressExtensions = "MINIO_COMPRESS_EXTENSIONS" + EnvCompressMimeTypes = "MINIO_COMPRESS_MIME_TYPES" // Include-list for compression. DefaultExtensions = ".txt,.log,.csv,.json,.tar,.xml,.bin" @@ -52,6 +55,10 @@ var ( Key: config.Enable, Value: config.EnableOff, }, + config.KV{ + Key: AllowEncrypted, + Value: config.EnableOff, + }, config.KV{ Key: Extensions, Value: DefaultExtensions, @@ -101,6 +108,12 @@ func LookupConfig(kvs config.KVS) (Config, error) { return cfg, nil } + allowEnc := env.Get(EnvCompressAllowEncryption, kvs.Get(AllowEncrypted)) + cfg.AllowEncrypted, err = config.ParseBool(allowEnc) + if err != nil { + return cfg, err + } + compressExtensions := env.Get(EnvCompressExtensions, kvs.Get(Extensions)) compressMimeTypes := env.Get(EnvCompressMimeTypes, kvs.Get(MimeTypes)) compressMimeTypesLegacy := env.Get(EnvCompressMimeTypesLegacy, kvs.Get(MimeTypes)) diff --git a/cmd/encryption-v1.go b/cmd/encryption-v1.go index def150262..7cf19bc51 100644 --- a/cmd/encryption-v1.go +++ b/cmd/encryption-v1.go @@ -353,9 +353,7 @@ func newDecryptReaderWithObjectKey(client io.Reader, objectEncryptionKey []byte, // DecryptBlocksRequestR - same as DecryptBlocksRequest but with a // reader -func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset, - length int64, seqNumber uint32, partStart int, oi ObjectInfo, copySource bool) ( - io.Reader, error) { +func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, seqNumber uint32, partStart int, oi ObjectInfo, copySource bool) (io.Reader, error) { bucket, object := oi.Bucket, oi.Name // Single part case diff --git a/cmd/object-api-getobject_test.go b/cmd/object-api-getobject_test.go index ecb10a711..03d9f4ee6 100644 --- a/cmd/object-api-getobject_test.go +++ b/cmd/object-api-getobject_test.go @@ -31,7 +31,7 @@ import ( // Wrapper for calling GetObject tests for both Erasure multiple disks and single node setup. func TestGetObject(t *testing.T) { - ExecObjectLayerTest(t, testGetObject) + ExecExtendedObjectLayerTest(t, testGetObject) } // ObjectLayer.GetObject is called with series of cases for valid and erroneous inputs and the result is validated. diff --git a/cmd/object-api-multipart_test.go b/cmd/object-api-multipart_test.go index 854962104..a65bfd425 100644 --- a/cmd/object-api-multipart_test.go +++ b/cmd/object-api-multipart_test.go @@ -157,7 +157,7 @@ func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t TestE // Wrapper for calling PutObjectPart tests for both Erasure multiple disks and single node setup. func TestObjectAPIPutObjectPart(t *testing.T) { - ExecObjectLayerTest(t, testObjectAPIPutObjectPart) + ExecExtendedObjectLayerTest(t, testObjectAPIPutObjectPart) } // Tests validate correctness of PutObjectPart. @@ -289,7 +289,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH // Wrapper for calling TestListMultipartUploads tests for both Erasure multiple disks and single node setup. func TestListMultipartUploads(t *testing.T) { - ExecObjectLayerTest(t, testListMultipartUploads) + ExecExtendedObjectLayerTest(t, testListMultipartUploads) } // testListMultipartUploads - Tests validate listing of multipart uploads. @@ -1643,7 +1643,7 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler) // Test for validating complete Multipart upload. func TestObjectCompleteMultipartUpload(t *testing.T) { - ExecObjectLayerTest(t, testObjectCompleteMultipartUpload) + ExecExtendedObjectLayerTest(t, testObjectCompleteMultipartUpload) } // Tests validate CompleteMultipart functionality. diff --git a/cmd/object-api-putobject_test.go b/cmd/object-api-putobject_test.go index 0d131afb4..b5a7cfb18 100644 --- a/cmd/object-api-putobject_test.go +++ b/cmd/object-api-putobject_test.go @@ -37,7 +37,7 @@ func md5Header(data []byte) map[string]string { // Wrapper for calling PutObject tests for both Erasure multiple disks and single node setup. func TestObjectAPIPutObjectSingle(t *testing.T) { - ExecObjectLayerTest(t, testObjectAPIPutObject) + ExecExtendedObjectLayerTest(t, testObjectAPIPutObject) } // Tests validate correctness of PutObject. diff --git a/cmd/object-api-utils.go b/cmd/object-api-utils.go index 5108c44b5..f93d0dcbc 100644 --- a/cmd/object-api-utils.go +++ b/cmd/object-api-utils.go @@ -394,9 +394,6 @@ func (o ObjectInfo) IsCompressedOK() (bool, error) { if !ok { return false, nil } - if crypto.IsEncrypted(o.UserDefined) { - return true, fmt.Errorf("compression %q and encryption enabled on same object", scheme) - } switch scheme { case compressionAlgorithmV1, compressionAlgorithmV2: return true, nil @@ -415,9 +412,6 @@ func (o ObjectInfo) GetActualETag(h http.Header) string { // GetActualSize - returns the actual size of the stored object func (o ObjectInfo) GetActualSize() (int64, error) { - if crypto.IsEncrypted(o.UserDefined) { - return o.DecryptedSize() - } if o.IsCompressed() { sizeStr, ok := o.UserDefined[ReservedMetadataPrefix+"actual-size"] if !ok { @@ -429,6 +423,10 @@ func (o ObjectInfo) GetActualSize() (int64, error) { } return size, nil } + if crypto.IsEncrypted(o.UserDefined) { + return o.DecryptedSize() + } + return o.Size, nil } @@ -441,7 +439,7 @@ func isCompressible(header http.Header, object string) bool { globalCompressConfigMu.Unlock() _, ok := crypto.IsRequested(header) - if !cfg.Enabled || ok || excludeForCompression(header, object, cfg) { + if !cfg.Enabled || (ok && !cfg.AllowEncrypted) || excludeForCompression(header, object, cfg) { return false } return true @@ -461,16 +459,15 @@ func excludeForCompression(header http.Header, object string, cfg compress.Confi } // Filter compression includes. - if len(cfg.Extensions) == 0 || len(cfg.MimeTypes) == 0 { - return false + exclude := len(cfg.Extensions) > 0 || len(cfg.MimeTypes) > 0 + if len(cfg.Extensions) > 0 && hasStringSuffixInSlice(objStr, cfg.Extensions) { + exclude = false } - extensions := cfg.Extensions - mimeTypes := cfg.MimeTypes - if hasStringSuffixInSlice(objStr, extensions) || hasPattern(mimeTypes, contentType) { - return false + if len(cfg.MimeTypes) > 0 && hasPattern(cfg.MimeTypes, contentType) { + exclude = false } - return true + return exclude } // Utility which returns if a string is present in the list. @@ -520,21 +517,29 @@ func partNumberToRangeSpec(oi ObjectInfo, partNumber int) *HTTPRangeSpec { } // Returns the compressed offset which should be skipped. -func getCompressedOffsets(objectInfo ObjectInfo, offset int64) (int64, int64) { - var compressedOffset int64 +// If encrypted offsets are adjusted for encrypted block headers/trailers. +// Since de-compression is after decryption encryption overhead is only added to compressedOffset. +func getCompressedOffsets(objectInfo ObjectInfo, offset int64) (compressedOffset int64, partSkip int64) { var skipLength int64 var cumulativeActualSize int64 + var firstPartIdx int if len(objectInfo.Parts) > 0 { - for _, part := range objectInfo.Parts { + for i, part := range objectInfo.Parts { cumulativeActualSize += part.ActualSize if cumulativeActualSize <= offset { compressedOffset += part.Size } else { + firstPartIdx = i skipLength = cumulativeActualSize - part.ActualSize break } } } + if isEncryptedMultipart(objectInfo) && firstPartIdx > 0 { + off, _, _, _, _, err := objectInfo.GetDecryptedRange(partNumberToRangeSpec(objectInfo, firstPartIdx)) + logger.LogIf(context.Background(), err) + compressedOffset += off + } return compressedOffset, offset - skipLength } @@ -604,9 +609,92 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, cl isEncrypted = false } var skipLen int64 - // Calculate range to read (different for - // e.g. encrypted/compressed objects) + // Calculate range to read (different for encrypted/compressed objects) switch { + case isCompressed: + // If compressed, we start from the beginning of the part. + // Read the decompressed size from the meta.json. + actualSize, err := oi.GetActualSize() + if err != nil { + return nil, 0, 0, err + } + off, length = int64(0), oi.Size + decOff, decLength := int64(0), actualSize + if rs != nil { + off, length, err = rs.GetOffsetLength(actualSize) + if err != nil { + return nil, 0, 0, err + } + // In case of range based queries on multiparts, the offset and length are reduced. + off, decOff = getCompressedOffsets(oi, off) + decLength = length + length = oi.Size - off + + // For negative length we read everything. + if decLength < 0 { + decLength = actualSize - decOff + } + + // Reply back invalid range if the input offset and length fall out of range. + if decOff > actualSize || decOff+decLength > actualSize { + return nil, 0, 0, errInvalidRange + } + } + fn = func(inputReader io.Reader, h http.Header, pcfn CheckPreconditionFn, cFns ...func()) (r *GetObjectReader, err error) { + cFns = append(cleanUpFns, cFns...) + if opts.CheckPrecondFn != nil && opts.CheckPrecondFn(oi) { + // Call the cleanup funcs + for i := len(cFns) - 1; i >= 0; i-- { + cFns[i]() + } + return nil, PreConditionFailed{} + } + if isEncrypted { + copySource := h.Get(xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm) != "" + // Attach decrypter on inputReader + inputReader, err = DecryptBlocksRequestR(inputReader, h, 0, opts.PartNumber, oi, copySource) + if err != nil { + // Call the cleanup funcs + for i := len(cFns) - 1; i >= 0; i-- { + cFns[i]() + } + return nil, err + } + } + // Decompression reader. + s2Reader := s2.NewReader(inputReader) + // Apply the skipLen and limit on the decompressed stream. + err = s2Reader.Skip(decOff) + if err != nil { + // Call the cleanup funcs + for i := len(cFns) - 1; i >= 0; i-- { + cFns[i]() + } + return nil, err + } + + decReader := io.LimitReader(s2Reader, decLength) + if decLength > compReadAheadSize { + rah, err := readahead.NewReaderSize(decReader, compReadAheadBuffers, compReadAheadBufSize) + if err == nil { + decReader = rah + cFns = append(cFns, func() { + rah.Close() + }) + } + } + oi.Size = decLength + + // Assemble the GetObjectReader + r = &GetObjectReader{ + ObjInfo: oi, + pReader: decReader, + cleanUpFns: cFns, + opts: opts, + } + return r, nil + } + case isEncrypted: var seqNumber uint32 var partStart int @@ -635,8 +723,7 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, cl cFns = append(cleanUpFns, cFns...) // Attach decrypter on inputReader var decReader io.Reader - decReader, err = DecryptBlocksRequestR(inputReader, h, - off, length, seqNumber, partStart, oi, copySource) + decReader, err = DecryptBlocksRequestR(inputReader, h, seqNumber, partStart, oi, copySource) if err != nil { // Call the cleanup funcs for i := len(cFns) - 1; i >= 0; i-- { @@ -659,76 +746,6 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, cl // decrypted stream decReader = io.LimitReader(ioutil.NewSkipReader(decReader, skipLen), decRangeLength) - // Assemble the GetObjectReader - r = &GetObjectReader{ - ObjInfo: oi, - pReader: decReader, - cleanUpFns: cFns, - opts: opts, - } - return r, nil - } - case isCompressed: - // Read the decompressed size from the meta.json. - actualSize, err := oi.GetActualSize() - if err != nil { - return nil, 0, 0, err - } - off, length = int64(0), oi.Size - decOff, decLength := int64(0), actualSize - if rs != nil { - off, length, err = rs.GetOffsetLength(actualSize) - if err != nil { - return nil, 0, 0, err - } - // In case of range based queries on multiparts, the offset and length are reduced. - off, decOff = getCompressedOffsets(oi, off) - decLength = length - length = oi.Size - off - - // For negative length we read everything. - if decLength < 0 { - decLength = actualSize - decOff - } - - // Reply back invalid range if the input offset and length fall out of range. - if decOff > actualSize || decOff+decLength > actualSize { - return nil, 0, 0, errInvalidRange - } - } - fn = func(inputReader io.Reader, _ http.Header, pcfn CheckPreconditionFn, cFns ...func()) (r *GetObjectReader, err error) { - cFns = append(cleanUpFns, cFns...) - if opts.CheckPrecondFn != nil && opts.CheckPrecondFn(oi) { - // Call the cleanup funcs - for i := len(cFns) - 1; i >= 0; i-- { - cFns[i]() - } - return nil, PreConditionFailed{} - } - // Decompression reader. - s2Reader := s2.NewReader(inputReader) - // Apply the skipLen and limit on the decompressed stream. - err = s2Reader.Skip(decOff) - if err != nil { - // Call the cleanup funcs - for i := len(cFns) - 1; i >= 0; i-- { - cFns[i]() - } - return nil, err - } - - decReader := io.LimitReader(s2Reader, decLength) - if decLength > compReadAheadSize { - rah, err := readahead.NewReaderSize(decReader, compReadAheadBuffers, compReadAheadBufSize) - if err == nil { - decReader = rah - cFns = append(cFns, func() { - rah.Close() - }) - } - } - oi.Size = decLength - // Assemble the GetObjectReader r = &GetObjectReader{ ObjInfo: oi, diff --git a/cmd/object-api-utils_test.go b/cmd/object-api-utils_test.go index 781288429..a98c3fa70 100644 --- a/cmd/object-api-utils_test.go +++ b/cmd/object-api-utils_test.go @@ -317,7 +317,7 @@ func TestIsCompressed(t *testing.T) { result bool err bool }{ - { + 0: { objInfo: ObjectInfo{ UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV1, "content-type": "application/octet-stream", @@ -325,7 +325,7 @@ func TestIsCompressed(t *testing.T) { }, result: true, }, - { + 1: { objInfo: ObjectInfo{ UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV2, "content-type": "application/octet-stream", @@ -333,7 +333,7 @@ func TestIsCompressed(t *testing.T) { }, result: true, }, - { + 2: { objInfo: ObjectInfo{ UserDefined: map[string]string{"X-Minio-Internal-compression": "unknown/compression/type", "content-type": "application/octet-stream", @@ -342,7 +342,7 @@ func TestIsCompressed(t *testing.T) { result: true, err: true, }, - { + 3: { objInfo: ObjectInfo{ UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV2, "content-type": "application/octet-stream", @@ -351,9 +351,9 @@ func TestIsCompressed(t *testing.T) { }, }, result: true, - err: true, + err: false, }, - { + 4: { objInfo: ObjectInfo{ UserDefined: map[string]string{"X-Minio-Internal-XYZ": "klauspost/compress/s2", "content-type": "application/octet-stream", @@ -361,7 +361,7 @@ func TestIsCompressed(t *testing.T) { }, result: false, }, - { + 5: { objInfo: ObjectInfo{ UserDefined: map[string]string{"content-type": "application/octet-stream", "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"}, diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 498073dcd..26e15fbf4 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -977,18 +977,14 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re } var reader io.Reader - var length = srcInfo.Size - // Set the actual size to the decrypted size if encrypted. - actualSize := srcInfo.Size - if crypto.IsEncrypted(srcInfo.UserDefined) { - actualSize, err = srcInfo.DecryptedSize() - if err != nil { - writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) - return - } - length = actualSize + // Set the actual size to the compressed/decrypted size if encrypted. + actualSize, err := srcInfo.GetActualSize() + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) + return } + length := actualSize if !cpSrcDstSame { if err := enforceBucketQuota(ctx, dstBucket, actualSize); err != nil { @@ -1000,8 +996,10 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re var compressMetadata map[string]string // No need to compress for remote etcd calls // Pass the decompressed stream to such calls. - isCompressed := objectAPI.IsCompressionSupported() && isCompressible(r.Header, srcObject) && !isRemoteCopyRequired(ctx, srcBucket, dstBucket, objectAPI) - if isCompressed { + isDstCompressed := objectAPI.IsCompressionSupported() && + isCompressible(r.Header, srcObject) && + !isRemoteCopyRequired(ctx, srcBucket, dstBucket, objectAPI) + if isDstCompressed { compressMetadata = make(map[string]string, 2) // Preserving the compression metadata. compressMetadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2 @@ -1034,7 +1032,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re _, objectEncryption := crypto.IsRequested(r.Header) objectEncryption = objectEncryption || crypto.IsSourceEncrypted(srcInfo.UserDefined) var encMetadata = make(map[string]string) - if objectAPI.IsEncryptionSupported() && !isCompressed { + if objectAPI.IsEncryptionSupported() { // Encryption parameters not applicable for this object. if !crypto.IsEncrypted(srcInfo.UserDefined) && crypto.SSECopy.IsRequested(r.Header) { writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL, guessIsBrowserReq(r)) @@ -1105,8 +1103,10 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re var targetSize int64 switch { + case isDstCompressed: + targetSize = -1 case !isSourceEncrypted && !isTargetEncrypted: - targetSize = srcInfo.Size + targetSize, _ = srcInfo.GetActualSize() case isSourceEncrypted && isTargetEncrypted: objInfo := ObjectInfo{Size: actualSize} targetSize = objInfo.EncryptedSize() @@ -1131,7 +1131,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re } // do not try to verify encrypted content - srcInfo.Reader, err = hash.NewReader(reader, targetSize, "", "", targetSize, globalCLIContext.StrictS3Compat) + srcInfo.Reader, err = hash.NewReader(reader, targetSize, "", "", actualSize, globalCLIContext.StrictS3Compat) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) return @@ -1541,10 +1541,15 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) return } - info := ObjectInfo{Size: size} + + wantSize := int64(-1) + if size >= 0 { + info := ObjectInfo{Size: size} + wantSize = info.EncryptedSize() + } // do not try to verify encrypted content - hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", size, globalCLIContext.StrictS3Compat) + hashReader, err = hash.NewReader(reader, wantSize, "", "", actualSize, globalCLIContext.StrictS3Compat) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) return @@ -1564,10 +1569,6 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req } switch { - case objInfo.IsCompressed(): - if !strings.HasSuffix(objInfo.ETag, "-1") { - objInfo.ETag = objInfo.ETag + "-1" - } case crypto.IsEncrypted(objInfo.UserDefined): switch { case crypto.S3.IsEncrypted(objInfo.UserDefined): @@ -1581,6 +1582,10 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req objInfo.ETag = objInfo.ETag[len(objInfo.ETag)-32:] } } + case objInfo.IsCompressed(): + if !strings.HasSuffix(objInfo.ETag, "-1") { + objInfo.ETag = objInfo.ETag + "-1" + } } if mustReplicate(ctx, r, bucket, object, metadata, "") { globalReplicationState.queueReplicaTask(objInfo) @@ -1892,7 +1897,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt actualPartSize := srcInfo.Size if crypto.IsEncrypted(srcInfo.UserDefined) { - actualPartSize, err = srcInfo.DecryptedSize() + actualPartSize, err = srcInfo.GetActualSize() if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) return @@ -1991,7 +1996,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt isEncrypted := crypto.IsEncrypted(mi.UserDefined) var objectEncryptionKey crypto.ObjectKey - if objectAPI.IsEncryptionSupported() && !isCompressed && isEncrypted { + if objectAPI.IsEncryptionSupported() && isEncrypted { if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrSSEMultipartEncrypted), r.URL, guessIsBrowserReq(r)) return @@ -2022,8 +2027,13 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt return } - info := ObjectInfo{Size: length} - srcInfo.Reader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", length, globalCLIContext.StrictS3Compat) + wantSize := int64(-1) + if length >= 0 { + info := ObjectInfo{Size: length} + wantSize = info.EncryptedSize() + } + + srcInfo.Reader, err = hash.NewReader(reader, wantSize, "", "", actualPartSize, globalCLIContext.StrictS3Compat) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) return @@ -2226,7 +2236,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http isEncrypted := crypto.IsEncrypted(mi.UserDefined) var objectEncryptionKey crypto.ObjectKey - if objectAPI.IsEncryptionSupported() && !isCompressed && isEncrypted { + if objectAPI.IsEncryptionSupported() && isEncrypted { if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrSSEMultipartEncrypted), r.URL, guessIsBrowserReq(r)) return @@ -2267,9 +2277,13 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) return } - info := ObjectInfo{Size: size} + wantSize := int64(-1) + if size >= 0 { + info := ObjectInfo{Size: size} + wantSize = info.EncryptedSize() + } // do not try to verify encrypted content - hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", size, globalCLIContext.StrictS3Compat) + hashReader, err = hash.NewReader(reader, wantSize, "", "", actualSize, globalCLIContext.StrictS3Compat) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) return diff --git a/cmd/object-handlers_test.go b/cmd/object-handlers_test.go index 4c111dbb2..1d862d3fa 100644 --- a/cmd/object-handlers_test.go +++ b/cmd/object-handlers_test.go @@ -327,7 +327,7 @@ func TestAPIGetObjectHandler(t *testing.T) { defer func() { globalPolicySys = nil }() defer DetectTestLeak(t)() - ExecObjectLayerAPITest(t, testAPIGetObjectHandler, []string{"GetObject"}) + ExecExtendedObjectLayerAPITest(t, testAPIGetObjectHandler, []string{"GetObject"}) } func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, @@ -651,7 +651,7 @@ func TestAPIGetObjectWithMPHandler(t *testing.T) { defer func() { globalPolicySys = nil }() defer DetectTestLeak(t)() - ExecObjectLayerAPITest(t, testAPIGetObjectWithMPHandler, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject"}) + ExecExtendedObjectLayerAPITest(t, testAPIGetObjectWithMPHandler, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject"}) } func testAPIGetObjectWithMPHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, @@ -849,7 +849,7 @@ func TestAPIGetObjectWithPartNumberHandler(t *testing.T) { defer func() { globalPolicySys = nil }() defer DetectTestLeak(t)() - ExecObjectLayerAPITest(t, testAPIGetObjectWithPartNumberHandler, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject"}) + ExecExtendedObjectLayerAPITest(t, testAPIGetObjectWithPartNumberHandler, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject"}) } func testAPIGetObjectWithPartNumberHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, @@ -971,7 +971,7 @@ func testAPIGetObjectWithPartNumberHandler(obj ObjectLayer, instanceType, bucket // Wrapper for calling PutObject API handler tests using streaming signature v4 for both Erasure multiple disks and FS single drive setup. func TestAPIPutObjectStreamSigV4Handler(t *testing.T) { defer DetectTestLeak(t)() - ExecObjectLayerAPITest(t, testAPIPutObjectStreamSigV4Handler, []string{"PutObject"}) + ExecExtendedObjectLayerAPITest(t, testAPIPutObjectStreamSigV4Handler, []string{"PutObject"}) } func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, @@ -1289,7 +1289,7 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam // Wrapper for calling PutObject API handler tests for both Erasure multiple disks and FS single drive setup. func TestAPIPutObjectHandler(t *testing.T) { defer DetectTestLeak(t)() - ExecObjectLayerAPITest(t, testAPIPutObjectHandler, []string{"PutObject"}) + ExecExtendedObjectLayerAPITest(t, testAPIPutObjectHandler, []string{"PutObject"}) } func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, @@ -1538,7 +1538,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a // expected. func TestAPICopyObjectPartHandlerSanity(t *testing.T) { defer DetectTestLeak(t)() - ExecObjectLayerAPITest(t, testAPICopyObjectPartHandlerSanity, []string{"CopyObjectPart"}) + ExecExtendedObjectLayerAPITest(t, testAPICopyObjectPartHandlerSanity, []string{"CopyObjectPart"}) } func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, @@ -1649,7 +1649,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam // Wrapper for calling Copy Object Part API handler tests for both Erasure multiple disks and single node setup. func TestAPICopyObjectPartHandler(t *testing.T) { defer DetectTestLeak(t)() - ExecObjectLayerAPITest(t, testAPICopyObjectPartHandler, []string{"CopyObjectPart"}) + ExecExtendedObjectLayerAPITest(t, testAPICopyObjectPartHandler, []string{"CopyObjectPart"}) } func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, @@ -1965,7 +1965,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri // Wrapper for calling Copy Object API handler tests for both Erasure multiple disks and single node setup. func TestAPICopyObjectHandler(t *testing.T) { defer DetectTestLeak(t)() - ExecObjectLayerAPITest(t, testAPICopyObjectHandler, []string{"CopyObject"}) + ExecExtendedObjectLayerAPITest(t, testAPICopyObjectHandler, []string{"CopyObject"}) } func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, @@ -2044,8 +2044,11 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, // expected output. expectedRespStatus int }{ + 0: { + expectedRespStatus: http.StatusMethodNotAllowed, + }, // Test case - 1, copy metadata from newObject1, ignore request headers. - { + 1: { bucketName: bucketName, newObjectName: "newObject1", copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), @@ -2059,7 +2062,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, // Test case - 2. // Test case with invalid source object. - { + 2: { bucketName: bucketName, newObjectName: "newObject1", copySourceHeader: url.QueryEscape(SlashSeparator), @@ -2071,7 +2074,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, // Test case - 3. // Test case with new object name is same as object to be copied. - { + 3: { bucketName: bucketName, newObjectName: objectName, copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), @@ -2084,7 +2087,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, // Test case - 4. // Test case with new object name is same as object to be copied. // But source copy is without leading slash - { + 4: { bucketName: bucketName, newObjectName: objectName, copySourceHeader: url.QueryEscape(bucketName + SlashSeparator + objectName), @@ -2097,7 +2100,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, // Test case - 5. // Test case with new object name is same as object to be copied // but metadata is updated. - { + 5: { bucketName: bucketName, newObjectName: objectName, copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), @@ -2113,7 +2116,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, // Test case - 6. // Test case with invalid metadata-directive. - { + 6: { bucketName: bucketName, newObjectName: "newObject1", copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), @@ -2130,7 +2133,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, // Test case - 7. // Test case with new object name is same as object to be copied // fail with BadRequest. - { + 7: { bucketName: bucketName, newObjectName: objectName, copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), @@ -2148,7 +2151,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, // Test case with non-existent source file. // Case for the purpose of failing `api.ObjectAPI.GetObjectInfo`. // Expecting the response status code to http.StatusNotFound (404). - { + 8: { bucketName: bucketName, newObjectName: objectName, copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + "non-existent-object"), @@ -2162,7 +2165,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, // Test case with non-existent source file. // Case for the purpose of failing `api.ObjectAPI.PutObject`. // Expecting the response status code to http.StatusNotFound (404). - { + 9: { bucketName: "non-existent-destination-bucket", newObjectName: objectName, copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), @@ -2174,7 +2177,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, // Test case - 10. // Case with invalid AccessKey. - { + 10: { bucketName: bucketName, newObjectName: objectName, copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), @@ -2184,7 +2187,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, expectedRespStatus: http.StatusForbidden, }, // Test case - 11, copy metadata from newObject1 with satisfying modified header. - { + 11: { bucketName: bucketName, newObjectName: "newObject1", copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), @@ -2194,7 +2197,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, expectedRespStatus: http.StatusOK, }, // Test case - 12, copy metadata from newObject1 with unsatisfying modified header. - { + 12: { bucketName: bucketName, newObjectName: "newObject1", copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), @@ -2204,7 +2207,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, expectedRespStatus: http.StatusPreconditionFailed, }, // Test case - 13, copy metadata from newObject1 with wrong modified header format - { + 13: { bucketName: bucketName, newObjectName: "newObject1", copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), @@ -2214,7 +2217,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, expectedRespStatus: http.StatusOK, }, // Test case - 14, copy metadata from newObject1 with satisfying unmodified header. - { + 14: { bucketName: bucketName, newObjectName: "newObject1", copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), @@ -2224,7 +2227,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, expectedRespStatus: http.StatusOK, }, // Test case - 15, copy metadata from newObject1 with unsatisfying unmodified header. - { + 15: { bucketName: bucketName, newObjectName: "newObject1", copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), @@ -2234,7 +2237,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, expectedRespStatus: http.StatusPreconditionFailed, }, // Test case - 16, copy metadata from newObject1 with incorrect unmodified header format. - { + 16: { bucketName: bucketName, newObjectName: "newObject1", copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), @@ -2244,7 +2247,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, expectedRespStatus: http.StatusOK, }, // Test case - 17, copy metadata from newObject1 with null versionId - { + 17: { bucketName: bucketName, newObjectName: "newObject1", copySourceHeader: url.QueryEscape(SlashSeparator+bucketName+SlashSeparator+objectName) + "?versionId=null", @@ -2253,7 +2256,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, expectedRespStatus: http.StatusOK, }, // Test case - 18, copy metadata from newObject1 with non null versionId - { + 18: { bucketName: bucketName, newObjectName: "newObject1", copySourceHeader: url.QueryEscape(SlashSeparator+bucketName+SlashSeparator+objectName) + "?versionId=17", @@ -2273,7 +2276,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, 0, nil, testCase.accessKey, testCase.secretKey, nil) if err != nil { - t.Fatalf("Test %d: Failed to create HTTP request for copy Object: %v", i+1, err) + t.Fatalf("Test %d: Failed to create HTTP request for copy Object: %v", i, err) } // "X-Amz-Copy-Source" header contains the information about the source bucket and the object to copied. if testCase.copySourceHeader != "" { @@ -2303,25 +2306,35 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter.ServeHTTP(rec, req) // Assert the response code with the expected status. if rec.Code != testCase.expectedRespStatus { - t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i, instanceType, testCase.expectedRespStatus, rec.Code) continue } if rec.Code == http.StatusOK { var cpObjResp CopyObjectResponse if err = xml.Unmarshal(rec.Body.Bytes(), &cpObjResp); err != nil { - t.Fatalf("Test %d: %s: Failed to parse the CopyObjectResult response: %s", i+1, instanceType, err) + t.Fatalf("Test %d: %s: Failed to parse the CopyObjectResult response: %s", i, instanceType, err) } // See if the new object is formed. // testing whether the copy was successful. - err = obj.GetObject(context.Background(), testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0], "", opts) - if err != nil { - t.Fatalf("Test %d: %s: Failed to fetch the copied object: %s", i+1, instanceType, err) + // Note that this goes directly to the file system, + // so encryption/compression may interfere at some point. + + globalCompressConfigMu.Lock() + cfg := globalCompressConfig + globalCompressConfigMu.Unlock() + if !cfg.Enabled { + err = obj.GetObject(context.Background(), testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0], "", opts) + if err != nil { + t.Fatalf("Test %d: %s: Failed to fetch the copied object: %s", i, instanceType, err) + } + if !bytes.Equal(bytesData[0].byteData, buffers[0].Bytes()) { + t.Errorf("Test %d: %s: Data Mismatch: Data fetched back from the copied object doesn't match the original one.", i, instanceType) + } + buffers[0].Reset() + } else { + t.Log("object not validated due to compression") } - if !bytes.Equal(bytesData[0].byteData, buffers[0].Bytes()) { - t.Errorf("Test %d: %s: Data Mismatch: Data fetched back from the copied object doesn't match the original one.", i+1, instanceType) - } - buffers[0].Reset() } // Verify response of the V2 signed HTTP request. @@ -2330,7 +2343,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, reqV2, err = newTestRequest(http.MethodPut, getCopyObjectURL("", testCase.bucketName, testCase.newObjectName), 0, nil) if err != nil { - t.Fatalf("Test %d: Failed to create HTTP request for copy Object: %v", i+1, err) + t.Fatalf("Test %d: Failed to create HTTP request for copy Object: %v", i, err) } // "X-Amz-Copy-Source" header contains the information about the source bucket and the object to copied. if testCase.copySourceHeader != "" { @@ -2366,7 +2379,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, // Call the ServeHTTP to execute the handler. apiRouter.ServeHTTP(recV2, reqV2) if recV2.Code != testCase.expectedRespStatus { - t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code) + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i, instanceType, testCase.expectedRespStatus, recV2.Code) } } @@ -3304,7 +3317,7 @@ func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string // when the request signature type is `streaming signature`. func TestAPIPutObjectPartHandlerStreaming(t *testing.T) { defer DetectTestLeak(t)() - ExecObjectLayerAPITest(t, testAPIPutObjectPartHandlerStreaming, []string{"NewMultipart", "PutObjectPart"}) + ExecExtendedObjectLayerAPITest(t, testAPIPutObjectPartHandlerStreaming, []string{"NewMultipart", "PutObjectPart"}) } func testAPIPutObjectPartHandlerStreaming(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, @@ -3392,7 +3405,7 @@ func testAPIPutObjectPartHandlerStreaming(obj ObjectLayer, instanceType, bucketN // for variety of inputs. func TestAPIPutObjectPartHandler(t *testing.T) { defer DetectTestLeak(t)() - ExecObjectLayerAPITest(t, testAPIPutObjectPartHandler, []string{"PutObjectPart"}) + ExecExtendedObjectLayerAPITest(t, testAPIPutObjectPartHandler, []string{"PutObjectPart"}) } func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, @@ -3797,7 +3810,7 @@ func testAPIListObjectPartsHandlerPreSign(obj ObjectLayer, instanceType, bucketN // for variety of success/failure cases. func TestAPIListObjectPartsHandler(t *testing.T) { defer DetectTestLeak(t)() - ExecObjectLayerAPITest(t, testAPIListObjectPartsHandler, []string{"ListObjectParts"}) + ExecExtendedObjectLayerAPITest(t, testAPIListObjectPartsHandler, []string{"ListObjectParts"}) } func testAPIListObjectPartsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, diff --git a/cmd/object_api_suite_test.go b/cmd/object_api_suite_test.go index 124002cc5..82876e041 100644 --- a/cmd/object_api_suite_test.go +++ b/cmd/object_api_suite_test.go @@ -21,10 +21,12 @@ import ( "context" "io" "math/rand" + "os" "strconv" "testing" - humanize "github.com/dustin/go-humanize" + "github.com/dustin/go-humanize" + "github.com/minio/minio/cmd/crypto" ) // Return pointer to testOneByteReadEOF{} @@ -68,10 +70,8 @@ func (r *testOneByteReadNoEOF) Read(p []byte) (n int, err error) { return n, nil } -type ObjectLayerAPISuite struct{} - // Wrapper for calling testMakeBucket for both Erasure and FS. -func (s *ObjectLayerAPISuite) TestMakeBucket(t *testing.T) { +func TestMakeBucket(t *testing.T) { ExecObjectLayerTest(t, testMakeBucket) } @@ -84,8 +84,8 @@ func testMakeBucket(obj ObjectLayer, instanceType string, t TestErrHandler) { } // Wrapper for calling testMultipartObjectCreation for both Erasure and FS. -func (s *ObjectLayerAPISuite) TestMultipartObjectCreation(t *testing.T) { - ExecObjectLayerTest(t, testMultipartObjectCreation) +func TestMultipartObjectCreation(t *testing.T) { + ExecExtendedObjectLayerTest(t, testMultipartObjectCreation) } // Tests validate creation of part files during Multipart operation. @@ -128,7 +128,7 @@ func testMultipartObjectCreation(obj ObjectLayer, instanceType string, t TestErr } // Wrapper for calling testMultipartObjectAbort for both Erasure and FS. -func (s *ObjectLayerAPISuite) TestMultipartObjectAbort(t *testing.T) { +func TestMultipartObjectAbort(t *testing.T) { ExecObjectLayerTest(t, testMultipartObjectAbort) } @@ -173,8 +173,8 @@ func testMultipartObjectAbort(obj ObjectLayer, instanceType string, t TestErrHan } // Wrapper for calling testMultipleObjectCreation for both Erasure and FS. -func (s *ObjectLayerAPISuite) TestMultipleObjectCreation(t *testing.T) { - ExecObjectLayerTest(t, testMultipleObjectCreation) +func TestMultipleObjectCreation(t *testing.T) { + ExecExtendedObjectLayerTest(t, testMultipleObjectCreation) } // Tests validate object creation. @@ -230,7 +230,7 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrH } // Wrapper for calling TestPaging for both Erasure and FS. -func (s *ObjectLayerAPISuite) TestPaging(t *testing.T) { +func TestPaging(t *testing.T) { ExecObjectLayerTest(t, testPaging) } @@ -434,7 +434,7 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) { } // Wrapper for calling testObjectOverwriteWorks for both Erasure and FS. -func (s *ObjectLayerAPISuite) TestObjectOverwriteWorks(t *testing.T) { +func TestObjectOverwriteWorks(t *testing.T) { ExecObjectLayerTest(t, testObjectOverwriteWorks) } @@ -471,7 +471,7 @@ func testObjectOverwriteWorks(obj ObjectLayer, instanceType string, t TestErrHan } // Wrapper for calling testNonExistantBucketOperations for both Erasure and FS. -func (s *ObjectLayerAPISuite) TestNonExistantBucketOperations(t *testing.T) { +func TestNonExistantBucketOperations(t *testing.T) { ExecObjectLayerTest(t, testNonExistantBucketOperations) } @@ -488,7 +488,7 @@ func testNonExistantBucketOperations(obj ObjectLayer, instanceType string, t Tes } // Wrapper for calling testBucketRecreateFails for both Erasure and FS. -func (s *ObjectLayerAPISuite) TestBucketRecreateFails(t *testing.T) { +func TestBucketRecreateFails(t *testing.T) { ExecObjectLayerTest(t, testBucketRecreateFails) } @@ -508,9 +508,68 @@ func testBucketRecreateFails(obj ObjectLayer, instanceType string, t TestErrHand } } +func execExtended(t *testing.T, fn func(t *testing.T)) { + // Exec with default settings... + globalCompressConfigMu.Lock() + globalCompressConfig.Enabled = false + globalCompressConfigMu.Unlock() + t.Run("default", func(t *testing.T) { + fn(t) + }) + if testing.Short() { + return + } + + // Enable compression and exec... + globalCompressConfigMu.Lock() + globalCompressConfig.Enabled = true + globalCompressConfig.MimeTypes = nil + globalCompressConfig.Extensions = nil + globalCompressConfigMu.Unlock() + t.Run("compressed", func(t *testing.T) { + fn(t) + }) + + globalAutoEncryption = true + os.Setenv("MINIO_KMS_MASTER_KEY", "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574") + defer os.Setenv("MINIO_KMS_MASTER_KEY", "") + var err error + GlobalKMS, err = crypto.NewKMS(crypto.KMSConfig{}) + if err != nil { + t.Fatal(err) + } + + t.Run("encrypted", func(t *testing.T) { + fn(t) + }) + + // Enable compression of encrypted and exec... + globalCompressConfigMu.Lock() + globalCompressConfig.AllowEncrypted = true + globalCompressConfigMu.Unlock() + t.Run("compressed+encrypted", func(t *testing.T) { + fn(t) + }) + + // Reset... + globalCompressConfigMu.Lock() + globalCompressConfig.Enabled = false + globalCompressConfig.AllowEncrypted = false + globalCompressConfigMu.Unlock() + globalAutoEncryption = false +} + +// ExecExtendedObjectLayerTest will execute the tests with combinations of encrypted & compressed. +// This can be used to test functionality when reading and writing data. +func ExecExtendedObjectLayerTest(t *testing.T, objTest objTestType) { + execExtended(t, func(t *testing.T) { + ExecObjectLayerTest(t, objTest) + }) +} + // Wrapper for calling testPutObject for both Erasure and FS. -func (s *ObjectLayerAPISuite) TestPutObject(t *testing.T) { - ExecObjectLayerTest(t, testPutObject) +func TestPutObject(t *testing.T) { + ExecExtendedObjectLayerTest(t, testPutObject) } // Tests validate PutObject without prefix. @@ -553,8 +612,8 @@ func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) { } // Wrapper for calling testPutObjectInSubdir for both Erasure and FS. -func (s *ObjectLayerAPISuite) TestPutObjectInSubdir(t *testing.T) { - ExecObjectLayerTest(t, testPutObjectInSubdir) +func TestPutObjectInSubdir(t *testing.T) { + ExecExtendedObjectLayerTest(t, testPutObjectInSubdir) } // Tests validate PutObject with subdirectory prefix. @@ -585,7 +644,7 @@ func testPutObjectInSubdir(obj ObjectLayer, instanceType string, t TestErrHandle } // Wrapper for calling testListBuckets for both Erasure and FS. -func (s *ObjectLayerAPISuite) TestListBuckets(t *testing.T) { +func TestListBuckets(t *testing.T) { ExecObjectLayerTest(t, testListBuckets) } @@ -644,7 +703,7 @@ func testListBuckets(obj ObjectLayer, instanceType string, t TestErrHandler) { } // Wrapper for calling testListBucketsOrder for both Erasure and FS. -func (s *ObjectLayerAPISuite) TestListBucketsOrder(t *testing.T) { +func TestListBucketsOrder(t *testing.T) { ExecObjectLayerTest(t, testListBucketsOrder) } @@ -678,7 +737,7 @@ func testListBucketsOrder(obj ObjectLayer, instanceType string, t TestErrHandler } // Wrapper for calling testListObjectsTestsForNonExistantBucket for both Erasure and FS. -func (s *ObjectLayerAPISuite) TestListObjectsTestsForNonExistantBucket(t *testing.T) { +func TestListObjectsTestsForNonExistantBucket(t *testing.T) { ExecObjectLayerTest(t, testListObjectsTestsForNonExistantBucket) } @@ -700,7 +759,7 @@ func testListObjectsTestsForNonExistantBucket(obj ObjectLayer, instanceType stri } // Wrapper for calling testNonExistantObjectInBucket for both Erasure and FS. -func (s *ObjectLayerAPISuite) TestNonExistantObjectInBucket(t *testing.T) { +func TestNonExistantObjectInBucket(t *testing.T) { ExecObjectLayerTest(t, testNonExistantObjectInBucket) } @@ -716,8 +775,8 @@ func testNonExistantObjectInBucket(obj ObjectLayer, instanceType string, t TestE t.Fatalf("%s: Expected error but found nil", instanceType) } if isErrObjectNotFound(err) { - if err.Error() != "Object not found: bucket#dir1" { - t.Errorf("%s: Expected the Error message to be `%s`, but instead found `%s`", instanceType, "Object not found: bucket#dir1", err.Error()) + if err.Error() != "Object not found: bucket/dir1" { + t.Errorf("%s: Expected the Error message to be `%s`, but instead found `%s`", instanceType, "Object not found: bucket/dir1", err.Error()) } } else { if err.Error() != "fails" { @@ -727,7 +786,7 @@ func testNonExistantObjectInBucket(obj ObjectLayer, instanceType string, t TestE } // Wrapper for calling testGetDirectoryReturnsObjectNotFound for both Erasure and FS. -func (s *ObjectLayerAPISuite) TestGetDirectoryReturnsObjectNotFound(t *testing.T) { +func TestGetDirectoryReturnsObjectNotFound(t *testing.T) { ExecObjectLayerTest(t, testGetDirectoryReturnsObjectNotFound) } @@ -770,7 +829,7 @@ func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string, } // Wrapper for calling testContentType for both Erasure and FS. -func (s *ObjectLayerAPISuite) TestContentType(t *testing.T) { +func TestContentType(t *testing.T) { ExecObjectLayerTest(t, testContentType) } diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index 6c4588387..82e217c89 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -1866,6 +1866,14 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [ removeRoots(append(erasureDisks, fsDir)) } +// ExecExtendedObjectLayerTest will execute the tests with combinations of encrypted & compressed. +// This can be used to test functionality when reading and writing data. +func ExecExtendedObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints []string) { + execExtended(t, func(t *testing.T) { + ExecObjectLayerAPITest(t, objAPITest, endpoints) + }) +} + // function to be passed to ExecObjectLayerAPITest, for executing object layr API handler tests. type objAPITestType func(obj ObjectLayer, instanceType string, bucketName string, apiRouter http.Handler, credentials auth.Credentials, t *testing.T) diff --git a/docs/bucket/versioning/DESIGN.md b/docs/bucket/versioning/DESIGN.md index fd2745ddf..df651ff5b 100644 --- a/docs/bucket/versioning/DESIGN.md +++ b/docs/bucket/versioning/DESIGN.md @@ -26,7 +26,7 @@ Once the header is validated, we proceed to the actual data structure of the `xl - LegacyObjectType (preserves existing deployments and older xl.json format) - DeleteMarker (a versionId to capture the DELETE sequences implemented primarily for AWS spec compatibility) -A sample msgpack-JSON `xl.meta`, you can debug the content inside `xl.meta` using [xl-meta-to-json.go](https://github.com/minio/minio/blob/master/docs/bucket/versioning/xl-meta-to-json.go) program. +A sample msgpack-JSON `xl.meta`, you can debug the content inside `xl.meta` using [xl-meta.go](https://github.com/minio/minio/blob/master/docs/bucket/versioning/xl-meta.go) program. ```json { "Versions": [ diff --git a/docs/bucket/versioning/xl-meta-to-json.go b/docs/bucket/versioning/xl-meta.go similarity index 94% rename from docs/bucket/versioning/xl-meta-to-json.go rename to docs/bucket/versioning/xl-meta.go index 6c4c187d5..090fc7b68 100644 --- a/docs/bucket/versioning/xl-meta-to-json.go +++ b/docs/bucket/versioning/xl-meta.go @@ -56,11 +56,12 @@ GLOBAL FLAGS: } app.Action = func(c *cli.Context) error { - if !c.Args().Present() { - cli.ShowAppHelp(c) - return nil + files := c.Args() + if len(files) == 0 { + // If no args, assume xl.meta + files = []string{"xl.meta"} } - for _, file := range c.Args() { + for _, file := range files { var r io.Reader switch file { case "-": diff --git a/docs/compression/README.md b/docs/compression/README.md index effafbc7a..3adbd42eb 100644 --- a/docs/compression/README.md +++ b/docs/compression/README.md @@ -1,10 +1,19 @@ # Compression Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) -MinIO server allows streaming compression to ensure efficient disk space usage. Compression happens inflight, i.e objects are compressed before being written to disk(s). MinIO uses [`klauspost/compress/s2`](https://github.com/klauspost/compress/tree/master/s2) streaming compression due to its stability and performance. +MinIO server allows streaming compression to ensure efficient disk space usage. +Compression happens inflight, i.e objects are compressed before being written to disk(s). +MinIO uses [`klauspost/compress/s2`](https://github.com/klauspost/compress/tree/master/s2) +streaming compression due to its stability and performance. -This algorithm is specifically optimized for machine generated content. Write throughput is typically at least 300MB/s per CPU core. Decompression speed is typically at least 1GB/s. -This means that in cases where raw IO is below these numbers compression will not only reduce disk usage but also help increase system throughput. -Typically enabling compression on spinning disk systems will increase speed when the content can be compressed. +This algorithm is specifically optimized for machine generated content. +Write throughput is typically at least 500MB/s per CPU core, +and scales with the number of available CPU cores. +Decompression speed is typically at least 1GB/s. + +This means that in cases where raw IO is below these numbers +compression will not only reduce disk usage but also help increase system throughput. +Typically, enabling compression on spinning disk systems +will increase speed when the content can be compressed. ## Get Started @@ -14,40 +23,71 @@ Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/docs/minio-quicksta ### 2. Run MinIO with compression -Compression can be enabled by updating the `compress` config settings for MinIO server config. Config `compress` settings take extensions and mime-types to be compressed. +Compression can be enabled by updating the `compress` config settings for MinIO server config. +Config `compress` settings take extensions and mime-types to be compressed. -``` -$ mc admin config get myminio compression +```bash +~ mc admin config get myminio compression compression extensions=".txt,.log,.csv,.json,.tar,.xml,.bin" mime_types="text/*,application/json,application/xml" ``` Default config includes most common highly compressible content extensions and mime-types. -``` -$ mc admin config set myminio compression extensions=".pdf" mime_types="application/pdf" +```bash +~ mc admin config set myminio compression extensions=".pdf" mime_types="application/pdf" ``` To show help on setting compression config values. -``` +```bash ~ mc admin config set myminio compression ``` -To enable compression for all content, with default extensions and mime-types. -``` -~ mc admin config set myminio compression enable="on" +To enable compression for all content, no matter the extension and content type +(except for the default excluded types) set BOTH extensions and mime types to empty. + +```bash +~ mc admin config set myminio compression enable="on" extensions="" mime_types="" ``` -The compression settings may also be set through environment variables. When set, environment variables override the defined `compress` config settings in the server config. +The compression settings may also be set through environment variables. +When set, environment variables override the defined `compress` config settings in the server config. ```bash export MINIO_COMPRESS="on" -export MINIO_COMPRESS_EXTENSIONS=".pdf,.doc" -export MINIO_COMPRESS_MIME_TYPES="application/pdf" +export MINIO_COMPRESS_EXTENSIONS=".txt,.log,.csv,.json,.tar,.xml,.bin" +export MINIO_COMPRESS_MIME_TYPES="text/*,application/json,application/xml" ``` -### 3. Note +### 3. Compression + Encryption -- Already compressed objects are not fit for compression since they do not have compressible patterns. Such objects do not produce efficient [`LZ compression`](https://en.wikipedia.org/wiki/LZ77_and_LZ78) which is a fitness factor for a lossless data compression. Below is a list of common files and content-types which are not suitable for compression. +Combining encryption and compression is not safe in all setups. +This is particularly so if the compression ratio of your content reveals information about it. +See [CRIME TLS](https://en.wikipedia.org/wiki/CRIME) as an example of this. + +Therefore, compression is disabled when encrypting by default, and must be enabled separately. + +Consult our security experts on [SUBNET](https://min.io/pricing) to help you evaluate if +your setup can use this feature combination safely. + +To enable compression+encryption use: + +```bash +~ mc admin config set myminio compression allow_encryption=on +``` + +Or alternatively through the environment variable `MINIO_COMPRESS_ALLOW_ENCRYPTION=on`. + +### 4. Excluded Types + +- Already compressed objects are not fit for compression since they do not have compressible patterns. +Such objects do not produce efficient [`LZ compression`](https://en.wikipedia.org/wiki/LZ77_and_LZ78) +which is a fitness factor for a lossless data compression. + +Pre-compressed input typically compresses in excess of 2GiB/s per core, +so performance impact should be minimal even if precompressed data is re-compressed. +Decompressing incompressible data has no significant performance impact. + +Below is a list of common files and content-types which are typically not suitable for compression. - Extensions @@ -72,15 +112,17 @@ export MINIO_COMPRESS_MIME_TYPES="application/pdf" | `application/x-compress` | | `application/x-xz` | -All files with these extensions and mime types are excluded from compression, even if compression is enabled for all types. +All files with these extensions and mime types are excluded from compression, +even if compression is enabled for all types. -- MinIO does not support encryption with compression because compression and encryption together potentially enables room for side channel attacks like [`CRIME and BREACH`](https://blog.minio.io/c-e-compression-encryption-cb6b7f04a369) +### 5. Notes - MinIO does not support compression for Gateway (Azure/GCS/NAS) implementations. ## To test the setup -To test this setup, practice put calls to the server using `mc` and use `mc ls` on the data directory to view the size of the object. +To test this setup, practice put calls to the server using `mc` and use `mc ls` on +the data directory to view the size of the object. ## Explore Further