maxmustermann
/
minio
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Allow Compression + encryption (#11103)

This commit is contained in:
Klaus Post 2021-01-05 20:08:35 -08:00 committed by GitHub
parent 97a4c120e9
commit eb9172eecb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 399 additions and 234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
cmd/config/compress/compress.go
View File

@ -26,19 +26,22 @@ import (
// Config represents the compression settings.
type Config struct {
Enabled bool `json:"enabled"`
Extensions []string `json:"extensions"`
MimeTypes []string `json:"mime-types"`
Enabled bool `json:"enabled"`
AllowEncrypted bool `json:"allow_encryption"`
Extensions []string `json:"extensions"`
MimeTypes []string `json:"mime-types"`
}
// Compression environment variables
const (
Extensions = "extensions"
MimeTypes = "mime_types"
Extensions = "extensions"
AllowEncrypted = "allow_encryption"
MimeTypes = "mime_types"
EnvCompressState = "MINIO_COMPRESS_ENABLE"
EnvCompressExtensions = "MINIO_COMPRESS_EXTENSIONS"
EnvCompressMimeTypes = "MINIO_COMPRESS_MIME_TYPES"
EnvCompressState = "MINIO_COMPRESS_ENABLE"
EnvCompressAllowEncryption = "MINIO_COMPRESS_ALLOW_ENCRYPTION"
EnvCompressExtensions = "MINIO_COMPRESS_EXTENSIONS"
EnvCompressMimeTypes = "MINIO_COMPRESS_MIME_TYPES"
// Include-list for compression.
DefaultExtensions = ".txt,.log,.csv,.json,.tar,.xml,.bin"
@ -52,6 +55,10 @@ var (
Key: config.Enable,
Value: config.EnableOff,
},
config.KV{
Key: AllowEncrypted,
Value: config.EnableOff,
},
config.KV{
Key: Extensions,
Value: DefaultExtensions,
@ -101,6 +108,12 @@ func LookupConfig(kvs config.KVS) (Config, error) {
return cfg, nil
}
allowEnc := env.Get(EnvCompressAllowEncryption, kvs.Get(AllowEncrypted))
cfg.AllowEncrypted, err = config.ParseBool(allowEnc)
if err != nil {
return cfg, err
}
compressExtensions := env.Get(EnvCompressExtensions, kvs.Get(Extensions))
compressMimeTypes := env.Get(EnvCompressMimeTypes, kvs.Get(MimeTypes))
compressMimeTypesLegacy := env.Get(EnvCompressMimeTypesLegacy, kvs.Get(MimeTypes))

4
cmd/encryption-v1.go
View File

@ -353,9 +353,7 @@ func newDecryptReaderWithObjectKey(client io.Reader, objectEncryptionKey []byte,
// DecryptBlocksRequestR - same as DecryptBlocksRequest but with a
// reader
func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
length int64, seqNumber uint32, partStart int, oi ObjectInfo, copySource bool) (
io.Reader, error) {
func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, seqNumber uint32, partStart int, oi ObjectInfo, copySource bool) (io.Reader, error) {
bucket, object := oi.Bucket, oi.Name
// Single part case

2
cmd/object-api-getobject_test.go
View File

@ -31,7 +31,7 @@ import (
// Wrapper for calling GetObject tests for both Erasure multiple disks and single node setup.
func TestGetObject(t *testing.T) {
ExecObjectLayerTest(t, testGetObject)
ExecExtendedObjectLayerTest(t, testGetObject)
}
// ObjectLayer.GetObject is called with series of cases for valid and erroneous inputs and the result is validated.

6
cmd/object-api-multipart_test.go
View File

@ -157,7 +157,7 @@ func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t TestE
// Wrapper for calling PutObjectPart tests for both Erasure multiple disks and single node setup.
func TestObjectAPIPutObjectPart(t *testing.T) {
ExecObjectLayerTest(t, testObjectAPIPutObjectPart)
ExecExtendedObjectLayerTest(t, testObjectAPIPutObjectPart)
}
// Tests validate correctness of PutObjectPart.
@ -289,7 +289,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
// Wrapper for calling TestListMultipartUploads tests for both Erasure multiple disks and single node setup.
func TestListMultipartUploads(t *testing.T) {
ExecObjectLayerTest(t, testListMultipartUploads)
ExecExtendedObjectLayerTest(t, testListMultipartUploads)
}
// testListMultipartUploads - Tests validate listing of multipart uploads.
@ -1643,7 +1643,7 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler)
// Test for validating complete Multipart upload.
func TestObjectCompleteMultipartUpload(t *testing.T) {
ExecObjectLayerTest(t, testObjectCompleteMultipartUpload)
ExecExtendedObjectLayerTest(t, testObjectCompleteMultipartUpload)
}
// Tests validate CompleteMultipart functionality.

2
cmd/object-api-putobject_test.go
View File

@ -37,7 +37,7 @@ func md5Header(data []byte) map[string]string {
// Wrapper for calling PutObject tests for both Erasure multiple disks and single node setup.
func TestObjectAPIPutObjectSingle(t *testing.T) {
ExecObjectLayerTest(t, testObjectAPIPutObject)
ExecExtendedObjectLayerTest(t, testObjectAPIPutObject)
}
// Tests validate correctness of PutObject.

199
cmd/object-api-utils.go
View File

@ -394,9 +394,6 @@ func (o ObjectInfo) IsCompressedOK() (bool, error) {
if !ok {
return false, nil
}
if crypto.IsEncrypted(o.UserDefined) {
return true, fmt.Errorf("compression %q and encryption enabled on same object", scheme)
}
switch scheme {
case compressionAlgorithmV1, compressionAlgorithmV2:
return true, nil
@ -415,9 +412,6 @@ func (o ObjectInfo) GetActualETag(h http.Header) string {
// GetActualSize - returns the actual size of the stored object
func (o ObjectInfo) GetActualSize() (int64, error) {
if crypto.IsEncrypted(o.UserDefined) {
return o.DecryptedSize()
}
if o.IsCompressed() {
sizeStr, ok := o.UserDefined[ReservedMetadataPrefix+"actual-size"]
if !ok {
@ -429,6 +423,10 @@ func (o ObjectInfo) GetActualSize() (int64, error) {
}
return size, nil
}
if crypto.IsEncrypted(o.UserDefined) {
return o.DecryptedSize()
}
return o.Size, nil
}
@ -441,7 +439,7 @@ func isCompressible(header http.Header, object string) bool {
globalCompressConfigMu.Unlock()
_, ok := crypto.IsRequested(header)
if !cfg.Enabled || ok || excludeForCompression(header, object, cfg) {
if !cfg.Enabled || (ok && !cfg.AllowEncrypted) || excludeForCompression(header, object, cfg) {
return false
}
return true
@ -461,16 +459,15 @@ func excludeForCompression(header http.Header, object string, cfg compress.Confi
}
// Filter compression includes.
if len(cfg.Extensions) == 0 || len(cfg.MimeTypes) == 0 {
return false
exclude := len(cfg.Extensions) > 0 || len(cfg.MimeTypes) > 0
if len(cfg.Extensions) > 0 && hasStringSuffixInSlice(objStr, cfg.Extensions) {
exclude = false
}
extensions := cfg.Extensions
mimeTypes := cfg.MimeTypes
if hasStringSuffixInSlice(objStr, extensions) || hasPattern(mimeTypes, contentType) {
return false
if len(cfg.MimeTypes) > 0 && hasPattern(cfg.MimeTypes, contentType) {
exclude = false
}
return true
return exclude
}
// Utility which returns if a string is present in the list.
@ -520,21 +517,29 @@ func partNumberToRangeSpec(oi ObjectInfo, partNumber int) *HTTPRangeSpec {
}
// Returns the compressed offset which should be skipped.
func getCompressedOffsets(objectInfo ObjectInfo, offset int64) (int64, int64) {
var compressedOffset int64
// If encrypted offsets are adjusted for encrypted block headers/trailers.
// Since de-compression is after decryption encryption overhead is only added to compressedOffset.
func getCompressedOffsets(objectInfo ObjectInfo, offset int64) (compressedOffset int64, partSkip int64) {
var skipLength int64
var cumulativeActualSize int64
var firstPartIdx int
if len(objectInfo.Parts) > 0 {
for _, part := range objectInfo.Parts {
for i, part := range objectInfo.Parts {
cumulativeActualSize += part.ActualSize
if cumulativeActualSize <= offset {
compressedOffset += part.Size
} else {
firstPartIdx = i
skipLength = cumulativeActualSize - part.ActualSize
break
}
}
}
if isEncryptedMultipart(objectInfo) && firstPartIdx > 0 {
off, _, _, _, _, err := objectInfo.GetDecryptedRange(partNumberToRangeSpec(objectInfo, firstPartIdx))
logger.LogIf(context.Background(), err)
compressedOffset += off
}
return compressedOffset, offset - skipLength
}
@ -604,9 +609,92 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, cl
isEncrypted = false
}
var skipLen int64
// Calculate range to read (different for
// e.g. encrypted/compressed objects)
// Calculate range to read (different for encrypted/compressed objects)
switch {
case isCompressed:
// If compressed, we start from the beginning of the part.
// Read the decompressed size from the meta.json.
actualSize, err := oi.GetActualSize()
if err != nil {
return nil, 0, 0, err
}
off, length = int64(0), oi.Size
decOff, decLength := int64(0), actualSize
if rs != nil {
off, length, err = rs.GetOffsetLength(actualSize)
if err != nil {
return nil, 0, 0, err
}
// In case of range based queries on multiparts, the offset and length are reduced.
off, decOff = getCompressedOffsets(oi, off)
decLength = length
length = oi.Size - off
// For negative length we read everything.
if decLength < 0 {
decLength = actualSize - decOff
}
// Reply back invalid range if the input offset and length fall out of range.
if decOff > actualSize || decOff+decLength > actualSize {
return nil, 0, 0, errInvalidRange
}
}
fn = func(inputReader io.Reader, h http.Header, pcfn CheckPreconditionFn, cFns ...func()) (r *GetObjectReader, err error) {
cFns = append(cleanUpFns, cFns...)
if opts.CheckPrecondFn != nil && opts.CheckPrecondFn(oi) {
// Call the cleanup funcs
for i := len(cFns) - 1; i >= 0; i-- {
cFns[i]()
}
return nil, PreConditionFailed{}
}
if isEncrypted {
copySource := h.Get(xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm) != ""
// Attach decrypter on inputReader
inputReader, err = DecryptBlocksRequestR(inputReader, h, 0, opts.PartNumber, oi, copySource)
if err != nil {
// Call the cleanup funcs
for i := len(cFns) - 1; i >= 0; i-- {
cFns[i]()
}
return nil, err
}
}
// Decompression reader.
s2Reader := s2.NewReader(inputReader)
// Apply the skipLen and limit on the decompressed stream.
err = s2Reader.Skip(decOff)
if err != nil {
// Call the cleanup funcs
for i := len(cFns) - 1; i >= 0; i-- {
cFns[i]()
}
return nil, err
}
decReader := io.LimitReader(s2Reader, decLength)
if decLength > compReadAheadSize {
rah, err := readahead.NewReaderSize(decReader, compReadAheadBuffers, compReadAheadBufSize)
if err == nil {
decReader = rah
cFns = append(cFns, func() {
rah.Close()
})
}
}
oi.Size = decLength
// Assemble the GetObjectReader
r = &GetObjectReader{
ObjInfo: oi,
pReader: decReader,
cleanUpFns: cFns,
opts: opts,
}
return r, nil
}
case isEncrypted:
var seqNumber uint32
var partStart int
@ -635,8 +723,7 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, cl
cFns = append(cleanUpFns, cFns...)
// Attach decrypter on inputReader
var decReader io.Reader
decReader, err = DecryptBlocksRequestR(inputReader, h,
off, length, seqNumber, partStart, oi, copySource)
decReader, err = DecryptBlocksRequestR(inputReader, h, seqNumber, partStart, oi, copySource)
if err != nil {
// Call the cleanup funcs
for i := len(cFns) - 1; i >= 0; i-- {
@ -659,76 +746,6 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, cl
// decrypted stream
decReader = io.LimitReader(ioutil.NewSkipReader(decReader, skipLen), decRangeLength)
// Assemble the GetObjectReader
r = &GetObjectReader{
ObjInfo: oi,
pReader: decReader,
cleanUpFns: cFns,
opts: opts,
}
return r, nil
}
case isCompressed:
// Read the decompressed size from the meta.json.
actualSize, err := oi.GetActualSize()
if err != nil {
return nil, 0, 0, err
}
off, length = int64(0), oi.Size
decOff, decLength := int64(0), actualSize
if rs != nil {
off, length, err = rs.GetOffsetLength(actualSize)
if err != nil {
return nil, 0, 0, err
}
// In case of range based queries on multiparts, the offset and length are reduced.
off, decOff = getCompressedOffsets(oi, off)
decLength = length
length = oi.Size - off
// For negative length we read everything.
if decLength < 0 {
decLength = actualSize - decOff
}
// Reply back invalid range if the input offset and length fall out of range.
if decOff > actualSize || decOff+decLength > actualSize {
return nil, 0, 0, errInvalidRange
}
}
fn = func(inputReader io.Reader, _ http.Header, pcfn CheckPreconditionFn, cFns ...func()) (r *GetObjectReader, err error) {
cFns = append(cleanUpFns, cFns...)
if opts.CheckPrecondFn != nil && opts.CheckPrecondFn(oi) {
// Call the cleanup funcs
for i := len(cFns) - 1; i >= 0; i-- {
cFns[i]()
}
return nil, PreConditionFailed{}
}
// Decompression reader.
s2Reader := s2.NewReader(inputReader)
// Apply the skipLen and limit on the decompressed stream.
err = s2Reader.Skip(decOff)
if err != nil {
// Call the cleanup funcs
for i := len(cFns) - 1; i >= 0; i-- {
cFns[i]()
}
return nil, err
}
decReader := io.LimitReader(s2Reader, decLength)
if decLength > compReadAheadSize {
rah, err := readahead.NewReaderSize(decReader, compReadAheadBuffers, compReadAheadBufSize)
if err == nil {
decReader = rah
cFns = append(cFns, func() {
rah.Close()
})
}
}
oi.Size = decLength
// Assemble the GetObjectReader
r = &GetObjectReader{
ObjInfo: oi,

14
cmd/object-api-utils_test.go
View File

@ -317,7 +317,7 @@ func TestIsCompressed(t *testing.T) {
result bool
err bool
}{
{
0: {
objInfo: ObjectInfo{
UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV1,
"content-type": "application/octet-stream",
@ -325,7 +325,7 @@ func TestIsCompressed(t *testing.T) {
},
result: true,
},
{
1: {
objInfo: ObjectInfo{
UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV2,
"content-type": "application/octet-stream",
@ -333,7 +333,7 @@ func TestIsCompressed(t *testing.T) {
},
result: true,
},
{
2: {
objInfo: ObjectInfo{
UserDefined: map[string]string{"X-Minio-Internal-compression": "unknown/compression/type",
"content-type": "application/octet-stream",
@ -342,7 +342,7 @@ func TestIsCompressed(t *testing.T) {
result: true,
err: true,
},
{
3: {
objInfo: ObjectInfo{
UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV2,
"content-type": "application/octet-stream",
@ -351,9 +351,9 @@ func TestIsCompressed(t *testing.T) {
},
},
result: true,
err: true,
err: false,
},
{
4: {
objInfo: ObjectInfo{
UserDefined: map[string]string{"X-Minio-Internal-XYZ": "klauspost/compress/s2",
"content-type": "application/octet-stream",
@ -361,7 +361,7 @@ func TestIsCompressed(t *testing.T) {
},
result: false,
},
{
5: {
objInfo: ObjectInfo{
UserDefined: map[string]string{"content-type": "application/octet-stream",
"etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"},

70
cmd/object-handlers.go
View File

@ -977,18 +977,14 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
}
var reader io.Reader
var length = srcInfo.Size
// Set the actual size to the decrypted size if encrypted.
actualSize := srcInfo.Size
if crypto.IsEncrypted(srcInfo.UserDefined) {
actualSize, err = srcInfo.DecryptedSize()
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
length = actualSize
// Set the actual size to the compressed/decrypted size if encrypted.
actualSize, err := srcInfo.GetActualSize()
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
length := actualSize
if !cpSrcDstSame {
if err := enforceBucketQuota(ctx, dstBucket, actualSize); err != nil {
@ -1000,8 +996,10 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
var compressMetadata map[string]string
// No need to compress for remote etcd calls
// Pass the decompressed stream to such calls.
isCompressed := objectAPI.IsCompressionSupported() && isCompressible(r.Header, srcObject) && !isRemoteCopyRequired(ctx, srcBucket, dstBucket, objectAPI)
if isCompressed {
isDstCompressed := objectAPI.IsCompressionSupported() &&
isCompressible(r.Header, srcObject) &&
!isRemoteCopyRequired(ctx, srcBucket, dstBucket, objectAPI)
if isDstCompressed {
compressMetadata = make(map[string]string, 2)
// Preserving the compression metadata.
compressMetadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
@ -1034,7 +1032,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
_, objectEncryption := crypto.IsRequested(r.Header)
objectEncryption = objectEncryption || crypto.IsSourceEncrypted(srcInfo.UserDefined)
var encMetadata = make(map[string]string)
if objectAPI.IsEncryptionSupported() && !isCompressed {
if objectAPI.IsEncryptionSupported() {
// Encryption parameters not applicable for this object.
if !crypto.IsEncrypted(srcInfo.UserDefined) && crypto.SSECopy.IsRequested(r.Header) {
writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL, guessIsBrowserReq(r))
@ -1105,8 +1103,10 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
var targetSize int64
switch {
case isDstCompressed:
targetSize = -1
case !isSourceEncrypted && !isTargetEncrypted:
targetSize = srcInfo.Size
targetSize, _ = srcInfo.GetActualSize()
case isSourceEncrypted && isTargetEncrypted:
objInfo := ObjectInfo{Size: actualSize}
targetSize = objInfo.EncryptedSize()
@ -1131,7 +1131,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
}
// do not try to verify encrypted content
srcInfo.Reader, err = hash.NewReader(reader, targetSize, "", "", targetSize, globalCLIContext.StrictS3Compat)
srcInfo.Reader, err = hash.NewReader(reader, targetSize, "", "", actualSize, globalCLIContext.StrictS3Compat)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -1541,10 +1541,15 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
info := ObjectInfo{Size: size}
wantSize := int64(-1)
if size >= 0 {
info := ObjectInfo{Size: size}
wantSize = info.EncryptedSize()
}
// do not try to verify encrypted content
hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", size, globalCLIContext.StrictS3Compat)
hashReader, err = hash.NewReader(reader, wantSize, "", "", actualSize, globalCLIContext.StrictS3Compat)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -1564,10 +1569,6 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
}
switch {
case objInfo.IsCompressed():
if !strings.HasSuffix(objInfo.ETag, "-1") {
objInfo.ETag = objInfo.ETag + "-1"
}
case crypto.IsEncrypted(objInfo.UserDefined):
switch {
case crypto.S3.IsEncrypted(objInfo.UserDefined):
@ -1581,6 +1582,10 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
objInfo.ETag = objInfo.ETag[len(objInfo.ETag)-32:]
}
}
case objInfo.IsCompressed():
if !strings.HasSuffix(objInfo.ETag, "-1") {
objInfo.ETag = objInfo.ETag + "-1"
}
}
if mustReplicate(ctx, r, bucket, object, metadata, "") {
globalReplicationState.queueReplicaTask(objInfo)
@ -1892,7 +1897,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
actualPartSize := srcInfo.Size
if crypto.IsEncrypted(srcInfo.UserDefined) {
actualPartSize, err = srcInfo.DecryptedSize()
actualPartSize, err = srcInfo.GetActualSize()
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -1991,7 +1996,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
isEncrypted := crypto.IsEncrypted(mi.UserDefined)
var objectEncryptionKey crypto.ObjectKey
if objectAPI.IsEncryptionSupported() && !isCompressed && isEncrypted {
if objectAPI.IsEncryptionSupported() && isEncrypted {
if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrSSEMultipartEncrypted), r.URL, guessIsBrowserReq(r))
return
@ -2022,8 +2027,13 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
return
}
info := ObjectInfo{Size: length}
srcInfo.Reader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", length, globalCLIContext.StrictS3Compat)
wantSize := int64(-1)
if length >= 0 {
info := ObjectInfo{Size: length}
wantSize = info.EncryptedSize()
}
srcInfo.Reader, err = hash.NewReader(reader, wantSize, "", "", actualPartSize, globalCLIContext.StrictS3Compat)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -2226,7 +2236,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
isEncrypted := crypto.IsEncrypted(mi.UserDefined)
var objectEncryptionKey crypto.ObjectKey
if objectAPI.IsEncryptionSupported() && !isCompressed && isEncrypted {
if objectAPI.IsEncryptionSupported() && isEncrypted {
if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrSSEMultipartEncrypted), r.URL, guessIsBrowserReq(r))
return
@ -2267,9 +2277,13 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
info := ObjectInfo{Size: size}
wantSize := int64(-1)
if size >= 0 {
info := ObjectInfo{Size: size}
wantSize = info.EncryptedSize()
}
// do not try to verify encrypted content
hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", size, globalCLIContext.StrictS3Compat)
hashReader, err = hash.NewReader(reader, wantSize, "", "", actualSize, globalCLIContext.StrictS3Compat)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return

95
cmd/object-handlers_test.go
View File

@ -327,7 +327,7 @@ func TestAPIGetObjectHandler(t *testing.T) {
defer func() { globalPolicySys = nil }()
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(t, testAPIGetObjectHandler, []string{"GetObject"})
ExecExtendedObjectLayerAPITest(t, testAPIGetObjectHandler, []string{"GetObject"})
}
func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@ -651,7 +651,7 @@ func TestAPIGetObjectWithMPHandler(t *testing.T) {
defer func() { globalPolicySys = nil }()
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(t, testAPIGetObjectWithMPHandler, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject"})
ExecExtendedObjectLayerAPITest(t, testAPIGetObjectWithMPHandler, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject"})
}
func testAPIGetObjectWithMPHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@ -849,7 +849,7 @@ func TestAPIGetObjectWithPartNumberHandler(t *testing.T) {
defer func() { globalPolicySys = nil }()
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(t, testAPIGetObjectWithPartNumberHandler, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject"})
ExecExtendedObjectLayerAPITest(t, testAPIGetObjectWithPartNumberHandler, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject"})
}
func testAPIGetObjectWithPartNumberHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@ -971,7 +971,7 @@ func testAPIGetObjectWithPartNumberHandler(obj ObjectLayer, instanceType, bucket
// Wrapper for calling PutObject API handler tests using streaming signature v4 for both Erasure multiple disks and FS single drive setup.
func TestAPIPutObjectStreamSigV4Handler(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(t, testAPIPutObjectStreamSigV4Handler, []string{"PutObject"})
ExecExtendedObjectLayerAPITest(t, testAPIPutObjectStreamSigV4Handler, []string{"PutObject"})
}
func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@ -1289,7 +1289,7 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
// Wrapper for calling PutObject API handler tests for both Erasure multiple disks and FS single drive setup.
func TestAPIPutObjectHandler(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(t, testAPIPutObjectHandler, []string{"PutObject"})
ExecExtendedObjectLayerAPITest(t, testAPIPutObjectHandler, []string{"PutObject"})
}
func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@ -1538,7 +1538,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
// expected.
func TestAPICopyObjectPartHandlerSanity(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(t, testAPICopyObjectPartHandlerSanity, []string{"CopyObjectPart"})
ExecExtendedObjectLayerAPITest(t, testAPICopyObjectPartHandlerSanity, []string{"CopyObjectPart"})
}
func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@ -1649,7 +1649,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
// Wrapper for calling Copy Object Part API handler tests for both Erasure multiple disks and single node setup.
func TestAPICopyObjectPartHandler(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(t, testAPICopyObjectPartHandler, []string{"CopyObjectPart"})
ExecExtendedObjectLayerAPITest(t, testAPICopyObjectPartHandler, []string{"CopyObjectPart"})
}
func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@ -1965,7 +1965,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
// Wrapper for calling Copy Object API handler tests for both Erasure multiple disks and single node setup.
func TestAPICopyObjectHandler(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(t, testAPICopyObjectHandler, []string{"CopyObject"})
ExecExtendedObjectLayerAPITest(t, testAPICopyObjectHandler, []string{"CopyObject"})
}
func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@ -2044,8 +2044,11 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
// expected output.
expectedRespStatus int
}{
0: {
expectedRespStatus: http.StatusMethodNotAllowed,
},
// Test case - 1, copy metadata from newObject1, ignore request headers.
{
1: {
bucketName: bucketName,
newObjectName: "newObject1",
copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@ -2059,7 +2062,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
// Test case - 2.
// Test case with invalid source object.
{
2: {
bucketName: bucketName,
newObjectName: "newObject1",
copySourceHeader: url.QueryEscape(SlashSeparator),
@ -2071,7 +2074,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
// Test case - 3.
// Test case with new object name is same as object to be copied.
{
3: {
bucketName: bucketName,
newObjectName: objectName,
copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@ -2084,7 +2087,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
// Test case - 4.
// Test case with new object name is same as object to be copied.
// But source copy is without leading slash
{
4: {
bucketName: bucketName,
newObjectName: objectName,
copySourceHeader: url.QueryEscape(bucketName + SlashSeparator + objectName),
@ -2097,7 +2100,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
// Test case - 5.
// Test case with new object name is same as object to be copied
// but metadata is updated.
{
5: {
bucketName: bucketName,
newObjectName: objectName,
copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@ -2113,7 +2116,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
// Test case - 6.
// Test case with invalid metadata-directive.
{
6: {
bucketName: bucketName,
newObjectName: "newObject1",
copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@ -2130,7 +2133,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
// Test case - 7.
// Test case with new object name is same as object to be copied
// fail with BadRequest.
{
7: {
bucketName: bucketName,
newObjectName: objectName,
copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@ -2148,7 +2151,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
// Test case with non-existent source file.
// Case for the purpose of failing `api.ObjectAPI.GetObjectInfo`.
// Expecting the response status code to http.StatusNotFound (404).
{
8: {
bucketName: bucketName,
newObjectName: objectName,
copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + "non-existent-object"),
@ -2162,7 +2165,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
// Test case with non-existent source file.
// Case for the purpose of failing `api.ObjectAPI.PutObject`.
// Expecting the response status code to http.StatusNotFound (404).
{
9: {
bucketName: "non-existent-destination-bucket",
newObjectName: objectName,
copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@ -2174,7 +2177,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
// Test case - 10.
// Case with invalid AccessKey.
{
10: {
bucketName: bucketName,
newObjectName: objectName,
copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@ -2184,7 +2187,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
expectedRespStatus: http.StatusForbidden,
},
// Test case - 11, copy metadata from newObject1 with satisfying modified header.
{
11: {
bucketName: bucketName,
newObjectName: "newObject1",
copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@ -2194,7 +2197,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
expectedRespStatus: http.StatusOK,
},
// Test case - 12, copy metadata from newObject1 with unsatisfying modified header.
{
12: {
bucketName: bucketName,
newObjectName: "newObject1",
copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@ -2204,7 +2207,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
expectedRespStatus: http.StatusPreconditionFailed,
},
// Test case - 13, copy metadata from newObject1 with wrong modified header format
{
13: {
bucketName: bucketName,
newObjectName: "newObject1",
copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@ -2214,7 +2217,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
expectedRespStatus: http.StatusOK,
},
// Test case - 14, copy metadata from newObject1 with satisfying unmodified header.
{
14: {
bucketName: bucketName,
newObjectName: "newObject1",
copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@ -2224,7 +2227,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
expectedRespStatus: http.StatusOK,
},
// Test case - 15, copy metadata from newObject1 with unsatisfying unmodified header.
{
15: {
bucketName: bucketName,
newObjectName: "newObject1",
copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@ -2234,7 +2237,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
expectedRespStatus: http.StatusPreconditionFailed,
},
// Test case - 16, copy metadata from newObject1 with incorrect unmodified header format.
{
16: {
bucketName: bucketName,
newObjectName: "newObject1",
copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@ -2244,7 +2247,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
expectedRespStatus: http.StatusOK,
},
// Test case - 17, copy metadata from newObject1 with null versionId
{
17: {
bucketName: bucketName,
newObjectName: "newObject1",
copySourceHeader: url.QueryEscape(SlashSeparator+bucketName+SlashSeparator+objectName) + "?versionId=null",
@ -2253,7 +2256,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
expectedRespStatus: http.StatusOK,
},
// Test case - 18, copy metadata from newObject1 with non null versionId
{
18: {
bucketName: bucketName,
newObjectName: "newObject1",
copySourceHeader: url.QueryEscape(SlashSeparator+bucketName+SlashSeparator+objectName) + "?versionId=17",
@ -2273,7 +2276,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
0, nil, testCase.accessKey, testCase.secretKey, nil)
if err != nil {
t.Fatalf("Test %d: Failed to create HTTP request for copy Object: <ERROR> %v", i+1, err)
t.Fatalf("Test %d: Failed to create HTTP request for copy Object: <ERROR> %v", i, err)
}
// "X-Amz-Copy-Source" header contains the information about the source bucket and the object to copied.
if testCase.copySourceHeader != "" {
@ -2303,25 +2306,35 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
apiRouter.ServeHTTP(rec, req)
// Assert the response code with the expected status.
if rec.Code != testCase.expectedRespStatus {
t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i, instanceType, testCase.expectedRespStatus, rec.Code)
continue
}
if rec.Code == http.StatusOK {
var cpObjResp CopyObjectResponse
if err = xml.Unmarshal(rec.Body.Bytes(), &cpObjResp); err != nil {
t.Fatalf("Test %d: %s: Failed to parse the CopyObjectResult response: <ERROR> %s", i+1, instanceType, err)
t.Fatalf("Test %d: %s: Failed to parse the CopyObjectResult response: <ERROR> %s", i, instanceType, err)
}
// See if the new object is formed.
// testing whether the copy was successful.
err = obj.GetObject(context.Background(), testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0], "", opts)
if err != nil {
t.Fatalf("Test %d: %s: Failed to fetch the copied object: <ERROR> %s", i+1, instanceType, err)
// Note that this goes directly to the file system,
// so encryption/compression may interfere at some point.
globalCompressConfigMu.Lock()
cfg := globalCompressConfig
globalCompressConfigMu.Unlock()
if !cfg.Enabled {
err = obj.GetObject(context.Background(), testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0], "", opts)
if err != nil {
t.Fatalf("Test %d: %s: Failed to fetch the copied object: <ERROR> %s", i, instanceType, err)
}
if !bytes.Equal(bytesData[0].byteData, buffers[0].Bytes()) {
t.Errorf("Test %d: %s: Data Mismatch: Data fetched back from the copied object doesn't match the original one.", i, instanceType)
}
buffers[0].Reset()
} else {
t.Log("object not validated due to compression")
}
if !bytes.Equal(bytesData[0].byteData, buffers[0].Bytes()) {
t.Errorf("Test %d: %s: Data Mismatch: Data fetched back from the copied object doesn't match the original one.", i+1, instanceType)
}
buffers[0].Reset()
}
// Verify response of the V2 signed HTTP request.
@ -2330,7 +2343,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
reqV2, err = newTestRequest(http.MethodPut, getCopyObjectURL("", testCase.bucketName, testCase.newObjectName), 0, nil)
if err != nil {
t.Fatalf("Test %d: Failed to create HTTP request for copy Object: <ERROR> %v", i+1, err)
t.Fatalf("Test %d: Failed to create HTTP request for copy Object: <ERROR> %v", i, err)
}
// "X-Amz-Copy-Source" header contains the information about the source bucket and the object to copied.
if testCase.copySourceHeader != "" {
@ -2366,7 +2379,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
// Call the ServeHTTP to execute the handler.
apiRouter.ServeHTTP(recV2, reqV2)
if recV2.Code != testCase.expectedRespStatus {
t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code)
t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i, instanceType, testCase.expectedRespStatus, recV2.Code)
}
}
@ -3304,7 +3317,7 @@ func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string
// when the request signature type is `streaming signature`.
func TestAPIPutObjectPartHandlerStreaming(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(t, testAPIPutObjectPartHandlerStreaming, []string{"NewMultipart", "PutObjectPart"})
ExecExtendedObjectLayerAPITest(t, testAPIPutObjectPartHandlerStreaming, []string{"NewMultipart", "PutObjectPart"})
}
func testAPIPutObjectPartHandlerStreaming(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@ -3392,7 +3405,7 @@ func testAPIPutObjectPartHandlerStreaming(obj ObjectLayer, instanceType, bucketN
// for variety of inputs.
func TestAPIPutObjectPartHandler(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(t, testAPIPutObjectPartHandler, []string{"PutObjectPart"})
ExecExtendedObjectLayerAPITest(t, testAPIPutObjectPartHandler, []string{"PutObjectPart"})
}
func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@ -3797,7 +3810,7 @@ func testAPIListObjectPartsHandlerPreSign(obj ObjectLayer, instanceType, bucketN
// for variety of success/failure cases.
func TestAPIListObjectPartsHandler(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(t, testAPIListObjectPartsHandler, []string{"ListObjectParts"})
ExecExtendedObjectLayerAPITest(t, testAPIListObjectPartsHandler, []string{"ListObjectParts"})
}
func testAPIListObjectPartsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,

109
cmd/object_api_suite_test.go
View File

@ -21,10 +21,12 @@ import (
"context"
"io"
"math/rand"
"os"
"strconv"
"testing"
humanize "github.com/dustin/go-humanize"
"github.com/dustin/go-humanize"
"github.com/minio/minio/cmd/crypto"
)
// Return pointer to testOneByteReadEOF{}
@ -68,10 +70,8 @@ func (r *testOneByteReadNoEOF) Read(p []byte) (n int, err error) {
return n, nil
}
type ObjectLayerAPISuite struct{}
// Wrapper for calling testMakeBucket for both Erasure and FS.
func (s *ObjectLayerAPISuite) TestMakeBucket(t *testing.T) {
func TestMakeBucket(t *testing.T) {
ExecObjectLayerTest(t, testMakeBucket)
}
@ -84,8 +84,8 @@ func testMakeBucket(obj ObjectLayer, instanceType string, t TestErrHandler) {
}
// Wrapper for calling testMultipartObjectCreation for both Erasure and FS.
func (s *ObjectLayerAPISuite) TestMultipartObjectCreation(t *testing.T) {
ExecObjectLayerTest(t, testMultipartObjectCreation)
func TestMultipartObjectCreation(t *testing.T) {
ExecExtendedObjectLayerTest(t, testMultipartObjectCreation)
}
// Tests validate creation of part files during Multipart operation.
@ -128,7 +128,7 @@ func testMultipartObjectCreation(obj ObjectLayer, instanceType string, t TestErr
}
// Wrapper for calling testMultipartObjectAbort for both Erasure and FS.
func (s *ObjectLayerAPISuite) TestMultipartObjectAbort(t *testing.T) {
func TestMultipartObjectAbort(t *testing.T) {
ExecObjectLayerTest(t, testMultipartObjectAbort)
}
@ -173,8 +173,8 @@ func testMultipartObjectAbort(obj ObjectLayer, instanceType string, t TestErrHan
}
// Wrapper for calling testMultipleObjectCreation for both Erasure and FS.
func (s *ObjectLayerAPISuite) TestMultipleObjectCreation(t *testing.T) {
ExecObjectLayerTest(t, testMultipleObjectCreation)
func TestMultipleObjectCreation(t *testing.T) {
ExecExtendedObjectLayerTest(t, testMultipleObjectCreation)
}
// Tests validate object creation.
@ -230,7 +230,7 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrH
}
// Wrapper for calling TestPaging for both Erasure and FS.
func (s *ObjectLayerAPISuite) TestPaging(t *testing.T) {
func TestPaging(t *testing.T) {
ExecObjectLayerTest(t, testPaging)
}
@ -434,7 +434,7 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
}
// Wrapper for calling testObjectOverwriteWorks for both Erasure and FS.
func (s *ObjectLayerAPISuite) TestObjectOverwriteWorks(t *testing.T) {
func TestObjectOverwriteWorks(t *testing.T) {
ExecObjectLayerTest(t, testObjectOverwriteWorks)
}
@ -471,7 +471,7 @@ func testObjectOverwriteWorks(obj ObjectLayer, instanceType string, t TestErrHan
}
// Wrapper for calling testNonExistantBucketOperations for both Erasure and FS.
func (s *ObjectLayerAPISuite) TestNonExistantBucketOperations(t *testing.T) {
func TestNonExistantBucketOperations(t *testing.T) {
ExecObjectLayerTest(t, testNonExistantBucketOperations)
}
@ -488,7 +488,7 @@ func testNonExistantBucketOperations(obj ObjectLayer, instanceType string, t Tes
}
// Wrapper for calling testBucketRecreateFails for both Erasure and FS.
func (s *ObjectLayerAPISuite) TestBucketRecreateFails(t *testing.T) {
func TestBucketRecreateFails(t *testing.T) {
ExecObjectLayerTest(t, testBucketRecreateFails)
}
@ -508,9 +508,68 @@ func testBucketRecreateFails(obj ObjectLayer, instanceType string, t TestErrHand
}
}
func execExtended(t *testing.T, fn func(t *testing.T)) {
// Exec with default settings...
globalCompressConfigMu.Lock()
globalCompressConfig.Enabled = false
globalCompressConfigMu.Unlock()
t.Run("default", func(t *testing.T) {
fn(t)
})
if testing.Short() {
return
}
// Enable compression and exec...
globalCompressConfigMu.Lock()
globalCompressConfig.Enabled = true
globalCompressConfig.MimeTypes = nil
globalCompressConfig.Extensions = nil
globalCompressConfigMu.Unlock()
t.Run("compressed", func(t *testing.T) {
fn(t)
})
globalAutoEncryption = true
os.Setenv("MINIO_KMS_MASTER_KEY", "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574")
defer os.Setenv("MINIO_KMS_MASTER_KEY", "")
var err error
GlobalKMS, err = crypto.NewKMS(crypto.KMSConfig{})
if err != nil {
t.Fatal(err)
}
t.Run("encrypted", func(t *testing.T) {
fn(t)
})
// Enable compression of encrypted and exec...
globalCompressConfigMu.Lock()
globalCompressConfig.AllowEncrypted = true
globalCompressConfigMu.Unlock()
t.Run("compressed+encrypted", func(t *testing.T) {
fn(t)
})
// Reset...
globalCompressConfigMu.Lock()
globalCompressConfig.Enabled = false
globalCompressConfig.AllowEncrypted = false
globalCompressConfigMu.Unlock()
globalAutoEncryption = false
}
// ExecExtendedObjectLayerTest will execute the tests with combinations of encrypted & compressed.
// This can be used to test functionality when reading and writing data.
func ExecExtendedObjectLayerTest(t *testing.T, objTest objTestType) {
execExtended(t, func(t *testing.T) {
ExecObjectLayerTest(t, objTest)
})
}
// Wrapper for calling testPutObject for both Erasure and FS.
func (s *ObjectLayerAPISuite) TestPutObject(t *testing.T) {
ExecObjectLayerTest(t, testPutObject)
func TestPutObject(t *testing.T) {
ExecExtendedObjectLayerTest(t, testPutObject)
}
// Tests validate PutObject without prefix.
@ -553,8 +612,8 @@ func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
}
// Wrapper for calling testPutObjectInSubdir for both Erasure and FS.
func (s *ObjectLayerAPISuite) TestPutObjectInSubdir(t *testing.T) {
ExecObjectLayerTest(t, testPutObjectInSubdir)
func TestPutObjectInSubdir(t *testing.T) {
ExecExtendedObjectLayerTest(t, testPutObjectInSubdir)
}
// Tests validate PutObject with subdirectory prefix.
@ -585,7 +644,7 @@ func testPutObjectInSubdir(obj ObjectLayer, instanceType string, t TestErrHandle
}
// Wrapper for calling testListBuckets for both Erasure and FS.
func (s *ObjectLayerAPISuite) TestListBuckets(t *testing.T) {
func TestListBuckets(t *testing.T) {
ExecObjectLayerTest(t, testListBuckets)
}
@ -644,7 +703,7 @@ func testListBuckets(obj ObjectLayer, instanceType string, t TestErrHandler) {
}
// Wrapper for calling testListBucketsOrder for both Erasure and FS.
func (s *ObjectLayerAPISuite) TestListBucketsOrder(t *testing.T) {
func TestListBucketsOrder(t *testing.T) {
ExecObjectLayerTest(t, testListBucketsOrder)
}
@ -678,7 +737,7 @@ func testListBucketsOrder(obj ObjectLayer, instanceType string, t TestErrHandler
}
// Wrapper for calling testListObjectsTestsForNonExistantBucket for both Erasure and FS.
func (s *ObjectLayerAPISuite) TestListObjectsTestsForNonExistantBucket(t *testing.T) {
func TestListObjectsTestsForNonExistantBucket(t *testing.T) {
ExecObjectLayerTest(t, testListObjectsTestsForNonExistantBucket)
}
@ -700,7 +759,7 @@ func testListObjectsTestsForNonExistantBucket(obj ObjectLayer, instanceType stri
}
// Wrapper for calling testNonExistantObjectInBucket for both Erasure and FS.
func (s *ObjectLayerAPISuite) TestNonExistantObjectInBucket(t *testing.T) {
func TestNonExistantObjectInBucket(t *testing.T) {
ExecObjectLayerTest(t, testNonExistantObjectInBucket)
}
@ -716,8 +775,8 @@ func testNonExistantObjectInBucket(obj ObjectLayer, instanceType string, t TestE
t.Fatalf("%s: Expected error but found nil", instanceType)
}
if isErrObjectNotFound(err) {
if err.Error() != "Object not found: bucket#dir1" {
t.Errorf("%s: Expected the Error message to be `%s`, but instead found `%s`", instanceType, "Object not found: bucket#dir1", err.Error())
if err.Error() != "Object not found: bucket/dir1" {
t.Errorf("%s: Expected the Error message to be `%s`, but instead found `%s`", instanceType, "Object not found: bucket/dir1", err.Error())
}
} else {
if err.Error() != "fails" {
@ -727,7 +786,7 @@ func testNonExistantObjectInBucket(obj ObjectLayer, instanceType string, t TestE
}
// Wrapper for calling testGetDirectoryReturnsObjectNotFound for both Erasure and FS.
func (s *ObjectLayerAPISuite) TestGetDirectoryReturnsObjectNotFound(t *testing.T) {
func TestGetDirectoryReturnsObjectNotFound(t *testing.T) {
ExecObjectLayerTest(t, testGetDirectoryReturnsObjectNotFound)
}
@ -770,7 +829,7 @@ func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string,
}
// Wrapper for calling testContentType for both Erasure and FS.
func (s *ObjectLayerAPISuite) TestContentType(t *testing.T) {
func TestContentType(t *testing.T) {
ExecObjectLayerTest(t, testContentType)
}

8
cmd/test-utils_test.go
View File

@ -1866,6 +1866,14 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
removeRoots(append(erasureDisks, fsDir))
}
// ExecExtendedObjectLayerTest will execute the tests with combinations of encrypted & compressed.
// This can be used to test functionality when reading and writing data.
func ExecExtendedObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints []string) {
execExtended(t, func(t *testing.T) {
ExecObjectLayerAPITest(t, objAPITest, endpoints)
})
}
// function to be passed to ExecObjectLayerAPITest, for executing object layr API handler tests.
type objAPITestType func(obj ObjectLayer, instanceType string, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T)

2
docs/bucket/versioning/DESIGN.md
View File

@ -26,7 +26,7 @@ Once the header is validated, we proceed to the actual data structure of the `xl
- LegacyObjectType (preserves existing deployments and older xl.json format)
- DeleteMarker (a versionId to capture the DELETE sequences implemented primarily for AWS spec compatibility)
A sample msgpack-JSON `xl.meta`, you can debug the content inside `xl.meta` using [xl-meta-to-json.go](https://github.com/minio/minio/blob/master/docs/bucket/versioning/xl-meta-to-json.go) program.
A sample msgpack-JSON `xl.meta`, you can debug the content inside `xl.meta` using [xl-meta.go](https://github.com/minio/minio/blob/master/docs/bucket/versioning/xl-meta.go) program.
```json
{
"Versions": [

9
docs/bucket/versioning/xl-meta-to-json.go → docs/bucket/versioning/xl-meta.go
View File

@ -56,11 +56,12 @@ GLOBAL FLAGS:
}
app.Action = func(c *cli.Context) error {
if !c.Args().Present() {
cli.ShowAppHelp(c)
return nil
files := c.Args()
if len(files) == 0 {
// If no args, assume xl.meta
files = []string{"xl.meta"}
}
for _, file := range c.Args() {
for _, file := range files {
var r io.Reader
switch file {
case "-":

84
docs/compression/README.md
View File

@ -1,10 +1,19 @@
# Compression Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io)
MinIO server allows streaming compression to ensure efficient disk space usage. Compression happens inflight, i.e objects are compressed before being written to disk(s). MinIO uses [`klauspost/compress/s2`](https://github.com/klauspost/compress/tree/master/s2) streaming compression due to its stability and performance.
MinIO server allows streaming compression to ensure efficient disk space usage.
Compression happens inflight, i.e objects are compressed before being written to disk(s).
MinIO uses [`klauspost/compress/s2`](https://github.com/klauspost/compress/tree/master/s2)
streaming compression due to its stability and performance.
This algorithm is specifically optimized for machine generated content. Write throughput is typically at least 300MB/s per CPU core. Decompression speed is typically at least 1GB/s.
This means that in cases where raw IO is below these numbers compression will not only reduce disk usage but also help increase system throughput.
Typically enabling compression on spinning disk systems will increase speed when the content can be compressed.
This algorithm is specifically optimized for machine generated content.
Write throughput is typically at least 500MB/s per CPU core,
and scales with the number of available CPU cores.
Decompression speed is typically at least 1GB/s.
This means that in cases where raw IO is below these numbers
compression will not only reduce disk usage but also help increase system throughput.
Typically, enabling compression on spinning disk systems
will increase speed when the content can be compressed.
## Get Started
@ -14,40 +23,71 @@ Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/docs/minio-quicksta
### 2. Run MinIO with compression
Compression can be enabled by updating the `compress` config settings for MinIO server config. Config `compress` settings take extensions and mime-types to be compressed.
Compression can be enabled by updating the `compress` config settings for MinIO server config.
Config `compress` settings take extensions and mime-types to be compressed.
```
$ mc admin config get myminio compression
```bash
~ mc admin config get myminio compression
compression extensions=".txt,.log,.csv,.json,.tar,.xml,.bin" mime_types="text/*,application/json,application/xml"
```
Default config includes most common highly compressible content extensions and mime-types.
```
$ mc admin config set myminio compression extensions=".pdf" mime_types="application/pdf"
```bash
~ mc admin config set myminio compression extensions=".pdf" mime_types="application/pdf"
```
To show help on setting compression config values.
```
```bash
~ mc admin config set myminio compression
```
To enable compression for all content, with default extensions and mime-types.
```
~ mc admin config set myminio compression enable="on"
To enable compression for all content, no matter the extension and content type
(except for the default excluded types) set BOTH extensions and mime types to empty.
```bash
~ mc admin config set myminio compression enable="on" extensions="" mime_types=""
```
The compression settings may also be set through environment variables. When set, environment variables override the defined `compress` config settings in the server config.
The compression settings may also be set through environment variables.
When set, environment variables override the defined `compress` config settings in the server config.
```bash
export MINIO_COMPRESS="on"
export MINIO_COMPRESS_EXTENSIONS=".pdf,.doc"
export MINIO_COMPRESS_MIME_TYPES="application/pdf"
export MINIO_COMPRESS_EXTENSIONS=".txt,.log,.csv,.json,.tar,.xml,.bin"
export MINIO_COMPRESS_MIME_TYPES="text/*,application/json,application/xml"
```
### 3. Note
### 3. Compression + Encryption
- Already compressed objects are not fit for compression since they do not have compressible patterns. Such objects do not produce efficient [`LZ compression`](https://en.wikipedia.org/wiki/LZ77_and_LZ78) which is a fitness factor for a lossless data compression. Below is a list of common files and content-types which are not suitable for compression.
Combining encryption and compression is not safe in all setups.
This is particularly so if the compression ratio of your content reveals information about it.
See [CRIME TLS](https://en.wikipedia.org/wiki/CRIME) as an example of this.
Therefore, compression is disabled when encrypting by default, and must be enabled separately.
Consult our security experts on [SUBNET](https://min.io/pricing) to help you evaluate if
your setup can use this feature combination safely.
To enable compression+encryption use:
```bash
~ mc admin config set myminio compression allow_encryption=on
```
Or alternatively through the environment variable `MINIO_COMPRESS_ALLOW_ENCRYPTION=on`.
### 4. Excluded Types
- Already compressed objects are not fit for compression since they do not have compressible patterns.
Such objects do not produce efficient [`LZ compression`](https://en.wikipedia.org/wiki/LZ77_and_LZ78)
which is a fitness factor for a lossless data compression.
Pre-compressed input typically compresses in excess of 2GiB/s per core,
so performance impact should be minimal even if precompressed data is re-compressed.
Decompressing incompressible data has no significant performance impact.
Below is a list of common files and content-types which are typically not suitable for compression.
- Extensions
@ -72,15 +112,17 @@ export MINIO_COMPRESS_MIME_TYPES="application/pdf"
| `application/x-compress` |
| `application/x-xz` |
All files with these extensions and mime types are excluded from compression, even if compression is enabled for all types.
All files with these extensions and mime types are excluded from compression,
even if compression is enabled for all types.
- MinIO does not support encryption with compression because compression and encryption together potentially enables room for side channel attacks like [`CRIME and BREACH`](https://blog.minio.io/c-e-compression-encryption-cb6b7f04a369)
### 5. Notes
- MinIO does not support compression for Gateway (Azure/GCS/NAS) implementations.
## To test the setup
To test this setup, practice put calls to the server using `mc` and use `mc ls` on the data directory to view the size of the object.
To test this setup, practice put calls to the server using `mc` and use `mc ls` on
the data directory to view the size of the object.
## Explore Further