maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

etag: compute ETag as MD5 for compressed single-part objects (#12375)

Browse source 
This commit fixes a bug causing the MinIO server to compute
the ETag of a single-part object as MD5 of the compressed
content - not as MD5 of the actual content.

This usually does not affect clients since the MinIO appended
a `-1` to indicate that the ETag belongs to a multipart object.
However, this behavior was problematic since:
 - A S3 client being very strict should reject such an ETag since
   the client uploaded the object via single-part API but got
   a multipart ETag that is not the content MD5.
 - The MinIO server leaks (via the ETag) that it compressed the
   object.

This commit addresses both cases. Now, the MinIO server returns
an ETag equal to the content MD5 for single-part objects that got
compressed.

Signed-off-by: Andreas Auernhammer <aead@mail.de>
This commit is contained in:
Andreas Auernhammer 2021-05-27 17:18:41 +02:00 committed by GitHub
parent e63908c391
commit e8a12cbfdd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 17 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
cmd/object-handlers.go
View file

@ -1092,6 +1092,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
compressMetadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
compressMetadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10)
reader = etag.NewReader(reader, nil)
s2c := newS2CompressReader(reader, actualSize)
defer s2c.Close()
reader = etag.Wrap(s2c, reader)
@ -1696,8 +1697,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
return
}
switch kind, encrypted := crypto.IsEncrypted(objInfo.UserDefined); {
case encrypted:
if kind, encrypted := crypto.IsEncrypted(objInfo.UserDefined); encrypted {
switch kind {
case crypto.S3:
w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
@ -1719,10 +1719,6 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
objInfo.ETag = objInfo.ETag[len(objInfo.ETag)-32:]
}
}
case objInfo.IsCompressed():
if !strings.HasSuffix(objInfo.ETag, "-1") {
objInfo.ETag = objInfo.ETag + "-1"
}
}
if replicate, sync := mustReplicate(ctx, r, bucket, object, metadata, "", false); replicate {
scheduleReplication(ctx, objInfo.Clone(), objectAPI, sync, replication.ObjectReplicationType)

2
pkg/etag/etag_test.go
View file

@ -25,6 +25,8 @@ import (
"testing"
)
var _ Tagger = Wrap(nil, nil).(Tagger) // runtime check that wrapReader implements Tagger
var parseTests = []struct {
String string
ETag ETag

4
pkg/etag/reader.go
View file

@ -34,8 +34,10 @@ type wrapReader struct {
Tagger
}
var _ Tagger = wrapReader{} // compiler check
// ETag returns the ETag of the underlying Tagger.
func (r *wrapReader) ETag() ETag {
func (r wrapReader) ETag() ETag {
if r.Tagger == nil {
return nil
}

13
pkg/hash/reader.go
View file

@ -110,15 +110,22 @@ func NewReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize i
return r, nil
}
var hash hash.Hash
if size >= 0 {
src = io.LimitReader(src, size)
r := io.LimitReader(src, size)
if _, ok := src.(etag.Tagger); !ok {
src = etag.NewReader(r, etag.ETag(MD5))
} else {
src = etag.Wrap(r, src)
}
} else if _, ok := src.(etag.Tagger); !ok {
src = etag.NewReader(src, etag.ETag(MD5))
}
var hash hash.Hash
if len(SHA256) != 0 {
hash = newSHA256()
}
return &Reader{
src: etag.NewReader(src, etag.ETag(MD5)),
src: src,
size: size,
actualSize: actualSize,
checksum: etag.ETag(MD5),