maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

sha256: Verify sha256 along with md5sum, signature is verified on the request early. (#2813)

Browse source
This commit is contained in:
Krishna Srinivas 2016-10-03 04:21:49 +05:30 committed by Harshavardhana
parent b5a6dd1395
commit 61a18ed48f
30 changed files with 341 additions and 166 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmd/api-errors.go
View file

@ -617,6 +617,8 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
apiErr = ErrNoSuchUpload
case PartTooSmall:
apiErr = ErrEntityTooSmall
case SHA256Mismatch:
apiErr = ErrContentSHA256Mismatch
default:
apiErr = ErrInternalError
}

15
cmd/auth-handler.go
View file

@ -130,6 +130,21 @@ func isReqAuthenticatedV2(r *http.Request) (s3Error APIErrorCode) {
return doesPresignV2SignatureMatch(r)
}
func reqSignatureV4Verify(r *http.Request) (s3Error APIErrorCode) {
sha256sum := r.Header.Get("X-Amz-Content-Sha256")
// Skips calculating sha256 on the payload on server,
// if client requested for it.
if skipContentSha256Cksum(r) {
sha256sum = unsignedPayload
}
if isRequestSignatureV4(r) {
return doesSignatureMatch(sha256sum, r, serverConfig.GetRegion())
} else if isRequestPresignedSignatureV4(r) {
return doesPresignedSignatureMatch(sha256sum, r, serverConfig.GetRegion())
}
return ErrAccessDenied
}
// Verify if request has valid AWS Signature Version '4'.
func isReqAuthenticated(r *http.Request, region string) (s3Error APIErrorCode) {
if r == nil {

15
cmd/benchmark-utils_test.go
View file

@ -62,13 +62,14 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
hasher.Write([]byte(textData))
metadata := make(map[string]string)
metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
sha256sum := ""
// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
b.ReportAllocs()
// the actual benchmark for PutObject starts here. Reset the benchmark timer.
b.ResetTimer()
for i := 0; i < b.N; i++ {
// insert the object.
objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata)
objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
if err != nil {
b.Fatal(err)
}
@ -107,6 +108,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
hasher.Write([]byte(textData))
metadata := make(map[string]string)
metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
sha256sum := ""
uploadID, err = obj.NewMultipartUpload(bucket, object, metadata)
if err != nil {
b.Fatal(err)
@ -130,7 +132,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
hasher.Write([]byte(textPartData))
metadata := make(map[string]string)
metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
md5Sum, err = obj.PutObjectPart(bucket, object, uploadID, j, int64(len(textPartData)), bytes.NewBuffer(textPartData), metadata["md5Sum"])
md5Sum, err = obj.PutObjectPart(bucket, object, uploadID, j, int64(len(textPartData)), bytes.NewBuffer(textPartData), metadata["md5Sum"], sha256sum)
if err != nil {
b.Fatal(err)
}
@ -194,6 +196,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
b.Fatal(err)
}
sha256sum := ""
for i := 0; i < 10; i++ {
// get text data generated for number of bytes equal to object size.
textData := generateBytesData(objSize)
@ -206,7 +209,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
// insert the object.
var objInfo ObjectInfo
objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata)
objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
if err != nil {
b.Fatal(err)
}
@ -294,6 +297,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
hasher.Write([]byte(textData))
metadata := make(map[string]string)
metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
sha256sum := ""
// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
b.ReportAllocs()
// the actual benchmark for PutObject starts here. Reset the benchmark timer.
@ -303,7 +307,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
i := 0
for pb.Next() {
// insert the object.
objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata)
objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
if err != nil {
b.Fatal(err)
}
@ -340,9 +344,10 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
hasher.Write([]byte(textData))
metadata := make(map[string]string)
metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
sha256sum := ""
// insert the object.
var objInfo ObjectInfo
objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata)
objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
if err != nil {
b.Fatal(err)
}

4
cmd/bucket-handlers.go
View file

@ -459,7 +459,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
metadata := make(map[string]string)
// Nothing to store right now.
objInfo, err := objectAPI.PutObject(bucket, object, -1, fileBody, metadata)
sha256sum := ""
objInfo, err := objectAPI.PutObject(bucket, object, -1, fileBody, metadata, sha256sum)
if err != nil {
errorIf(err, "Unable to create object.")
writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)

4
cmd/bucket-notification-handlers.go
View file

@ -146,7 +146,9 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
// Proceed to save notification configuration.
notificationConfigPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig)
_, err = objectAPI.PutObject(minioMetaBucket, notificationConfigPath, bufferSize, bytes.NewReader(buffer.Bytes()), nil)
sha256sum := ""
var metadata map[string]string
_, err = objectAPI.PutObject(minioMetaBucket, notificationConfigPath, bufferSize, bytes.NewReader(buffer.Bytes()), metadata, sha256sum)
if err != nil {
errorIf(err, "Unable to write bucket notification configuration.")
writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)

4
cmd/bucket-policy-migrate.go
View file

@ -74,8 +74,10 @@ func migrateBucketPolicyConfig(objAPI ObjectLayer) error {
policyBytes, err := ioutil.ReadFile(policyPath)
fatalIf(err, "Unable to read bucket policy to migrate bucket policy", policyPath)
newPolicyPath := retainSlash(bucketConfigPrefix) + retainSlash(bucketName) + policyJSON
var metadata map[string]string
sha256sum := ""
// Erasure code the policy config to all the disks.
_, err = objAPI.PutObject(minioMetaBucket, newPolicyPath, int64(len(policyBytes)), bytes.NewReader(policyBytes), nil)
_, err = objAPI.PutObject(minioMetaBucket, newPolicyPath, int64(len(policyBytes)), bytes.NewReader(policyBytes), metadata, sha256sum)
fatalIf(err, "Unable to write bucket policy during migration.", newPolicyPath)
return nil
}

3
cmd/bucket-policy.go
View file

@ -210,7 +210,8 @@ func writeBucketPolicy(bucket string, objAPI ObjectLayer, reader io.Reader, size
}
policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON)
if _, err := objAPI.PutObject(minioMetaBucket, policyPath, size, reader, nil); err != nil {
sha256sum := ""
if _, err := objAPI.PutObject(minioMetaBucket, policyPath, size, reader, nil, sha256sum); err != nil {
errorIf(err, "Unable to set policy for the bucket %s", bucket)
return errorCause(err)
}

5
cmd/controller_test.go
View file

@ -330,7 +330,7 @@ func (s *TestRPCControllerSuite) testControllerHealObjectH(t *testing.T) {
datum := strings.NewReader("a")
_, err = s.testServer.Obj.PutObject("testbucket", "testobject", 1,
datum, nil)
datum, nil, "")
if err != nil {
t.Fatalf("Controller.HealObjectH - put object failed with <ERROR> %s",
err.Error())
@ -373,8 +373,7 @@ func (s *TestRPCControllerSuite) testControllerListObjectsHealH(t *testing.T) {
r := strings.NewReader("0")
_, err = s.testServer.Obj.PutObject(
"testbucket", "testObj-0", 1, r, nil,
)
"testbucket", "testObj-0", 1, r, nil, "")
if err != nil {
t.Fatalf("Controller.ListObjectsHealH - object creation failed - %s",
err.Error())

9
cmd/format-config-v1_test.go
View file

@ -225,8 +225,9 @@ func prepareFormatXLHealFreshDisks(obj ObjectLayer) ([]StorageAPI, error) {
bucket := "bucket"
object := "object"
sha256sum := ""
_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil)
_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
if err != nil {
return []StorageAPI{}, err
}
@ -349,8 +350,9 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) {
bucket := "bucket"
object := "object"
sha256sum := ""
_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil)
_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
if err != nil {
t.Fatal(err)
}
@ -421,8 +423,9 @@ func TestFormatXLReorderByInspection(t *testing.T) {
bucket := "bucket"
object := "object"
sha256sum := ""
_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil)
_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
if err != nil {
t.Fatal(err)
}

6
cmd/fs-v1-metadata_test.go
View file

@ -83,8 +83,9 @@ func TestReadFSMetadata(t *testing.T) {
if err = obj.MakeBucket(bucketName); err != nil {
t.Fatal("Unexpected err: ", err)
}
sha256sum := ""
if _, err = obj.PutObject(bucketName, objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")),
map[string]string{"X-Amz-Meta-AppId": "a"}); err != nil {
map[string]string{"X-Amz-Meta-AppId": "a"}, sha256sum); err != nil {
t.Fatal("Unexpected err: ", err)
}
@ -130,8 +131,9 @@ func TestWriteFSMetadata(t *testing.T) {
if err = obj.MakeBucket(bucketName); err != nil {
t.Fatal("Unexpected err: ", err)
}
sha256sum := ""
if _, err = obj.PutObject(bucketName, objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")),
map[string]string{"X-Amz-Meta-AppId": "a"}); err != nil {
map[string]string{"X-Amz-Meta-AppId": "a"}, sha256sum); err != nil {
t.Fatal("Unexpected err: ", err)
}

27
cmd/fs-v1-multipart.go
View file

@ -18,8 +18,10 @@ package cmd
import (
"crypto/md5"
"crypto/sha256"
"encoding/hex"
"fmt"
"hash"
"io"
"path"
"strconv"
@ -392,7 +394,7 @@ func appendParts(disk StorageAPI, bucket, object, uploadID, opsID string) {
// an ongoing multipart transaction. Internally incoming data is
// written to '.minio.sys/tmp' location and safely renamed to
// '.minio.sys/multipart' for reach parts.
func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, error) {
func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (string, error) {
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
return "", traceError(BucketNameInvalid{Bucket: bucket})
@ -424,6 +426,14 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
// Initialize md5 writer.
md5Writer := md5.New()
hashWriters := []io.Writer{md5Writer}
var sha256Writer hash.Hash
if sha256sum != "" {
sha256Writer = sha256.New()
hashWriters = append(hashWriters, sha256Writer)
}
multiWriter := io.MultiWriter(hashWriters...)
// Limit the reader to its provided size if specified.
var limitDataReader io.Reader
if size > 0 {
@ -434,7 +444,7 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
limitDataReader = data
}
teeReader := io.TeeReader(limitDataReader, md5Writer)
teeReader := io.TeeReader(limitDataReader, multiWriter)
bufSize := int64(readSizeV1)
if size > 0 && bufSize > size {
bufSize = size
@ -467,12 +477,23 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
if newMD5Hex != md5Hex {
// MD5 mismatch, delete the temporary object.
fs.storage.DeleteFile(minioMetaBucket, tmpPartPath)
// Returns md5 mismatch.
return "", traceError(BadDigest{md5Hex, newMD5Hex})
}
}
if sha256sum != "" {
newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil))
if newSHA256sum != sha256sum {
// SHA256 mismatch, delete the temporary object.
fs.storage.DeleteFile(minioMetaBucket, tmpPartPath)
return "", traceError(SHA256Mismatch{})
}
}
// get a random ID for lock instrumentation.
// generates random string on setting MINIO_DEBUG=lock, else returns empty string.
// used for instrumentation on locks.
opsID = getOpsID()
// Hold write lock as we are updating fs.json

9
cmd/fs-v1-multipart_test.go
View file

@ -90,13 +90,14 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
md5Writer := md5.New()
md5Writer.Write(data)
md5Hex := hex.EncodeToString(md5Writer.Sum(nil))
sha256sum := ""
// Test with faulty disk
fsStorage := fs.storage.(*posix)
for i := 1; i <= 7; i++ {
// Faulty disk generates errFaultyDisk at 'i' storage api call number
fs.storage = newNaughtyDisk(fsStorage, map[int]error{i: errFaultyDisk}, nil)
if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, dataLen, bytes.NewReader(data), md5Hex); errorCause(err) != errFaultyDisk {
if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, dataLen, bytes.NewReader(data), md5Hex, sha256sum); errorCause(err) != errFaultyDisk {
switch i {
case 1:
if !isSameType(errorCause(err), BucketNotFound{}) {
@ -140,8 +141,9 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
md5Writer := md5.New()
md5Writer.Write(data)
md5Hex := hex.EncodeToString(md5Writer.Sum(nil))
sha256sum := ""
if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, 5, bytes.NewReader(data), md5Hex); err != nil {
if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, 5, bytes.NewReader(data), md5Hex, sha256sum); err != nil {
t.Fatal("Unexpected error ", err)
}
@ -195,8 +197,9 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
md5Writer := md5.New()
md5Writer.Write(data)
md5Hex := hex.EncodeToString(md5Writer.Sum(nil))
sha256sum := ""
if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, 5, bytes.NewReader(data), md5Hex); err != nil {
if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, 5, bytes.NewReader(data), md5Hex, sha256sum); err != nil {
t.Fatal("Unexpected error ", err)
}

24
cmd/fs-v1.go
View file

@ -18,8 +18,10 @@ package cmd
import (
"crypto/md5"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"hash"
"io"
"os"
"path"
@ -368,7 +370,7 @@ func (fs fsObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
}
// PutObject - create an object.
func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) {
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
return ObjectInfo{}, traceError(BucketNameInvalid{Bucket: bucket})
@ -394,6 +396,15 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
// Initialize md5 writer.
md5Writer := md5.New()
hashWriters := []io.Writer{md5Writer}
var sha256Writer hash.Hash
if sha256sum != "" {
sha256Writer = sha256.New()
hashWriters = append(hashWriters, sha256Writer)
}
multiWriter := io.MultiWriter(hashWriters...)
// Limit the reader to its provided size if specified.
var limitDataReader io.Reader
if size > 0 {
@ -417,7 +428,7 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
bufSize = size
}
buf := make([]byte, int(bufSize))
teeReader := io.TeeReader(limitDataReader, md5Writer)
teeReader := io.TeeReader(limitDataReader, multiWriter)
var bytesWritten int64
bytesWritten, err = fsCreateFile(fs.storage, teeReader, buf, minioMetaBucket, tempObj)
if err != nil {
@ -460,6 +471,15 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
}
}
if sha256sum != "" {
newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil))
if newSHA256sum != sha256sum {
// SHA256 mismatch, delete the temporary object.
fs.storage.DeleteFile(minioMetaBucket, tempObj)
return ObjectInfo{}, traceError(SHA256Mismatch{})
}
}
// Entire object was written to the temp location, now it's safe to rename it to the actual location.
err = fs.storage.RenameFile(minioMetaBucket, tempObj, bucket, object)
if err != nil {

6
cmd/fs-v1_test.go
View file

@ -84,7 +84,8 @@ func TestFSShutdown(t *testing.T) {
objectContent := "12345"
obj.MakeBucket(bucketName)
obj.PutObject(bucketName, objectName, int64(len(objectContent)), bytes.NewReader([]byte(objectContent)), nil)
sha256sum := ""
obj.PutObject(bucketName, objectName, int64(len(objectContent)), bytes.NewReader([]byte(objectContent)), nil, sha256sum)
// Test Shutdown with regular conditions
if err := fs.Shutdown(); err != nil {
@ -187,7 +188,8 @@ func TestFSDeleteObject(t *testing.T) {
objectName := "object"
obj.MakeBucket(bucketName)
obj.PutObject(bucketName, objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil)
sha256sum := ""
obj.PutObject(bucketName, objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
// Test with invalid bucket name
if err := fs.DeleteObject("fo", objectName); !isSameType(errorCause(err), BucketNameInvalid{}) {

6
cmd/object-api-getobject_test.go
View file

@ -61,10 +61,11 @@ func testGetObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
// case - 1.
{bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)},
}
sha256sum := ""
// iterate through the above set of inputs and upkoad the object.
for i, input := range putObjectInputs {
// uploading the object.
_, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData)
_, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData, sha256sum)
// if object upload fails stop the test.
if err != nil {
t.Fatalf("Put Object case %d: Error uploading object: <ERROR> %v", i+1, err)
@ -212,10 +213,11 @@ func testGetObjectDiskNotFound(obj ObjectLayer, instanceType string, disks []str
// case - 1.
{bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)},
}
sha256sum := ""
// iterate through the above set of inputs and upkoad the object.
for i, input := range putObjectInputs {
// uploading the object.
_, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData)
_, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData, sha256sum)
// if object upload fails stop the test.
if err != nil {
t.Fatalf("Put Object case %d: Error uploading object: <ERROR> %v", i+1, err)

3
cmd/object-api-getobjectinfo_test.go
View file

@ -33,7 +33,8 @@ func testGetObjectInfo(obj ObjectLayer, instanceType string, t TestErrHandler) {
if err != nil {
t.Fatalf("%s : %s", instanceType, err.Error())
}
_, err = obj.PutObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil)
sha256sum := ""
_, err = obj.PutObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil, sha256sum)
if err != nil {
t.Fatalf("%s : %s", instanceType, err.Error())
}

6
cmd/object-api-listobjects_test.go
View file

@ -62,8 +62,9 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
{"obj1", "obj1"},
{"obj2", "obj2"},
}
sha256sum := ""
for _, object := range testObjects {
_, err = obj.PutObject(testBuckets[0], object.name, int64(len(object.content)), bytes.NewBufferString(object.content), nil)
_, err = obj.PutObject(testBuckets[0], object.name, int64(len(object.content)), bytes.NewBufferString(object.content), nil, sha256sum)
if err != nil {
t.Fatalf("%s : %s", instanceType, err.Error())
}
@ -583,9 +584,10 @@ func BenchmarkListObjects(b *testing.B) {
b.Fatal(err)
}
sha256sum := ""
for i := 0; i < 20000; i++ {
key := "obj" + strconv.Itoa(i)
_, err = obj.PutObject("ls-benchmark-bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
_, err = obj.PutObject("ls-benchmark-bucket", key, int64(len(key)), bytes.NewBufferString(key), nil, sha256sum)
if err != nil {
b.Fatal(err)
}

65
cmd/object-api-multipart_test.go
View file

@ -214,9 +214,10 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [
{bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("mnop")), "e132e96a5ddad6da8b07bba6f6131fef"},
{bucketNames[0], objectNames[0], uploadIDs[0], 5, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("mnop")), "e132e96a5ddad6da8b07bba6f6131fef"},
}
sha256sum := ""
// Iterating over creatPartCases to generate multipart chunks.
for _, testCase := range createPartCases {
_, err = obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5)
_, err = obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5, sha256sum)
if err != nil {
t.Fatalf("%s : %s", instanceType, err.Error())
}
@ -230,7 +231,7 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [
// Object part upload should fail with quorum not available.
testCase := createPartCases[len(createPartCases)-1]
_, err = obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5)
_, err = obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5, sha256sum)
if err == nil {
t.Fatalf("Test %s: expected to fail but passed instead", instanceType)
}
@ -279,6 +280,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
PartID int
inputReaderData string
inputMd5 string
inputSHA256 string
intputDataSize int64
// flag indicating whether the test should pass.
shouldPass bool
@ -288,60 +290,63 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
}{
// Test case 1-4.
// Cases with invalid bucket name.
{".test", "obj", "", 1, "", "", 0, false, "", fmt.Errorf("%s", "Bucket name invalid: .test")},
{"------", "obj", "", 1, "", "", 0, false, "", fmt.Errorf("%s", "Bucket name invalid: ------")},
{"$this-is-not-valid-too", "obj", "", 1, "", "", 0, false, "",
{".test", "obj", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Bucket name invalid: .test")},
{"------", "obj", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Bucket name invalid: ------")},
{"$this-is-not-valid-too", "obj", "", 1, "", "", "", 0, false, "",
fmt.Errorf("%s", "Bucket name invalid: $this-is-not-valid-too")},
{"a", "obj", "", 1, "", "", 0, false, "", fmt.Errorf("%s", "Bucket name invalid: a")},
{"a", "obj", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Bucket name invalid: a")},
// Test case - 5.
// Case with invalid object names.
{bucket, "", "", 1, "", "", 0, false, "", fmt.Errorf("%s", "Object name invalid: minio-bucket#")},
{bucket, "", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Object name invalid: minio-bucket#")},
// Test case - 6.
// Valid object and bucket names but non-existent bucket.
{"abc", "def", "", 1, "", "", 0, false, "", fmt.Errorf("%s", "Bucket not found: abc")},
{"abc", "def", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Bucket not found: abc")},
// Test Case - 7.
// Existing bucket, but using a bucket on which NewMultipartUpload is not Initiated.
{"unused-bucket", "def", "xyz", 1, "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id xyz")},
{"unused-bucket", "def", "xyz", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id xyz")},
// Test Case - 8.
// Existing bucket, object name different from which NewMultipartUpload is constructed from.
// Expecting "Invalid upload id".
{bucket, "def", "xyz", 1, "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id xyz")},
{bucket, "def", "xyz", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id xyz")},
// Test Case - 9.
// Existing bucket, bucket and object name are the ones from which NewMultipartUpload is constructed from.
// But the uploadID is invalid.
// Expecting "Invalid upload id".
{bucket, object, "xyz", 1, "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id xyz")},
{bucket, object, "xyz", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id xyz")},
// Test Case - 10.
// Case with valid UploadID, existing bucket name.
// But using the bucket name from which NewMultipartUpload is not constructed from.
{"unused-bucket", object, uploadID, 1, "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id "+uploadID)},
{"unused-bucket", object, uploadID, 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id "+uploadID)},
// Test Case - 11.
// Case with valid UploadID, existing bucket name.
// But using the object name from which NewMultipartUpload is not constructed from.
{bucket, "none-object", uploadID, 1, "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id "+uploadID)},
{bucket, "none-object", uploadID, 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id "+uploadID)},
// Test case - 12.
// Input to replicate Md5 mismatch.
{bucket, object, uploadID, 1, "", "a35", 0, false, "",
{bucket, object, uploadID, 1, "", "a35", "", 0, false, "",
fmt.Errorf("%s", "Bad digest: Expected a35 is not valid with what we calculated "+"d41d8cd98f00b204e9800998ecf8427e")},
// Test case - 13.
// Input with size more than the size of actual data inside the reader.
{bucket, object, uploadID, 1, "abcd", "a35", int64(len("abcd") + 1), false, "",
IncompleteBody{}},
// When incorrect sha256 is provided.
{bucket, object, uploadID, 1, "", "", "incorrect-sha256", 0, false, "", SHA256Mismatch{}},
// Test case - 14.
// Input with size more than the size of actual data inside the reader.
{bucket, object, uploadID, 1, "abcd", "a35", "", int64(len("abcd") + 1), false, "", IncompleteBody{}},
// Test case - 15.
// Input with size less than the size of actual data inside the reader.
{bucket, object, uploadID, 1, "abcd", "a35", int64(len("abcd") - 1), false, "",
{bucket, object, uploadID, 1, "abcd", "a35", "", int64(len("abcd") - 1), false, "",
fmt.Errorf("%s", "Bad digest: Expected a35 is not valid with what we calculated 900150983cd24fb0d6963f7d28e17f72")},
// Test case - 15-18.
// Test case - 16-19.
// Validating for success cases.
{bucket, object, uploadID, 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), true, "", nil},
{bucket, object, uploadID, 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh")), true, "", nil},
{bucket, object, uploadID, 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd")), true, "", nil},
{bucket, object, uploadID, 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd")), true, "", nil},
{bucket, object, uploadID, 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", int64(len("abcd")), true, "", nil},
{bucket, object, uploadID, 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", "e5e088a0b66163a0a26a5e053d2a4496dc16ab6e0e3dd1adf2d16aa84a078c9d", int64(len("efgh")), true, "", nil},
{bucket, object, uploadID, 3, "ijkl", "09a0877d04abf8759f99adec02baf579", "005c19658919186b85618c5870463eec8d9b8c1a9d00208a5352891ba5bbe086", int64(len("abcd")), true, "", nil},
{bucket, object, uploadID, 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", "f1afc31479522d6cff1ed068f93998f05a8cd3b22f5c37d7f307084f62d1d270", int64(len("abcd")), true, "", nil},
}
// Validate all the test cases.
for i, testCase := range testCases {
actualMd5Hex, actualErr := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5)
actualMd5Hex, actualErr := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5, testCase.inputSHA256)
// All are test cases above are expected to fail.
if actualErr != nil && testCase.shouldPass {
t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", i+1, instanceType, actualErr.Error())
@ -472,9 +477,10 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
{bucketNames[2], objectNames[4], uploadIDs[8], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
{bucketNames[2], objectNames[5], uploadIDs[9], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
}
sha256sum := ""
// Iterating over creatPartCases to generate multipart chunks.
for _, testCase := range createPartCases {
_, err := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5)
_, err := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5, sha256sum)
if err != nil {
t.Fatalf("%s : %s", instanceType, err.Error())
}
@ -1319,9 +1325,10 @@ func testListObjectPartsDiskNotFound(obj ObjectLayer, instanceType string, disks
{bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd")), "09a0877d04abf8759f99adec02baf579"},
{bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd")), "e132e96a5ddad6da8b07bba6f6131fef"},
}
sha256sum := ""
// Iterating over creatPartCases to generate multipart chunks.
for _, testCase := range createPartCases {
_, err := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5)
_, err := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5, sha256sum)
if err != nil {
t.Fatalf("%s : %s", instanceType, err.Error())
}
@ -1558,9 +1565,10 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler)
{bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd")), "09a0877d04abf8759f99adec02baf579"},
{bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd")), "e132e96a5ddad6da8b07bba6f6131fef"},
}
sha256sum := ""
// Iterating over creatPartCases to generate multipart chunks.
for _, testCase := range createPartCases {
_, err := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5)
_, err := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5, sha256sum)
if err != nil {
t.Fatalf("%s : %s", instanceType, err.Error())
}
@ -1805,9 +1813,10 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T
{bucketNames[0], objectNames[0], uploadIDs[0], 5, string(validPart), validPartMD5, int64(len(string(validPart)))},
{bucketNames[0], objectNames[0], uploadIDs[0], 6, string(validPart), validPartMD5, int64(len(string(validPart)))},
}
sha256sum := ""
// Iterating over creatPartCases to generate multipart chunks.
for _, part := range parts {
_, err = obj.PutObjectPart(part.bucketName, part.objName, part.uploadID, part.PartID, part.intputDataSize, bytes.NewBufferString(part.inputReaderData), part.inputMd5)
_, err = obj.PutObjectPart(part.bucketName, part.objName, part.uploadID, part.PartID, part.intputDataSize, bytes.NewBufferString(part.inputReaderData), part.inputMd5, sha256sum)
if err != nil {
t.Fatalf("%s : %s", instanceType, err)
}

99
cmd/object-api-putobject_test.go
View file

@ -76,6 +76,7 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl
objName string
inputData []byte
inputMeta map[string]string
inputSHA256 string
intputDataSize int64
// expected error output.
expectedMd5 string
@ -83,79 +84,83 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl
}{
// Test case 1-4.
// Cases with invalid bucket name.
{".test", "obj", []byte(""), nil, 0, "", BucketNameInvalid{Bucket: ".test"}},
{"------", "obj", []byte(""), nil, 0, "", BucketNameInvalid{Bucket: "------"}},
{"$this-is-not-valid-too", "obj", []byte(""), nil, 0, "",
{".test", "obj", []byte(""), nil, "", 0, "", BucketNameInvalid{Bucket: ".test"}},
{"------", "obj", []byte(""), nil, "", 0, "", BucketNameInvalid{Bucket: "------"}},
{"$this-is-not-valid-too", "obj", []byte(""), nil, "", 0, "",
BucketNameInvalid{Bucket: "$this-is-not-valid-too"}},
{"a", "obj", []byte(""), nil, 0, "", BucketNameInvalid{Bucket: "a"}},
{"a", "obj", []byte(""), nil, "", 0, "", BucketNameInvalid{Bucket: "a"}},
// Test case - 5.
// Case with invalid object names.
{bucket, "", []byte(""), nil, 0, "", ObjectNameInvalid{Bucket: bucket, Object: ""}},
{bucket, "", []byte(""), nil, "", 0, "", ObjectNameInvalid{Bucket: bucket, Object: ""}},
// Test case - 6.
// Valid object and bucket names but non-existent bucket.
{"abc", "def", []byte(""), nil, 0, "", BucketNotFound{Bucket: "abc"}},
{"abc", "def", []byte(""), nil, "", 0, "", BucketNotFound{Bucket: "abc"}},
// Test case - 7.
// Input to replicate Md5 mismatch.
{bucket, object, []byte(""), map[string]string{"md5Sum": "a35"}, 0, "",
{bucket, object, []byte(""), map[string]string{"md5Sum": "a35"}, "", 0, "",
BadDigest{ExpectedMD5: "a35", CalculatedMD5: "d41d8cd98f00b204e9800998ecf8427e"}},
// Test case - 8.
// Input with size more than the size of actual data inside the reader.
{bucket, object, []byte("abcd"), map[string]string{"md5Sum": "a35"}, int64(len("abcd") + 1), "",
IncompleteBody{}},
// With incorrect sha256.
{bucket, object, []byte("abcd"), map[string]string{"md5Sum": "e2fc714c4727ee9395f324cd2e7f331f"}, "incorrect-sha256", int64(len("abcd")), "", SHA256Mismatch{}},
// Test case - 9.
// Input with size more than the size of actual data inside the reader.
{bucket, object, []byte("abcd"), map[string]string{"md5Sum": "a35"}, "", int64(len("abcd") + 1), "",
IncompleteBody{}},
// Test case - 10.
// Input with size less than the size of actual data inside the reader.
{bucket, object, []byte("abcd"), map[string]string{"md5Sum": "a35"}, int64(len("abcd") - 1), "",
{bucket, object, []byte("abcd"), map[string]string{"md5Sum": "a35"}, "", int64(len("abcd") - 1), "",
BadDigest{ExpectedMD5: "a35", CalculatedMD5: "900150983cd24fb0d6963f7d28e17f72"}},
// Test case - 10-13.
// Test case - 11-14.
// Validating for success cases.
{bucket, object, []byte("abcd"), map[string]string{"md5Sum": "e2fc714c4727ee9395f324cd2e7f331f"}, int64(len("abcd")), "", nil},
{bucket, object, []byte("efgh"), map[string]string{"md5Sum": "1f7690ebdd9b4caf8fab49ca1757bf27"}, int64(len("efgh")), "", nil},
{bucket, object, []byte("ijkl"), map[string]string{"md5Sum": "09a0877d04abf8759f99adec02baf579"}, int64(len("ijkl")), "", nil},
{bucket, object, []byte("mnop"), map[string]string{"md5Sum": "e132e96a5ddad6da8b07bba6f6131fef"}, int64(len("mnop")), "", nil},
{bucket, object, []byte("abcd"), map[string]string{"md5Sum": "e2fc714c4727ee9395f324cd2e7f331f"}, "", int64(len("abcd")), "", nil},
{bucket, object, []byte("efgh"), map[string]string{"md5Sum": "1f7690ebdd9b4caf8fab49ca1757bf27"}, "", int64(len("efgh")), "", nil},
{bucket, object, []byte("ijkl"), map[string]string{"md5Sum": "09a0877d04abf8759f99adec02baf579"}, "", int64(len("ijkl")), "", nil},
{bucket, object, []byte("mnop"), map[string]string{"md5Sum": "e132e96a5ddad6da8b07bba6f6131fef"}, "", int64(len("mnop")), "", nil},
// Test case 14-16.
// Test case 15-17.
// With no metadata
{bucket, object, data, nil, int64(len(data)), md5Hex(data), nil},
{bucket, object, nilBytes, nil, int64(len(nilBytes)), md5Hex(nilBytes), nil},
{bucket, object, fiveMBBytes, nil, int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), nil},
{bucket, object, data, nil, "", int64(len(data)), md5Hex(data), nil},
{bucket, object, nilBytes, nil, "", int64(len(nilBytes)), md5Hex(nilBytes), nil},
{bucket, object, fiveMBBytes, nil, "", int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), nil},
// Test case 17-19.
// Test case 18-20.
// With arbitrary metadata
{bucket, object, data, map[string]string{"answer": "42"}, int64(len(data)), md5Hex(data), nil},
{bucket, object, nilBytes, map[string]string{"answer": "42"}, int64(len(nilBytes)), md5Hex(nilBytes), nil},
{bucket, object, fiveMBBytes, map[string]string{"answer": "42"}, int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), nil},
{bucket, object, data, map[string]string{"answer": "42"}, "", int64(len(data)), md5Hex(data), nil},
{bucket, object, nilBytes, map[string]string{"answer": "42"}, "", int64(len(nilBytes)), md5Hex(nilBytes), nil},
{bucket, object, fiveMBBytes, map[string]string{"answer": "42"}, "", int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), nil},
// Test case 20-22.
// With valid md5sum in header
{bucket, object, data, md5Header(data), int64(len(data)), md5Hex(data), nil},
{bucket, object, nilBytes, md5Header(nilBytes), int64(len(nilBytes)), md5Hex(nilBytes), nil},
{bucket, object, fiveMBBytes, md5Header(fiveMBBytes), int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), nil},
// Test case 21-23.
// With valid md5sum and sha256.
{bucket, object, data, md5Header(data), hex.EncodeToString(sum256(data)), int64(len(data)), md5Hex(data), nil},
{bucket, object, nilBytes, md5Header(nilBytes), hex.EncodeToString(sum256(nilBytes)), int64(len(nilBytes)), md5Hex(nilBytes), nil},
{bucket, object, fiveMBBytes, md5Header(fiveMBBytes), hex.EncodeToString(sum256(fiveMBBytes)), int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), nil},
// Test case 23-25.
// Test case 24-26.
// data with invalid md5sum in header
{bucket, object, data, invalidMD5Header, int64(len(data)), md5Hex(data), BadDigest{invalidMD5, md5Hex(data)}},
{bucket, object, nilBytes, invalidMD5Header, int64(len(nilBytes)), md5Hex(nilBytes), BadDigest{invalidMD5, md5Hex(nilBytes)}},
{bucket, object, fiveMBBytes, invalidMD5Header, int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), BadDigest{invalidMD5, md5Hex(fiveMBBytes)}},
{bucket, object, data, invalidMD5Header, "", int64(len(data)), md5Hex(data), BadDigest{invalidMD5, md5Hex(data)}},
{bucket, object, nilBytes, invalidMD5Header, "", int64(len(nilBytes)), md5Hex(nilBytes), BadDigest{invalidMD5, md5Hex(nilBytes)}},
{bucket, object, fiveMBBytes, invalidMD5Header, "", int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), BadDigest{invalidMD5, md5Hex(fiveMBBytes)}},
// Test case 26-28.
// Test case 27-29.
// data with size different from the actual number of bytes available in the reader
{bucket, object, data, nil, int64(len(data) - 1), md5Hex(data[:len(data)-1]), nil},
{bucket, object, nilBytes, nil, int64(len(nilBytes) + 1), md5Hex(nilBytes), IncompleteBody{}},
{bucket, object, fiveMBBytes, nil, int64(0), md5Hex(fiveMBBytes), nil},
{bucket, object, data, nil, "", int64(len(data) - 1), md5Hex(data[:len(data)-1]), nil},
{bucket, object, nilBytes, nil, "", int64(len(nilBytes) + 1), md5Hex(nilBytes), IncompleteBody{}},
{bucket, object, fiveMBBytes, nil, "", int64(0), md5Hex(fiveMBBytes), nil},
// Test case 29
// Test case 30
// valid data with X-Amz-Meta- meta
{bucket, object, data, map[string]string{"X-Amz-Meta-AppID": "a42"}, int64(len(data)), md5Hex(data), nil},
{bucket, object, data, map[string]string{"X-Amz-Meta-AppID": "a42"}, "", int64(len(data)), md5Hex(data), nil},
}
for i, testCase := range testCases {
objInfo, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta)
objInfo, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta, testCase.inputSHA256)
actualErr = errorCause(actualErr)
if actualErr != nil && testCase.expectedError == nil {
t.Errorf("Test %d: %s: Expected to pass, but failed with: error %s.", i+1, instanceType, actualErr.Error())
@ -227,8 +232,9 @@ func testObjectAPIPutObjectDiskNotFOund(obj ObjectLayer, instanceType string, di
{bucket, object, []byte("mnop"), map[string]string{"md5Sum": "e132e96a5ddad6da8b07bba6f6131fef"}, int64(len("mnop")), true, "", nil},
}
sha256sum := ""
for i, testCase := range testCases {
objInfo, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta)
objInfo, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta, sha256sum)
actualErr = errorCause(err)
if actualErr != nil && testCase.shouldPass {
t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", i+1, instanceType, actualErr.Error())
@ -277,7 +283,8 @@ func testObjectAPIPutObjectDiskNotFOund(obj ObjectLayer, instanceType string, di
"",
InsufficientWriteQuorum{},
}
_, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta)
_, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta, sha256sum)
actualErr = errorCause(actualErr)
if actualErr != nil && testCase.shouldPass {
t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", len(testCases)+1, instanceType, actualErr.Error())
@ -309,8 +316,9 @@ func testObjectAPIPutObjectStaleFiles(obj ObjectLayer, instanceType string, disk
}
data := []byte("hello, world")
sha256sum := ""
// Create object.
_, err = obj.PutObject(bucket, object, int64(len(data)), bytes.NewReader(data), nil)
_, err = obj.PutObject(bucket, object, int64(len(data)), bytes.NewReader(data), nil, sha256sum)
if err != nil {
// Failed to create object, abort.
t.Fatalf("%s : %s", instanceType, err.Error())
@ -354,7 +362,8 @@ func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType str
md5Writer := md5.New()
md5Writer.Write(fiveMBBytes)
etag1 := hex.EncodeToString(md5Writer.Sum(nil))
_, err = obj.PutObjectPart(bucket, object, uploadID, 1, int64(len(fiveMBBytes)), bytes.NewReader(fiveMBBytes), etag1)
sha256sum := ""
_, err = obj.PutObjectPart(bucket, object, uploadID, 1, int64(len(fiveMBBytes)), bytes.NewReader(fiveMBBytes), etag1, sha256sum)
if err != nil {
// Failed to upload object part, abort.
t.Fatalf("%s : %s", instanceType, err.Error())
@ -365,7 +374,7 @@ func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType str
md5Writer = md5.New()
md5Writer.Write(data)
etag2 := hex.EncodeToString(md5Writer.Sum(nil))
_, err = obj.PutObjectPart(bucket, object, uploadID, 2, int64(len(data)), bytes.NewReader(data), etag2)
_, err = obj.PutObjectPart(bucket, object, uploadID, 2, int64(len(data)), bytes.NewReader(data), etag2, sha256sum)
if err != nil {
// Failed to upload object part, abort.
t.Fatalf("%s : %s", instanceType, err.Error())

9
cmd/object-errors.go
View file

@ -73,6 +73,8 @@ func toObjectErr(err error, params ...string) error {
err = InsufficientWriteQuorum{}
case io.ErrUnexpectedEOF, io.ErrShortWrite:
err = IncompleteBody{}
case errContentSHA256Mismatch:
err = SHA256Mismatch{}
}
if ok {
e.e = err
@ -81,6 +83,13 @@ func toObjectErr(err error, params ...string) error {
return err
}
// SHA256Mismatch - when content sha256 does not match with what was sent from client.
type SHA256Mismatch struct{}
func (e SHA256Mismatch) Error() string {
return "sha256 computed does not match with what is expected"
}
// StorageFull storage ran out of space.
type StorageFull struct{}

43
cmd/object-handlers.go
View file

@ -379,8 +379,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
// Do not set `md5sum` as CopyObject will not keep the
// same md5sum as the source.
sha256sum := ""
// Create the object.
objInfo, err = objectAPI.PutObject(bucket, object, size, pipeReader, metadata)
objInfo, err = objectAPI.PutObject(bucket, object, size, pipeReader, metadata, sha256sum)
if err != nil {
// Close the this end of the pipe upon error in PutObject.
pipeReader.CloseWithError(err)
@ -466,6 +467,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
// Make sure we hex encode md5sum here.
metadata["md5Sum"] = hex.EncodeToString(md5Bytes)
sha256sum := ""
var objInfo ObjectInfo
switch rAuthType {
default:
@ -479,7 +482,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
return
}
// Create anonymous object.
objInfo, err = objectAPI.PutObject(bucket, object, size, r.Body, metadata)
objInfo, err = objectAPI.PutObject(bucket, object, size, r.Body, metadata, sha256sum)
case authTypeStreamingSigned:
// Initialize stream signature verifier.
reader, s3Error := newSignV4ChunkedReader(r)
@ -488,7 +491,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
objInfo, err = objectAPI.PutObject(bucket, object, size, reader, metadata)
objInfo, err = objectAPI.PutObject(bucket, object, size, reader, metadata, sha256sum)
case authTypeSignedV2, authTypePresignedV2:
s3Error := isReqAuthenticatedV2(r)
if s3Error != ErrNone {
@ -496,12 +499,18 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
objInfo, err = objectAPI.PutObject(bucket, object, size, r.Body, metadata)
objInfo, err = objectAPI.PutObject(bucket, object, size, r.Body, metadata, sha256sum)
case authTypePresigned, authTypeSigned:
// Initialize signature verifier.
reader := newSignVerify(r)
if s3Error := reqSignatureV4Verify(r); s3Error != ErrNone {
errorIf(errSignatureMismatch, dumpRequest(r))
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
if !skipContentSha256Cksum(r) {
sha256sum = r.Header.Get("X-Amz-Content-Sha256")
}
// Create object.
objInfo, err = objectAPI.PutObject(bucket, object, size, reader, metadata)
objInfo, err = objectAPI.PutObject(bucket, object, size, r.Body, metadata, sha256sum)
}
if err != nil {
errorIf(err, "Unable to create an object.")
@ -642,6 +651,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
var partMD5 string
incomingMD5 := hex.EncodeToString(md5Bytes)
sha256sum := ""
switch rAuthType {
default:
// For all unknown auth types return error.
@ -654,7 +664,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
return
}
// No need to verify signature, anonymous request access is already allowed.
partMD5, err = objectAPI.PutObjectPart(bucket, object, uploadID, partID, size, r.Body, incomingMD5)
partMD5, err = objectAPI.PutObjectPart(bucket, object, uploadID, partID, size, r.Body, incomingMD5, sha256sum)
case authTypeStreamingSigned:
// Initialize stream signature verifier.
reader, s3Error := newSignV4ChunkedReader(r)
@ -663,7 +673,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
partMD5, err = objectAPI.PutObjectPart(bucket, object, uploadID, partID, size, reader, incomingMD5)
partMD5, err = objectAPI.PutObjectPart(bucket, object, uploadID, partID, size, reader, incomingMD5, sha256sum)
case authTypeSignedV2, authTypePresignedV2:
s3Error := isReqAuthenticatedV2(r)
if s3Error != ErrNone {
@ -671,11 +681,18 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
partMD5, err = objectAPI.PutObjectPart(bucket, object, uploadID, partID, size, r.Body, incomingMD5)
partMD5, err = objectAPI.PutObjectPart(bucket, object, uploadID, partID, size, r.Body, incomingMD5, sha256sum)
case authTypePresigned, authTypeSigned:
// Initialize signature verifier.
reader := newSignVerify(r)
partMD5, err = objectAPI.PutObjectPart(bucket, object, uploadID, partID, size, reader, incomingMD5)
if s3Error := reqSignatureV4Verify(r); s3Error != ErrNone {
errorIf(errSignatureMismatch, dumpRequest(r))
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
if !skipContentSha256Cksum(r) {
sha256sum = r.Header.Get("X-Amz-Content-Sha256")
}
partMD5, err = objectAPI.PutObjectPart(bucket, object, uploadID, partID, size, r.Body, incomingMD5, sha256sum)
}
if err != nil {
errorIf(err, "Unable to create object part.")

15
cmd/object-handlers_test.go
View file

@ -71,10 +71,11 @@ func testAPIGetOjectHandler(obj ObjectLayer, instanceType, bucketName string, ap
// case - 1.
{bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)},
}
sha256sum := ""
// iterate through the above set of inputs and upload the object.
for i, input := range putObjectInputs {
// uploading the object.
_, err := obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData)
_, err := obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData, sha256sum)
// if object upload fails stop the test.
if err != nil {
t.Fatalf("Put Object case %d: Error uploading object: <ERROR> %v", i+1, err)
@ -430,10 +431,11 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
// case - 1.
{bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)},
}
sha256sum := ""
// iterate through the above set of inputs and upload the object.
for i, input := range putObjectInputs {
// uploading the object.
_, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData)
_, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData, sha256sum)
// if object upload fails stop the test.
if err != nil {
t.Fatalf("Put Object case %d: Error uploading object: <ERROR> %v", i+1, err)
@ -683,7 +685,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
}
// Iterating over creatPartCases to generate multipart chunks.
for _, part := range parts {
_, err = obj.PutObjectPart(part.bucketName, part.objName, part.uploadID, part.PartID, part.intputDataSize, bytes.NewBufferString(part.inputReaderData), part.inputMd5)
_, err = obj.PutObjectPart(part.bucketName, part.objName, part.uploadID, part.PartID, part.intputDataSize, bytes.NewBufferString(part.inputReaderData), part.inputMd5, "")
if err != nil {
t.Fatalf("%s : %s", instanceType, err)
}
@ -905,7 +907,7 @@ func testAPIDeleteOjectHandler(obj ObjectLayer, instanceType, bucketName string,
// iterate through the above set of inputs and upload the object.
for i, input := range putObjectInputs {
// uploading the object.
_, err := obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData)
_, err := obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData, "")
// if object upload fails stop the test.
if err != nil {
t.Fatalf("Put Object case %d: Error uploading object: <ERROR> %v", i+1, err)
@ -1211,7 +1213,7 @@ func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName strin
NoAPIErr := APIError{}
MissingContent := getAPIError(ErrMissingContentLength)
EntityTooLarge := getAPIError(ErrEntityTooLarge)
BadSigning := getAPIError(ErrContentSHA256Mismatch)
BadSigning := getAPIError(ErrSignatureDoesNotMatch)
BadChecksum := getAPIError(ErrInvalidDigest)
InvalidPart := getAPIError(ErrInvalidPart)
InvalidMaxParts := getAPIError(ErrInvalidMaxParts)
@ -1248,7 +1250,8 @@ func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName strin
case TooBigObject:
tReq.ContentLength = maxObjectSize + 1
case BadSignature:
tReq.Header.Set("x-amz-content-sha256", "somethingElse")
// Mangle signature
tReq.Header.Set("authorization", tReq.Header.Get("authorization")+"a")
case BadMD5:
tReq.Header.Set("Content-MD5", "badmd5")
}

4
cmd/object-interface.go
View file

@ -36,14 +36,14 @@ type ObjectLayer interface {
// Object operations.
GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error)
GetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error)
PutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string) (objInto ObjectInfo, err error)
PutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInto ObjectInfo, err error)
DeleteObject(bucket, object string) error
HealObject(bucket, object string) error
// Multipart operations.
ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error)
NewMultipartUpload(bucket, object string, metadata map[string]string) (uploadID string, err error)
PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (md5 string, err error)
PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (md5 string, err error)
ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error)
AbortMultipartUpload(bucket, object, uploadID string) error
CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []completePart) (md5 string, err error)

34
cmd/object_api_suite_test.go
View file

@ -109,7 +109,7 @@ func testMultipartObjectCreation(obj ObjectLayer, instanceType string, c TestErr
expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil))
var calculatedMD5sum string
calculatedMD5sum, err = obj.PutObjectPart("bucket", "key", uploadID, i, int64(len(data)), bytes.NewBuffer(data), expectedMD5Sumhex)
calculatedMD5sum, err = obj.PutObjectPart("bucket", "key", uploadID, i, int64(len(data)), bytes.NewBuffer(data), expectedMD5Sumhex, "")
if err != nil {
c.Errorf("%s: <ERROR> %s", instanceType, err)
}
@ -158,7 +158,7 @@ func testMultipartObjectAbort(obj ObjectLayer, instanceType string, c TestErrHan
metadata["md5"] = expectedMD5Sumhex
var calculatedMD5sum string
calculatedMD5sum, err = obj.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex)
calculatedMD5sum, err = obj.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex, "")
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@ -201,7 +201,7 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, c TestErrH
metadata := make(map[string]string)
metadata["md5Sum"] = expectedMD5Sumhex
var objInfo ObjectInfo
objInfo, err = obj.PutObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), metadata)
objInfo, err = obj.PutObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), metadata, "")
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@ -254,7 +254,7 @@ func testPaging(obj ObjectLayer, instanceType string, c TestErrHandler) {
// check before paging occurs.
for i := 0; i < 5; i++ {
key := "obj" + strconv.Itoa(i)
_, err = obj.PutObject("bucket", key, int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil)
_, err = obj.PutObject("bucket", key, int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "")
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@ -274,7 +274,7 @@ func testPaging(obj ObjectLayer, instanceType string, c TestErrHandler) {
// check after paging occurs pages work.
for i := 6; i <= 10; i++ {
key := "obj" + strconv.Itoa(i)
_, err = obj.PutObject("bucket", key, int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil)
_, err = obj.PutObject("bucket", key, int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "")
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@ -291,11 +291,11 @@ func testPaging(obj ObjectLayer, instanceType string, c TestErrHandler) {
}
// check paging with prefix at end returns less objects.
{
_, err = obj.PutObject("bucket", "newPrefix", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil)
_, err = obj.PutObject("bucket", "newPrefix", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "")
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
_, err = obj.PutObject("bucket", "newPrefix2", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil)
_, err = obj.PutObject("bucket", "newPrefix2", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "")
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@ -333,11 +333,11 @@ func testPaging(obj ObjectLayer, instanceType string, c TestErrHandler) {
// check delimited results with delimiter and prefix.
{
_, err = obj.PutObject("bucket", "this/is/delimited", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil)
_, err = obj.PutObject("bucket", "this/is/delimited", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "")
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
_, err = obj.PutObject("bucket", "this/is/also/a/delimited/file", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil)
_, err = obj.PutObject("bucket", "this/is/also/a/delimited/file", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "")
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@ -446,14 +446,14 @@ func testObjectOverwriteWorks(obj ObjectLayer, instanceType string, c TestErrHan
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
_, err = obj.PutObject("bucket", "object", int64(len("The list of parts was not in ascending order. The parts list must be specified in order by part number.")), bytes.NewBufferString("The list of parts was not in ascending order. The parts list must be specified in order by part number."), nil)
_, err = obj.PutObject("bucket", "object", int64(len("The list of parts was not in ascending order. The parts list must be specified in order by part number.")), bytes.NewBufferString("The list of parts was not in ascending order. The parts list must be specified in order by part number."), nil, "")
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
uploadContent := "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."
length := int64(len(uploadContent))
_, err = obj.PutObject("bucket", "object", length, bytes.NewBufferString(uploadContent), nil)
_, err = obj.PutObject("bucket", "object", length, bytes.NewBufferString(uploadContent), nil, "")
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@ -475,7 +475,7 @@ func (s *ObjectLayerAPISuite) TestNonExistantBucketOperations(c *C) {
// Tests validate that bucket operation on non-existent bucket fails.
func testNonExistantBucketOperations(obj ObjectLayer, instanceType string, c TestErrHandler) {
_, err := obj.PutObject("bucket1", "object", int64(len("one")), bytes.NewBufferString("one"), nil)
_, err := obj.PutObject("bucket1", "object", int64(len("one")), bytes.NewBufferString("one"), nil, "")
if err == nil {
c.Fatal("Expected error but found nil")
}
@ -522,7 +522,7 @@ func testPutObject(obj ObjectLayer, instanceType string, c TestErrHandler) {
}
var bytesBuffer1 bytes.Buffer
_, err = obj.PutObject("bucket", "object", length, readerEOF, nil)
_, err = obj.PutObject("bucket", "object", length, readerEOF, nil, "")
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@ -535,7 +535,7 @@ func testPutObject(obj ObjectLayer, instanceType string, c TestErrHandler) {
}
var bytesBuffer2 bytes.Buffer
_, err = obj.PutObject("bucket", "object", length, readerNoEOF, nil)
_, err = obj.PutObject("bucket", "object", length, readerNoEOF, nil, "")
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@ -563,7 +563,7 @@ func testPutObjectInSubdir(obj ObjectLayer, instanceType string, c TestErrHandle
uploadContent := `The specified multipart upload does not exist. The upload ID might be invalid, or the multipart
upload might have been aborted or completed.`
length := int64(len(uploadContent))
_, err = obj.PutObject("bucket", "dir1/dir2/object", length, bytes.NewBufferString(uploadContent), nil)
_, err = obj.PutObject("bucket", "dir1/dir2/object", length, bytes.NewBufferString(uploadContent), nil, "")
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@ -737,7 +737,7 @@ func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string,
_, err = obj.PutObject("bucket", "dir1/dir3/object",
int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")),
bytes.NewBufferString("One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tag might not have matched the part's entity tag."), nil)
bytes.NewBufferString("One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tag might not have matched the part's entity tag."), nil, "")
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
@ -790,7 +790,7 @@ func testContentType(obj ObjectLayer, instanceType string, c TestErrHandler) {
}
uploadContent := "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."
// Test empty.
_, err = obj.PutObject("bucket", "minio.png", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil)
_, err = obj.PutObject("bucket", "minio.png", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "")
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}

3
cmd/server_test.go
View file

@ -1091,8 +1091,7 @@ func (s *TestSuiteCommon) TestSHA256Mismatch(c *C) {
// Body is on purpose set to nil so that we get payload generated for empty bytes.
// Create new HTTP request with incorrect secretKey to generate an incorrect signature.
secretKey := s.secretKey + "a"
request, err = newTestSignedRequestV4("PUT", getPutObjectURL(s.endPoint, bucketName, objName), 0, nil, s.accessKey, secretKey)
request, err = newTestSignedRequestV4("PUT", getPutObjectURL(s.endPoint, bucketName, objName), 0, nil, s.accessKey, s.secretKey)
c.Assert(request.Header.Get("x-amz-content-sha256"), Equals, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
// Set the body to generate signature mismatch.
request.Body = ioutil.NopCloser(bytes.NewReader([]byte("Hello, World")))

3
cmd/web-handlers.go
View file

@ -416,7 +416,8 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
writeWebErrorResponse(w, errors.New("Server not initialized"))
return
}
if _, err := objectAPI.PutObject(bucket, object, -1, r.Body, metadata); err != nil {
sha256sum := ""
if _, err := objectAPI.PutObject(bucket, object, -1, r.Body, metadata, sha256sum); err != nil {
writeWebErrorResponse(w, err)
return
}

8
cmd/web-handlers_test.go
View file

@ -351,7 +351,7 @@ func testListObjectsWebHandler(obj ObjectLayer, instanceType string, t TestErrHa
data := bytes.Repeat([]byte("a"), objectSize)
_, err = obj.PutObject(bucketName, objectName, int64(len(data)), bytes.NewReader(data), map[string]string{"md5Sum": "c9a34cfc85d982698c6ac89f76071abd"})
_, err = obj.PutObject(bucketName, objectName, int64(len(data)), bytes.NewReader(data), map[string]string{"md5Sum": "c9a34cfc85d982698c6ac89f76071abd"}, "")
if err != nil {
t.Fatalf("Was not able to upload an object, %v", err)
@ -422,7 +422,7 @@ func testRemoveObjectWebHandler(obj ObjectLayer, instanceType string, t TestErrH
data := bytes.Repeat([]byte("a"), objectSize)
_, err = obj.PutObject(bucketName, objectName, int64(len(data)), bytes.NewReader(data), map[string]string{"md5Sum": "c9a34cfc85d982698c6ac89f76071abd"})
_, err = obj.PutObject(bucketName, objectName, int64(len(data)), bytes.NewReader(data), map[string]string{"md5Sum": "c9a34cfc85d982698c6ac89f76071abd"}, "")
if err != nil {
t.Fatalf("Was not able to upload an object, %v", err)
@ -696,7 +696,7 @@ func testDownloadWebHandler(obj ObjectLayer, instanceType string, t TestErrHandl
}
content := []byte("temporary file's content")
_, err = obj.PutObject(bucketName, objectName, int64(len(content)), bytes.NewReader(content), map[string]string{"md5Sum": "01ce59706106fe5e02e7f55fffda7f34"})
_, err = obj.PutObject(bucketName, objectName, int64(len(content)), bytes.NewReader(content), map[string]string{"md5Sum": "01ce59706106fe5e02e7f55fffda7f34"}, "")
if err != nil {
t.Fatalf("Was not able to upload an object, %v", err)
}
@ -755,7 +755,7 @@ func testWebPresignedGetHandler(obj ObjectLayer, instanceType string, t TestErrH
}
data := bytes.Repeat([]byte("a"), objectSize)
_, err = obj.PutObject(bucketName, objectName, int64(len(data)), bytes.NewReader(data), map[string]string{"md5Sum": "c9a34cfc85d982698c6ac89f76071abd"})
_, err = obj.PutObject(bucketName, objectName, int64(len(data)), bytes.NewReader(data), map[string]string{"md5Sum": "c9a34cfc85d982698c6ac89f76071abd"}, "")
if err != nil {
t.Fatalf("Was not able to upload an object, %v", err)
}

30
cmd/xl-v1-multipart.go
View file

@ -18,8 +18,10 @@ package cmd
import (
"crypto/md5"
"crypto/sha256"
"encoding/hex"
"fmt"
"hash"
"io"
"io/ioutil"
"path"
@ -333,7 +335,7 @@ func (xl xlObjects) NewMultipartUpload(bucket, object string, meta map[string]st
// of the multipart transaction.
//
// Implements S3 compatible Upload Part API.
func (xl xlObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, error) {
func (xl xlObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (string, error) {
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
return "", traceError(BucketNameInvalid{Bucket: bucket})
@ -384,10 +386,21 @@ func (xl xlObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
tmpSuffix := getUUID()
tmpPartPath := path.Join(tmpMetaPrefix, tmpSuffix)
lreader := data
// Initialize md5 writer.
md5Writer := md5.New()
lreader := data
writers := []io.Writer{md5Writer}
var sha256Writer hash.Hash
if sha256sum != "" {
sha256Writer = sha256.New()
writers = append(writers, sha256Writer)
}
mw := io.MultiWriter(writers...)
// Limit the reader to its provided size > 0.
if size > 0 {
// This is done so that we can avoid erroneous clients sending
@ -396,7 +409,7 @@ func (xl xlObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
} // else we read till EOF.
// Construct a tee reader for md5sum.
teeReader := io.TeeReader(lreader, md5Writer)
teeReader := io.TeeReader(lreader, mw)
// Erasure code data and write across all disks.
sizeWritten, checkSums, err := erasureCreateFile(onlineDisks, minioMetaBucket, tmpPartPath, teeReader, xlMeta.Erasure.BlockSize, xl.dataBlocks, xl.parityBlocks, bitRotAlgo, xl.writeQuorum)
@ -436,7 +449,18 @@ func (xl xlObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
}
}
if sha256sum != "" {
newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil))
if newSHA256sum != sha256sum {
// SHA256 mismatch, delete the temporary object.
xl.deleteObject(minioMetaBucket, tmpPartPath)
return "", traceError(SHA256Mismatch{})
}
}
// get a random ID for lock instrumentation.
// generates random string on setting MINIO_DEBUG=lock, else returns empty string.
// used for instrumentation on locks.
opsID = getOpsID()
nsMutex.Lock(minioMetaBucket, uploadIDPath, opsID)

30
cmd/xl-v1-object.go
View file

@ -18,7 +18,9 @@ package cmd
import (
"crypto/md5"
"crypto/sha256"
"encoding/hex"
"hash"
"io"
"path"
"strings"
@ -490,7 +492,7 @@ func renameObject(disks []StorageAPI, srcBucket, srcObject, dstBucket, dstObject
// until EOF, erasure codes the data across all disk and additionally
// writes `xl.json` which carries the necessary metadata for future
// object operations.
func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) {
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
return ObjectInfo{}, traceError(BucketNameInvalid{Bucket: bucket})
@ -515,10 +517,17 @@ func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.
minioMetaTmpBucket := path.Join(minioMetaBucket, tmpMetaPrefix)
tempObj := uniqueID
var mw io.Writer
// Initialize md5 writer.
md5Writer := md5.New()
writers := []io.Writer{md5Writer}
var sha256Writer hash.Hash
if sha256sum != "" {
sha256Writer = sha256.New()
writers = append(writers, sha256Writer)
}
// Proceed to set the cache.
var newBuffer io.WriteCloser
@ -531,17 +540,17 @@ func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.
newBuffer, err = xl.objCache.Create(path.Join(bucket, object), size)
if err == nil {
// Create a multi writer to write to both memory and client response.
mw = io.MultiWriter(newBuffer, md5Writer)
writers = append(writers, newBuffer)
}
// Ignore error if cache is full, proceed to write the object.
if err != nil && err != objcache.ErrCacheFull {
// For any other error return here.
return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
}
} else {
mw = md5Writer
}
mw := io.MultiWriter(writers...)
// Limit the reader to its provided size if specified.
var limitDataReader io.Reader
if size > 0 {
@ -621,7 +630,18 @@ func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.
}
}
if sha256sum != "" {
newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil))
if newSHA256sum != sha256sum {
// SHA256 mismatch, delete the temporary object.
xl.deleteObject(minioMetaBucket, tempObj)
return ObjectInfo{}, traceError(SHA256Mismatch{})
}
}
// get a random ID for lock instrumentation.
// generates random string on setting MINIO_DEBUG=lock, else returns empty string.
// used for instrumentation on locks.
opsID := getOpsID()
// Lock the object.

16
cmd/xl-v1-object_test.go
View file

@ -50,12 +50,12 @@ func TestRepeatPutObjectPart(t *testing.T) {
md5Writer := md5.New()
md5Writer.Write(fiveMBBytes)
md5Hex := hex.EncodeToString(md5Writer.Sum(nil))
_, err = objLayer.PutObjectPart("bucket1", "mpartObj1", uploadID, 1, 5*1024*1024, bytes.NewReader(fiveMBBytes), md5Hex)
_, err = objLayer.PutObjectPart("bucket1", "mpartObj1", uploadID, 1, 5*1024*1024, bytes.NewReader(fiveMBBytes), md5Hex, "")
if err != nil {
t.Fatal(err)
}
// PutObjectPart should succeed even if part already exists. ref: https://github.com/minio/minio/issues/1930
_, err = objLayer.PutObjectPart("bucket1", "mpartObj1", uploadID, 1, 5*1024*1024, bytes.NewReader(fiveMBBytes), md5Hex)
_, err = objLayer.PutObjectPart("bucket1", "mpartObj1", uploadID, 1, 5*1024*1024, bytes.NewReader(fiveMBBytes), md5Hex, "")
if err != nil {
t.Fatal(err)
}
@ -89,7 +89,7 @@ func TestXLDeleteObjectBasic(t *testing.T) {
}
// Create object "obj" under bucket "bucket" for Test 7 to pass
_, err = xl.PutObject("bucket", "obj", int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil)
_, err = xl.PutObject("bucket", "obj", int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, "")
if err != nil {
t.Fatalf("XL Object upload failed: <ERROR> %s", err)
}
@ -125,7 +125,7 @@ func TestXLDeleteObjectDiskNotFound(t *testing.T) {
bucket := "bucket"
object := "object"
// Create object "obj" under bucket "bucket".
_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil)
_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, "")
if err != nil {
t.Fatal(err)
}
@ -140,7 +140,7 @@ func TestXLDeleteObjectDiskNotFound(t *testing.T) {
}
// Create "obj" under "bucket".
_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil)
_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, "")
if err != nil {
t.Fatal(err)
}
@ -175,7 +175,7 @@ func TestGetObjectNoQuorum(t *testing.T) {
bucket := "bucket"
object := "object"
// Create "object" under "bucket".
_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil)
_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, "")
if err != nil {
t.Fatal(err)
}
@ -227,7 +227,7 @@ func TestPutObjectNoQuorum(t *testing.T) {
bucket := "bucket"
object := "object"
// Create "object" under "bucket".
_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil)
_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, "")
if err != nil {
t.Fatal(err)
}
@ -250,7 +250,7 @@ func TestPutObjectNoQuorum(t *testing.T) {
}
}
// Upload new content to same object "object"
_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil)
_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, "")
err = errorCause(err)
if err != toObjectErr(errXLWriteQuorum, bucket, object) {
t.Errorf("Expected putObject to fail with %v, but failed with %v", toObjectErr(errXLWriteQuorum, bucket, object), err)