From 61a18ed48f4e6602e2c09ea5c9fc2f99f046a1ae Mon Sep 17 00:00:00 2001 From: Krishna Srinivas Date: Mon, 3 Oct 2016 04:21:49 +0530 Subject: [PATCH] sha256: Verify sha256 along with md5sum, signature is verified on the request early. (#2813) --- cmd/api-errors.go | 2 + cmd/auth-handler.go | 15 +++++ cmd/benchmark-utils_test.go | 15 +++-- cmd/bucket-handlers.go | 4 +- cmd/bucket-notification-handlers.go | 4 +- cmd/bucket-policy-migrate.go | 4 +- cmd/bucket-policy.go | 3 +- cmd/controller_test.go | 5 +- cmd/format-config-v1_test.go | 9 ++- cmd/fs-v1-metadata_test.go | 6 +- cmd/fs-v1-multipart.go | 27 +++++++- cmd/fs-v1-multipart_test.go | 9 ++- cmd/fs-v1.go | 24 ++++++- cmd/fs-v1_test.go | 6 +- cmd/object-api-getobject_test.go | 6 +- cmd/object-api-getobjectinfo_test.go | 3 +- cmd/object-api-listobjects_test.go | 6 +- cmd/object-api-multipart_test.go | 65 ++++++++++-------- cmd/object-api-putobject_test.go | 99 +++++++++++++++------------- cmd/object-errors.go | 9 +++ cmd/object-handlers.go | 43 ++++++++---- cmd/object-handlers_test.go | 15 +++-- cmd/object-interface.go | 4 +- cmd/object_api_suite_test.go | 34 +++++----- cmd/server_test.go | 3 +- cmd/web-handlers.go | 3 +- cmd/web-handlers_test.go | 8 +-- cmd/xl-v1-multipart.go | 30 ++++++++- cmd/xl-v1-object.go | 30 +++++++-- cmd/xl-v1-object_test.go | 16 ++--- 30 files changed, 341 insertions(+), 166 deletions(-) diff --git a/cmd/api-errors.go b/cmd/api-errors.go index 192500ae8..c9834a083 100644 --- a/cmd/api-errors.go +++ b/cmd/api-errors.go @@ -617,6 +617,8 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) { apiErr = ErrNoSuchUpload case PartTooSmall: apiErr = ErrEntityTooSmall + case SHA256Mismatch: + apiErr = ErrContentSHA256Mismatch default: apiErr = ErrInternalError } diff --git a/cmd/auth-handler.go b/cmd/auth-handler.go index 420b57db8..0982403e7 100644 --- a/cmd/auth-handler.go +++ b/cmd/auth-handler.go @@ -130,6 +130,21 @@ func isReqAuthenticatedV2(r *http.Request) (s3Error APIErrorCode) { return doesPresignV2SignatureMatch(r) } +func reqSignatureV4Verify(r *http.Request) (s3Error APIErrorCode) { + sha256sum := r.Header.Get("X-Amz-Content-Sha256") + // Skips calculating sha256 on the payload on server, + // if client requested for it. + if skipContentSha256Cksum(r) { + sha256sum = unsignedPayload + } + if isRequestSignatureV4(r) { + return doesSignatureMatch(sha256sum, r, serverConfig.GetRegion()) + } else if isRequestPresignedSignatureV4(r) { + return doesPresignedSignatureMatch(sha256sum, r, serverConfig.GetRegion()) + } + return ErrAccessDenied +} + // Verify if request has valid AWS Signature Version '4'. func isReqAuthenticated(r *http.Request, region string) (s3Error APIErrorCode) { if r == nil { diff --git a/cmd/benchmark-utils_test.go b/cmd/benchmark-utils_test.go index ab4cf48fd..b18e56320 100644 --- a/cmd/benchmark-utils_test.go +++ b/cmd/benchmark-utils_test.go @@ -62,13 +62,14 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) { hasher.Write([]byte(textData)) metadata := make(map[string]string) metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil)) + sha256sum := "" // benchmark utility which helps obtain number of allocations and bytes allocated per ops. b.ReportAllocs() // the actual benchmark for PutObject starts here. Reset the benchmark timer. b.ResetTimer() for i := 0; i < b.N; i++ { // insert the object. - objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata) + objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum) if err != nil { b.Fatal(err) } @@ -107,6 +108,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) { hasher.Write([]byte(textData)) metadata := make(map[string]string) metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil)) + sha256sum := "" uploadID, err = obj.NewMultipartUpload(bucket, object, metadata) if err != nil { b.Fatal(err) @@ -130,7 +132,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) { hasher.Write([]byte(textPartData)) metadata := make(map[string]string) metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil)) - md5Sum, err = obj.PutObjectPart(bucket, object, uploadID, j, int64(len(textPartData)), bytes.NewBuffer(textPartData), metadata["md5Sum"]) + md5Sum, err = obj.PutObjectPart(bucket, object, uploadID, j, int64(len(textPartData)), bytes.NewBuffer(textPartData), metadata["md5Sum"], sha256sum) if err != nil { b.Fatal(err) } @@ -194,6 +196,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) { b.Fatal(err) } + sha256sum := "" for i := 0; i < 10; i++ { // get text data generated for number of bytes equal to object size. textData := generateBytesData(objSize) @@ -206,7 +209,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) { metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil)) // insert the object. var objInfo ObjectInfo - objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata) + objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum) if err != nil { b.Fatal(err) } @@ -294,6 +297,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) { hasher.Write([]byte(textData)) metadata := make(map[string]string) metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil)) + sha256sum := "" // benchmark utility which helps obtain number of allocations and bytes allocated per ops. b.ReportAllocs() // the actual benchmark for PutObject starts here. Reset the benchmark timer. @@ -303,7 +307,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) { i := 0 for pb.Next() { // insert the object. - objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata) + objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum) if err != nil { b.Fatal(err) } @@ -340,9 +344,10 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) { hasher.Write([]byte(textData)) metadata := make(map[string]string) metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil)) + sha256sum := "" // insert the object. var objInfo ObjectInfo - objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata) + objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum) if err != nil { b.Fatal(err) } diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index 1ba39523e..5daf259e2 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -459,7 +459,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h metadata := make(map[string]string) // Nothing to store right now. - objInfo, err := objectAPI.PutObject(bucket, object, -1, fileBody, metadata) + sha256sum := "" + + objInfo, err := objectAPI.PutObject(bucket, object, -1, fileBody, metadata, sha256sum) if err != nil { errorIf(err, "Unable to create object.") writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) diff --git a/cmd/bucket-notification-handlers.go b/cmd/bucket-notification-handlers.go index 7b09bd0a3..e34c5efad 100644 --- a/cmd/bucket-notification-handlers.go +++ b/cmd/bucket-notification-handlers.go @@ -146,7 +146,9 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, // Proceed to save notification configuration. notificationConfigPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig) - _, err = objectAPI.PutObject(minioMetaBucket, notificationConfigPath, bufferSize, bytes.NewReader(buffer.Bytes()), nil) + sha256sum := "" + var metadata map[string]string + _, err = objectAPI.PutObject(minioMetaBucket, notificationConfigPath, bufferSize, bytes.NewReader(buffer.Bytes()), metadata, sha256sum) if err != nil { errorIf(err, "Unable to write bucket notification configuration.") writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) diff --git a/cmd/bucket-policy-migrate.go b/cmd/bucket-policy-migrate.go index 50805cd8d..b54b04697 100644 --- a/cmd/bucket-policy-migrate.go +++ b/cmd/bucket-policy-migrate.go @@ -74,8 +74,10 @@ func migrateBucketPolicyConfig(objAPI ObjectLayer) error { policyBytes, err := ioutil.ReadFile(policyPath) fatalIf(err, "Unable to read bucket policy to migrate bucket policy", policyPath) newPolicyPath := retainSlash(bucketConfigPrefix) + retainSlash(bucketName) + policyJSON + var metadata map[string]string + sha256sum := "" // Erasure code the policy config to all the disks. - _, err = objAPI.PutObject(minioMetaBucket, newPolicyPath, int64(len(policyBytes)), bytes.NewReader(policyBytes), nil) + _, err = objAPI.PutObject(minioMetaBucket, newPolicyPath, int64(len(policyBytes)), bytes.NewReader(policyBytes), metadata, sha256sum) fatalIf(err, "Unable to write bucket policy during migration.", newPolicyPath) return nil } diff --git a/cmd/bucket-policy.go b/cmd/bucket-policy.go index f510ebabb..792768d38 100644 --- a/cmd/bucket-policy.go +++ b/cmd/bucket-policy.go @@ -210,7 +210,8 @@ func writeBucketPolicy(bucket string, objAPI ObjectLayer, reader io.Reader, size } policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON) - if _, err := objAPI.PutObject(minioMetaBucket, policyPath, size, reader, nil); err != nil { + sha256sum := "" + if _, err := objAPI.PutObject(minioMetaBucket, policyPath, size, reader, nil, sha256sum); err != nil { errorIf(err, "Unable to set policy for the bucket %s", bucket) return errorCause(err) } diff --git a/cmd/controller_test.go b/cmd/controller_test.go index f815b1afb..54d1029f7 100644 --- a/cmd/controller_test.go +++ b/cmd/controller_test.go @@ -330,7 +330,7 @@ func (s *TestRPCControllerSuite) testControllerHealObjectH(t *testing.T) { datum := strings.NewReader("a") _, err = s.testServer.Obj.PutObject("testbucket", "testobject", 1, - datum, nil) + datum, nil, "") if err != nil { t.Fatalf("Controller.HealObjectH - put object failed with %s", err.Error()) @@ -373,8 +373,7 @@ func (s *TestRPCControllerSuite) testControllerListObjectsHealH(t *testing.T) { r := strings.NewReader("0") _, err = s.testServer.Obj.PutObject( - "testbucket", "testObj-0", 1, r, nil, - ) + "testbucket", "testObj-0", 1, r, nil, "") if err != nil { t.Fatalf("Controller.ListObjectsHealH - object creation failed - %s", err.Error()) diff --git a/cmd/format-config-v1_test.go b/cmd/format-config-v1_test.go index 22bd58c99..b6d924bfa 100644 --- a/cmd/format-config-v1_test.go +++ b/cmd/format-config-v1_test.go @@ -225,8 +225,9 @@ func prepareFormatXLHealFreshDisks(obj ObjectLayer) ([]StorageAPI, error) { bucket := "bucket" object := "object" + sha256sum := "" - _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil) + _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum) if err != nil { return []StorageAPI{}, err } @@ -349,8 +350,9 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) { bucket := "bucket" object := "object" + sha256sum := "" - _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil) + _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum) if err != nil { t.Fatal(err) } @@ -421,8 +423,9 @@ func TestFormatXLReorderByInspection(t *testing.T) { bucket := "bucket" object := "object" + sha256sum := "" - _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil) + _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum) if err != nil { t.Fatal(err) } diff --git a/cmd/fs-v1-metadata_test.go b/cmd/fs-v1-metadata_test.go index c3ee7e22a..8d3552d56 100644 --- a/cmd/fs-v1-metadata_test.go +++ b/cmd/fs-v1-metadata_test.go @@ -83,8 +83,9 @@ func TestReadFSMetadata(t *testing.T) { if err = obj.MakeBucket(bucketName); err != nil { t.Fatal("Unexpected err: ", err) } + sha256sum := "" if _, err = obj.PutObject(bucketName, objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")), - map[string]string{"X-Amz-Meta-AppId": "a"}); err != nil { + map[string]string{"X-Amz-Meta-AppId": "a"}, sha256sum); err != nil { t.Fatal("Unexpected err: ", err) } @@ -130,8 +131,9 @@ func TestWriteFSMetadata(t *testing.T) { if err = obj.MakeBucket(bucketName); err != nil { t.Fatal("Unexpected err: ", err) } + sha256sum := "" if _, err = obj.PutObject(bucketName, objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")), - map[string]string{"X-Amz-Meta-AppId": "a"}); err != nil { + map[string]string{"X-Amz-Meta-AppId": "a"}, sha256sum); err != nil { t.Fatal("Unexpected err: ", err) } diff --git a/cmd/fs-v1-multipart.go b/cmd/fs-v1-multipart.go index a51085e53..e545831b5 100644 --- a/cmd/fs-v1-multipart.go +++ b/cmd/fs-v1-multipart.go @@ -18,8 +18,10 @@ package cmd import ( "crypto/md5" + "crypto/sha256" "encoding/hex" "fmt" + "hash" "io" "path" "strconv" @@ -392,7 +394,7 @@ func appendParts(disk StorageAPI, bucket, object, uploadID, opsID string) { // an ongoing multipart transaction. Internally incoming data is // written to '.minio.sys/tmp' location and safely renamed to // '.minio.sys/multipart' for reach parts. -func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, error) { +func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (string, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { return "", traceError(BucketNameInvalid{Bucket: bucket}) @@ -424,6 +426,14 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s // Initialize md5 writer. md5Writer := md5.New() + hashWriters := []io.Writer{md5Writer} + + var sha256Writer hash.Hash + if sha256sum != "" { + sha256Writer = sha256.New() + hashWriters = append(hashWriters, sha256Writer) + } + multiWriter := io.MultiWriter(hashWriters...) // Limit the reader to its provided size if specified. var limitDataReader io.Reader if size > 0 { @@ -434,7 +444,7 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s limitDataReader = data } - teeReader := io.TeeReader(limitDataReader, md5Writer) + teeReader := io.TeeReader(limitDataReader, multiWriter) bufSize := int64(readSizeV1) if size > 0 && bufSize > size { bufSize = size @@ -467,12 +477,23 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s if newMD5Hex != md5Hex { // MD5 mismatch, delete the temporary object. fs.storage.DeleteFile(minioMetaBucket, tmpPartPath) - // Returns md5 mismatch. + return "", traceError(BadDigest{md5Hex, newMD5Hex}) } } + if sha256sum != "" { + newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil)) + if newSHA256sum != sha256sum { + // SHA256 mismatch, delete the temporary object. + fs.storage.DeleteFile(minioMetaBucket, tmpPartPath) + return "", traceError(SHA256Mismatch{}) + } + } + // get a random ID for lock instrumentation. + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. opsID = getOpsID() // Hold write lock as we are updating fs.json diff --git a/cmd/fs-v1-multipart_test.go b/cmd/fs-v1-multipart_test.go index 3cd31901e..2b4cd7fe6 100644 --- a/cmd/fs-v1-multipart_test.go +++ b/cmd/fs-v1-multipart_test.go @@ -90,13 +90,14 @@ func TestPutObjectPartFaultyDisk(t *testing.T) { md5Writer := md5.New() md5Writer.Write(data) md5Hex := hex.EncodeToString(md5Writer.Sum(nil)) + sha256sum := "" // Test with faulty disk fsStorage := fs.storage.(*posix) for i := 1; i <= 7; i++ { // Faulty disk generates errFaultyDisk at 'i' storage api call number fs.storage = newNaughtyDisk(fsStorage, map[int]error{i: errFaultyDisk}, nil) - if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, dataLen, bytes.NewReader(data), md5Hex); errorCause(err) != errFaultyDisk { + if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, dataLen, bytes.NewReader(data), md5Hex, sha256sum); errorCause(err) != errFaultyDisk { switch i { case 1: if !isSameType(errorCause(err), BucketNotFound{}) { @@ -140,8 +141,9 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) { md5Writer := md5.New() md5Writer.Write(data) md5Hex := hex.EncodeToString(md5Writer.Sum(nil)) + sha256sum := "" - if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, 5, bytes.NewReader(data), md5Hex); err != nil { + if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, 5, bytes.NewReader(data), md5Hex, sha256sum); err != nil { t.Fatal("Unexpected error ", err) } @@ -195,8 +197,9 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) { md5Writer := md5.New() md5Writer.Write(data) md5Hex := hex.EncodeToString(md5Writer.Sum(nil)) + sha256sum := "" - if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, 5, bytes.NewReader(data), md5Hex); err != nil { + if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, 5, bytes.NewReader(data), md5Hex, sha256sum); err != nil { t.Fatal("Unexpected error ", err) } diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index 9d0666057..526d384a1 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -18,8 +18,10 @@ package cmd import ( "crypto/md5" + "crypto/sha256" "encoding/hex" "encoding/json" + "hash" "io" "os" "path" @@ -368,7 +370,7 @@ func (fs fsObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) { } // PutObject - create an object. -func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) { +func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { return ObjectInfo{}, traceError(BucketNameInvalid{Bucket: bucket}) @@ -394,6 +396,15 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io. // Initialize md5 writer. md5Writer := md5.New() + hashWriters := []io.Writer{md5Writer} + + var sha256Writer hash.Hash + if sha256sum != "" { + sha256Writer = sha256.New() + hashWriters = append(hashWriters, sha256Writer) + } + multiWriter := io.MultiWriter(hashWriters...) + // Limit the reader to its provided size if specified. var limitDataReader io.Reader if size > 0 { @@ -417,7 +428,7 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io. bufSize = size } buf := make([]byte, int(bufSize)) - teeReader := io.TeeReader(limitDataReader, md5Writer) + teeReader := io.TeeReader(limitDataReader, multiWriter) var bytesWritten int64 bytesWritten, err = fsCreateFile(fs.storage, teeReader, buf, minioMetaBucket, tempObj) if err != nil { @@ -460,6 +471,15 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io. } } + if sha256sum != "" { + newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil)) + if newSHA256sum != sha256sum { + // SHA256 mismatch, delete the temporary object. + fs.storage.DeleteFile(minioMetaBucket, tempObj) + return ObjectInfo{}, traceError(SHA256Mismatch{}) + } + } + // Entire object was written to the temp location, now it's safe to rename it to the actual location. err = fs.storage.RenameFile(minioMetaBucket, tempObj, bucket, object) if err != nil { diff --git a/cmd/fs-v1_test.go b/cmd/fs-v1_test.go index 10af51d2a..78fbc3969 100644 --- a/cmd/fs-v1_test.go +++ b/cmd/fs-v1_test.go @@ -84,7 +84,8 @@ func TestFSShutdown(t *testing.T) { objectContent := "12345" obj.MakeBucket(bucketName) - obj.PutObject(bucketName, objectName, int64(len(objectContent)), bytes.NewReader([]byte(objectContent)), nil) + sha256sum := "" + obj.PutObject(bucketName, objectName, int64(len(objectContent)), bytes.NewReader([]byte(objectContent)), nil, sha256sum) // Test Shutdown with regular conditions if err := fs.Shutdown(); err != nil { @@ -187,7 +188,8 @@ func TestFSDeleteObject(t *testing.T) { objectName := "object" obj.MakeBucket(bucketName) - obj.PutObject(bucketName, objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil) + sha256sum := "" + obj.PutObject(bucketName, objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum) // Test with invalid bucket name if err := fs.DeleteObject("fo", objectName); !isSameType(errorCause(err), BucketNameInvalid{}) { diff --git a/cmd/object-api-getobject_test.go b/cmd/object-api-getobject_test.go index 99ab8fc72..7a901d588 100644 --- a/cmd/object-api-getobject_test.go +++ b/cmd/object-api-getobject_test.go @@ -61,10 +61,11 @@ func testGetObject(obj ObjectLayer, instanceType string, t TestErrHandler) { // case - 1. {bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)}, } + sha256sum := "" // iterate through the above set of inputs and upkoad the object. for i, input := range putObjectInputs { // uploading the object. - _, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData) + _, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData, sha256sum) // if object upload fails stop the test. if err != nil { t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) @@ -212,10 +213,11 @@ func testGetObjectDiskNotFound(obj ObjectLayer, instanceType string, disks []str // case - 1. {bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)}, } + sha256sum := "" // iterate through the above set of inputs and upkoad the object. for i, input := range putObjectInputs { // uploading the object. - _, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData) + _, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData, sha256sum) // if object upload fails stop the test. if err != nil { t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) diff --git a/cmd/object-api-getobjectinfo_test.go b/cmd/object-api-getobjectinfo_test.go index ced333ff8..14730eede 100644 --- a/cmd/object-api-getobjectinfo_test.go +++ b/cmd/object-api-getobjectinfo_test.go @@ -33,7 +33,8 @@ func testGetObjectInfo(obj ObjectLayer, instanceType string, t TestErrHandler) { if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } - _, err = obj.PutObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil) + sha256sum := "" + _, err = obj.PutObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil, sha256sum) if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } diff --git a/cmd/object-api-listobjects_test.go b/cmd/object-api-listobjects_test.go index 1dc8983da..b74abacd3 100644 --- a/cmd/object-api-listobjects_test.go +++ b/cmd/object-api-listobjects_test.go @@ -62,8 +62,9 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) { {"obj1", "obj1"}, {"obj2", "obj2"}, } + sha256sum := "" for _, object := range testObjects { - _, err = obj.PutObject(testBuckets[0], object.name, int64(len(object.content)), bytes.NewBufferString(object.content), nil) + _, err = obj.PutObject(testBuckets[0], object.name, int64(len(object.content)), bytes.NewBufferString(object.content), nil, sha256sum) if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } @@ -583,9 +584,10 @@ func BenchmarkListObjects(b *testing.B) { b.Fatal(err) } + sha256sum := "" for i := 0; i < 20000; i++ { key := "obj" + strconv.Itoa(i) - _, err = obj.PutObject("ls-benchmark-bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) + _, err = obj.PutObject("ls-benchmark-bucket", key, int64(len(key)), bytes.NewBufferString(key), nil, sha256sum) if err != nil { b.Fatal(err) } diff --git a/cmd/object-api-multipart_test.go b/cmd/object-api-multipart_test.go index 372288d59..634d6e2d6 100644 --- a/cmd/object-api-multipart_test.go +++ b/cmd/object-api-multipart_test.go @@ -214,9 +214,10 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [ {bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("mnop")), "e132e96a5ddad6da8b07bba6f6131fef"}, {bucketNames[0], objectNames[0], uploadIDs[0], 5, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("mnop")), "e132e96a5ddad6da8b07bba6f6131fef"}, } + sha256sum := "" // Iterating over creatPartCases to generate multipart chunks. for _, testCase := range createPartCases { - _, err = obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5) + _, err = obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5, sha256sum) if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } @@ -230,7 +231,7 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [ // Object part upload should fail with quorum not available. testCase := createPartCases[len(createPartCases)-1] - _, err = obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5) + _, err = obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5, sha256sum) if err == nil { t.Fatalf("Test %s: expected to fail but passed instead", instanceType) } @@ -279,6 +280,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH PartID int inputReaderData string inputMd5 string + inputSHA256 string intputDataSize int64 // flag indicating whether the test should pass. shouldPass bool @@ -288,60 +290,63 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH }{ // Test case 1-4. // Cases with invalid bucket name. - {".test", "obj", "", 1, "", "", 0, false, "", fmt.Errorf("%s", "Bucket name invalid: .test")}, - {"------", "obj", "", 1, "", "", 0, false, "", fmt.Errorf("%s", "Bucket name invalid: ------")}, - {"$this-is-not-valid-too", "obj", "", 1, "", "", 0, false, "", + {".test", "obj", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Bucket name invalid: .test")}, + {"------", "obj", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Bucket name invalid: ------")}, + {"$this-is-not-valid-too", "obj", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Bucket name invalid: $this-is-not-valid-too")}, - {"a", "obj", "", 1, "", "", 0, false, "", fmt.Errorf("%s", "Bucket name invalid: a")}, + {"a", "obj", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Bucket name invalid: a")}, // Test case - 5. // Case with invalid object names. - {bucket, "", "", 1, "", "", 0, false, "", fmt.Errorf("%s", "Object name invalid: minio-bucket#")}, + {bucket, "", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Object name invalid: minio-bucket#")}, // Test case - 6. // Valid object and bucket names but non-existent bucket. - {"abc", "def", "", 1, "", "", 0, false, "", fmt.Errorf("%s", "Bucket not found: abc")}, + {"abc", "def", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Bucket not found: abc")}, // Test Case - 7. // Existing bucket, but using a bucket on which NewMultipartUpload is not Initiated. - {"unused-bucket", "def", "xyz", 1, "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id xyz")}, + {"unused-bucket", "def", "xyz", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id xyz")}, // Test Case - 8. // Existing bucket, object name different from which NewMultipartUpload is constructed from. // Expecting "Invalid upload id". - {bucket, "def", "xyz", 1, "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id xyz")}, + {bucket, "def", "xyz", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id xyz")}, // Test Case - 9. // Existing bucket, bucket and object name are the ones from which NewMultipartUpload is constructed from. // But the uploadID is invalid. // Expecting "Invalid upload id". - {bucket, object, "xyz", 1, "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id xyz")}, + {bucket, object, "xyz", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id xyz")}, // Test Case - 10. // Case with valid UploadID, existing bucket name. // But using the bucket name from which NewMultipartUpload is not constructed from. - {"unused-bucket", object, uploadID, 1, "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id "+uploadID)}, + {"unused-bucket", object, uploadID, 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id "+uploadID)}, // Test Case - 11. // Case with valid UploadID, existing bucket name. // But using the object name from which NewMultipartUpload is not constructed from. - {bucket, "none-object", uploadID, 1, "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id "+uploadID)}, + {bucket, "none-object", uploadID, 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Invalid upload id "+uploadID)}, // Test case - 12. // Input to replicate Md5 mismatch. - {bucket, object, uploadID, 1, "", "a35", 0, false, "", + {bucket, object, uploadID, 1, "", "a35", "", 0, false, "", fmt.Errorf("%s", "Bad digest: Expected a35 is not valid with what we calculated "+"d41d8cd98f00b204e9800998ecf8427e")}, // Test case - 13. - // Input with size more than the size of actual data inside the reader. - {bucket, object, uploadID, 1, "abcd", "a35", int64(len("abcd") + 1), false, "", - IncompleteBody{}}, + // When incorrect sha256 is provided. + {bucket, object, uploadID, 1, "", "", "incorrect-sha256", 0, false, "", SHA256Mismatch{}}, // Test case - 14. + // Input with size more than the size of actual data inside the reader. + {bucket, object, uploadID, 1, "abcd", "a35", "", int64(len("abcd") + 1), false, "", IncompleteBody{}}, + // Test case - 15. // Input with size less than the size of actual data inside the reader. - {bucket, object, uploadID, 1, "abcd", "a35", int64(len("abcd") - 1), false, "", + {bucket, object, uploadID, 1, "abcd", "a35", "", int64(len("abcd") - 1), false, "", fmt.Errorf("%s", "Bad digest: Expected a35 is not valid with what we calculated 900150983cd24fb0d6963f7d28e17f72")}, - // Test case - 15-18. + + // Test case - 16-19. // Validating for success cases. - {bucket, object, uploadID, 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), true, "", nil}, - {bucket, object, uploadID, 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh")), true, "", nil}, - {bucket, object, uploadID, 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd")), true, "", nil}, - {bucket, object, uploadID, 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd")), true, "", nil}, + {bucket, object, uploadID, 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", int64(len("abcd")), true, "", nil}, + {bucket, object, uploadID, 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", "e5e088a0b66163a0a26a5e053d2a4496dc16ab6e0e3dd1adf2d16aa84a078c9d", int64(len("efgh")), true, "", nil}, + {bucket, object, uploadID, 3, "ijkl", "09a0877d04abf8759f99adec02baf579", "005c19658919186b85618c5870463eec8d9b8c1a9d00208a5352891ba5bbe086", int64(len("abcd")), true, "", nil}, + {bucket, object, uploadID, 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", "f1afc31479522d6cff1ed068f93998f05a8cd3b22f5c37d7f307084f62d1d270", int64(len("abcd")), true, "", nil}, } // Validate all the test cases. for i, testCase := range testCases { - actualMd5Hex, actualErr := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5) + actualMd5Hex, actualErr := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5, testCase.inputSHA256) // All are test cases above are expected to fail. if actualErr != nil && testCase.shouldPass { t.Errorf("Test %d: %s: Expected to pass, but failed with: %s.", i+1, instanceType, actualErr.Error()) @@ -472,9 +477,10 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan {bucketNames[2], objectNames[4], uploadIDs[8], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"}, {bucketNames[2], objectNames[5], uploadIDs[9], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"}, } + sha256sum := "" // Iterating over creatPartCases to generate multipart chunks. for _, testCase := range createPartCases { - _, err := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5) + _, err := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5, sha256sum) if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } @@ -1319,9 +1325,10 @@ func testListObjectPartsDiskNotFound(obj ObjectLayer, instanceType string, disks {bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd")), "09a0877d04abf8759f99adec02baf579"}, {bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd")), "e132e96a5ddad6da8b07bba6f6131fef"}, } + sha256sum := "" // Iterating over creatPartCases to generate multipart chunks. for _, testCase := range createPartCases { - _, err := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5) + _, err := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5, sha256sum) if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } @@ -1558,9 +1565,10 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler) {bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd")), "09a0877d04abf8759f99adec02baf579"}, {bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd")), "e132e96a5ddad6da8b07bba6f6131fef"}, } + sha256sum := "" // Iterating over creatPartCases to generate multipart chunks. for _, testCase := range createPartCases { - _, err := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5) + _, err := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5, sha256sum) if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } @@ -1805,9 +1813,10 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T {bucketNames[0], objectNames[0], uploadIDs[0], 5, string(validPart), validPartMD5, int64(len(string(validPart)))}, {bucketNames[0], objectNames[0], uploadIDs[0], 6, string(validPart), validPartMD5, int64(len(string(validPart)))}, } + sha256sum := "" // Iterating over creatPartCases to generate multipart chunks. for _, part := range parts { - _, err = obj.PutObjectPart(part.bucketName, part.objName, part.uploadID, part.PartID, part.intputDataSize, bytes.NewBufferString(part.inputReaderData), part.inputMd5) + _, err = obj.PutObjectPart(part.bucketName, part.objName, part.uploadID, part.PartID, part.intputDataSize, bytes.NewBufferString(part.inputReaderData), part.inputMd5, sha256sum) if err != nil { t.Fatalf("%s : %s", instanceType, err) } diff --git a/cmd/object-api-putobject_test.go b/cmd/object-api-putobject_test.go index 44ca8e7d3..ed33b85bb 100644 --- a/cmd/object-api-putobject_test.go +++ b/cmd/object-api-putobject_test.go @@ -76,6 +76,7 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl objName string inputData []byte inputMeta map[string]string + inputSHA256 string intputDataSize int64 // expected error output. expectedMd5 string @@ -83,79 +84,83 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl }{ // Test case 1-4. // Cases with invalid bucket name. - {".test", "obj", []byte(""), nil, 0, "", BucketNameInvalid{Bucket: ".test"}}, - {"------", "obj", []byte(""), nil, 0, "", BucketNameInvalid{Bucket: "------"}}, - {"$this-is-not-valid-too", "obj", []byte(""), nil, 0, "", + {".test", "obj", []byte(""), nil, "", 0, "", BucketNameInvalid{Bucket: ".test"}}, + {"------", "obj", []byte(""), nil, "", 0, "", BucketNameInvalid{Bucket: "------"}}, + {"$this-is-not-valid-too", "obj", []byte(""), nil, "", 0, "", BucketNameInvalid{Bucket: "$this-is-not-valid-too"}}, - {"a", "obj", []byte(""), nil, 0, "", BucketNameInvalid{Bucket: "a"}}, + {"a", "obj", []byte(""), nil, "", 0, "", BucketNameInvalid{Bucket: "a"}}, // Test case - 5. // Case with invalid object names. - {bucket, "", []byte(""), nil, 0, "", ObjectNameInvalid{Bucket: bucket, Object: ""}}, + {bucket, "", []byte(""), nil, "", 0, "", ObjectNameInvalid{Bucket: bucket, Object: ""}}, // Test case - 6. // Valid object and bucket names but non-existent bucket. - {"abc", "def", []byte(""), nil, 0, "", BucketNotFound{Bucket: "abc"}}, + {"abc", "def", []byte(""), nil, "", 0, "", BucketNotFound{Bucket: "abc"}}, // Test case - 7. // Input to replicate Md5 mismatch. - {bucket, object, []byte(""), map[string]string{"md5Sum": "a35"}, 0, "", + {bucket, object, []byte(""), map[string]string{"md5Sum": "a35"}, "", 0, "", BadDigest{ExpectedMD5: "a35", CalculatedMD5: "d41d8cd98f00b204e9800998ecf8427e"}}, // Test case - 8. - // Input with size more than the size of actual data inside the reader. - {bucket, object, []byte("abcd"), map[string]string{"md5Sum": "a35"}, int64(len("abcd") + 1), "", - IncompleteBody{}}, + // With incorrect sha256. + {bucket, object, []byte("abcd"), map[string]string{"md5Sum": "e2fc714c4727ee9395f324cd2e7f331f"}, "incorrect-sha256", int64(len("abcd")), "", SHA256Mismatch{}}, // Test case - 9. + // Input with size more than the size of actual data inside the reader. + {bucket, object, []byte("abcd"), map[string]string{"md5Sum": "a35"}, "", int64(len("abcd") + 1), "", + IncompleteBody{}}, + + // Test case - 10. // Input with size less than the size of actual data inside the reader. - {bucket, object, []byte("abcd"), map[string]string{"md5Sum": "a35"}, int64(len("abcd") - 1), "", + {bucket, object, []byte("abcd"), map[string]string{"md5Sum": "a35"}, "", int64(len("abcd") - 1), "", BadDigest{ExpectedMD5: "a35", CalculatedMD5: "900150983cd24fb0d6963f7d28e17f72"}}, - // Test case - 10-13. + // Test case - 11-14. // Validating for success cases. - {bucket, object, []byte("abcd"), map[string]string{"md5Sum": "e2fc714c4727ee9395f324cd2e7f331f"}, int64(len("abcd")), "", nil}, - {bucket, object, []byte("efgh"), map[string]string{"md5Sum": "1f7690ebdd9b4caf8fab49ca1757bf27"}, int64(len("efgh")), "", nil}, - {bucket, object, []byte("ijkl"), map[string]string{"md5Sum": "09a0877d04abf8759f99adec02baf579"}, int64(len("ijkl")), "", nil}, - {bucket, object, []byte("mnop"), map[string]string{"md5Sum": "e132e96a5ddad6da8b07bba6f6131fef"}, int64(len("mnop")), "", nil}, + {bucket, object, []byte("abcd"), map[string]string{"md5Sum": "e2fc714c4727ee9395f324cd2e7f331f"}, "", int64(len("abcd")), "", nil}, + {bucket, object, []byte("efgh"), map[string]string{"md5Sum": "1f7690ebdd9b4caf8fab49ca1757bf27"}, "", int64(len("efgh")), "", nil}, + {bucket, object, []byte("ijkl"), map[string]string{"md5Sum": "09a0877d04abf8759f99adec02baf579"}, "", int64(len("ijkl")), "", nil}, + {bucket, object, []byte("mnop"), map[string]string{"md5Sum": "e132e96a5ddad6da8b07bba6f6131fef"}, "", int64(len("mnop")), "", nil}, - // Test case 14-16. + // Test case 15-17. // With no metadata - {bucket, object, data, nil, int64(len(data)), md5Hex(data), nil}, - {bucket, object, nilBytes, nil, int64(len(nilBytes)), md5Hex(nilBytes), nil}, - {bucket, object, fiveMBBytes, nil, int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), nil}, + {bucket, object, data, nil, "", int64(len(data)), md5Hex(data), nil}, + {bucket, object, nilBytes, nil, "", int64(len(nilBytes)), md5Hex(nilBytes), nil}, + {bucket, object, fiveMBBytes, nil, "", int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), nil}, - // Test case 17-19. + // Test case 18-20. // With arbitrary metadata - {bucket, object, data, map[string]string{"answer": "42"}, int64(len(data)), md5Hex(data), nil}, - {bucket, object, nilBytes, map[string]string{"answer": "42"}, int64(len(nilBytes)), md5Hex(nilBytes), nil}, - {bucket, object, fiveMBBytes, map[string]string{"answer": "42"}, int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), nil}, + {bucket, object, data, map[string]string{"answer": "42"}, "", int64(len(data)), md5Hex(data), nil}, + {bucket, object, nilBytes, map[string]string{"answer": "42"}, "", int64(len(nilBytes)), md5Hex(nilBytes), nil}, + {bucket, object, fiveMBBytes, map[string]string{"answer": "42"}, "", int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), nil}, - // Test case 20-22. - // With valid md5sum in header - {bucket, object, data, md5Header(data), int64(len(data)), md5Hex(data), nil}, - {bucket, object, nilBytes, md5Header(nilBytes), int64(len(nilBytes)), md5Hex(nilBytes), nil}, - {bucket, object, fiveMBBytes, md5Header(fiveMBBytes), int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), nil}, + // Test case 21-23. + // With valid md5sum and sha256. + {bucket, object, data, md5Header(data), hex.EncodeToString(sum256(data)), int64(len(data)), md5Hex(data), nil}, + {bucket, object, nilBytes, md5Header(nilBytes), hex.EncodeToString(sum256(nilBytes)), int64(len(nilBytes)), md5Hex(nilBytes), nil}, + {bucket, object, fiveMBBytes, md5Header(fiveMBBytes), hex.EncodeToString(sum256(fiveMBBytes)), int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), nil}, - // Test case 23-25. + // Test case 24-26. // data with invalid md5sum in header - {bucket, object, data, invalidMD5Header, int64(len(data)), md5Hex(data), BadDigest{invalidMD5, md5Hex(data)}}, - {bucket, object, nilBytes, invalidMD5Header, int64(len(nilBytes)), md5Hex(nilBytes), BadDigest{invalidMD5, md5Hex(nilBytes)}}, - {bucket, object, fiveMBBytes, invalidMD5Header, int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), BadDigest{invalidMD5, md5Hex(fiveMBBytes)}}, + {bucket, object, data, invalidMD5Header, "", int64(len(data)), md5Hex(data), BadDigest{invalidMD5, md5Hex(data)}}, + {bucket, object, nilBytes, invalidMD5Header, "", int64(len(nilBytes)), md5Hex(nilBytes), BadDigest{invalidMD5, md5Hex(nilBytes)}}, + {bucket, object, fiveMBBytes, invalidMD5Header, "", int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), BadDigest{invalidMD5, md5Hex(fiveMBBytes)}}, - // Test case 26-28. + // Test case 27-29. // data with size different from the actual number of bytes available in the reader - {bucket, object, data, nil, int64(len(data) - 1), md5Hex(data[:len(data)-1]), nil}, - {bucket, object, nilBytes, nil, int64(len(nilBytes) + 1), md5Hex(nilBytes), IncompleteBody{}}, - {bucket, object, fiveMBBytes, nil, int64(0), md5Hex(fiveMBBytes), nil}, + {bucket, object, data, nil, "", int64(len(data) - 1), md5Hex(data[:len(data)-1]), nil}, + {bucket, object, nilBytes, nil, "", int64(len(nilBytes) + 1), md5Hex(nilBytes), IncompleteBody{}}, + {bucket, object, fiveMBBytes, nil, "", int64(0), md5Hex(fiveMBBytes), nil}, - // Test case 29 + // Test case 30 // valid data with X-Amz-Meta- meta - {bucket, object, data, map[string]string{"X-Amz-Meta-AppID": "a42"}, int64(len(data)), md5Hex(data), nil}, + {bucket, object, data, map[string]string{"X-Amz-Meta-AppID": "a42"}, "", int64(len(data)), md5Hex(data), nil}, } for i, testCase := range testCases { - objInfo, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta) + objInfo, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta, testCase.inputSHA256) actualErr = errorCause(actualErr) if actualErr != nil && testCase.expectedError == nil { t.Errorf("Test %d: %s: Expected to pass, but failed with: error %s.", i+1, instanceType, actualErr.Error()) @@ -227,8 +232,9 @@ func testObjectAPIPutObjectDiskNotFOund(obj ObjectLayer, instanceType string, di {bucket, object, []byte("mnop"), map[string]string{"md5Sum": "e132e96a5ddad6da8b07bba6f6131fef"}, int64(len("mnop")), true, "", nil}, } + sha256sum := "" for i, testCase := range testCases { - objInfo, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta) + objInfo, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta, sha256sum) actualErr = errorCause(err) if actualErr != nil && testCase.shouldPass { t.Errorf("Test %d: %s: Expected to pass, but failed with: %s.", i+1, instanceType, actualErr.Error()) @@ -277,7 +283,8 @@ func testObjectAPIPutObjectDiskNotFOund(obj ObjectLayer, instanceType string, di "", InsufficientWriteQuorum{}, } - _, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta) + + _, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta, sha256sum) actualErr = errorCause(actualErr) if actualErr != nil && testCase.shouldPass { t.Errorf("Test %d: %s: Expected to pass, but failed with: %s.", len(testCases)+1, instanceType, actualErr.Error()) @@ -309,8 +316,9 @@ func testObjectAPIPutObjectStaleFiles(obj ObjectLayer, instanceType string, disk } data := []byte("hello, world") + sha256sum := "" // Create object. - _, err = obj.PutObject(bucket, object, int64(len(data)), bytes.NewReader(data), nil) + _, err = obj.PutObject(bucket, object, int64(len(data)), bytes.NewReader(data), nil, sha256sum) if err != nil { // Failed to create object, abort. t.Fatalf("%s : %s", instanceType, err.Error()) @@ -354,7 +362,8 @@ func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType str md5Writer := md5.New() md5Writer.Write(fiveMBBytes) etag1 := hex.EncodeToString(md5Writer.Sum(nil)) - _, err = obj.PutObjectPart(bucket, object, uploadID, 1, int64(len(fiveMBBytes)), bytes.NewReader(fiveMBBytes), etag1) + sha256sum := "" + _, err = obj.PutObjectPart(bucket, object, uploadID, 1, int64(len(fiveMBBytes)), bytes.NewReader(fiveMBBytes), etag1, sha256sum) if err != nil { // Failed to upload object part, abort. t.Fatalf("%s : %s", instanceType, err.Error()) @@ -365,7 +374,7 @@ func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType str md5Writer = md5.New() md5Writer.Write(data) etag2 := hex.EncodeToString(md5Writer.Sum(nil)) - _, err = obj.PutObjectPart(bucket, object, uploadID, 2, int64(len(data)), bytes.NewReader(data), etag2) + _, err = obj.PutObjectPart(bucket, object, uploadID, 2, int64(len(data)), bytes.NewReader(data), etag2, sha256sum) if err != nil { // Failed to upload object part, abort. t.Fatalf("%s : %s", instanceType, err.Error()) diff --git a/cmd/object-errors.go b/cmd/object-errors.go index e9ef85753..07590e694 100644 --- a/cmd/object-errors.go +++ b/cmd/object-errors.go @@ -73,6 +73,8 @@ func toObjectErr(err error, params ...string) error { err = InsufficientWriteQuorum{} case io.ErrUnexpectedEOF, io.ErrShortWrite: err = IncompleteBody{} + case errContentSHA256Mismatch: + err = SHA256Mismatch{} } if ok { e.e = err @@ -81,6 +83,13 @@ func toObjectErr(err error, params ...string) error { return err } +// SHA256Mismatch - when content sha256 does not match with what was sent from client. +type SHA256Mismatch struct{} + +func (e SHA256Mismatch) Error() string { + return "sha256 computed does not match with what is expected" +} + // StorageFull storage ran out of space. type StorageFull struct{} diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 50db9e64a..8d147a3a4 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -379,8 +379,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re // Do not set `md5sum` as CopyObject will not keep the // same md5sum as the source. + sha256sum := "" // Create the object. - objInfo, err = objectAPI.PutObject(bucket, object, size, pipeReader, metadata) + objInfo, err = objectAPI.PutObject(bucket, object, size, pipeReader, metadata, sha256sum) if err != nil { // Close the this end of the pipe upon error in PutObject. pipeReader.CloseWithError(err) @@ -466,6 +467,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req // Make sure we hex encode md5sum here. metadata["md5Sum"] = hex.EncodeToString(md5Bytes) + sha256sum := "" + var objInfo ObjectInfo switch rAuthType { default: @@ -479,7 +482,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req return } // Create anonymous object. - objInfo, err = objectAPI.PutObject(bucket, object, size, r.Body, metadata) + objInfo, err = objectAPI.PutObject(bucket, object, size, r.Body, metadata, sha256sum) case authTypeStreamingSigned: // Initialize stream signature verifier. reader, s3Error := newSignV4ChunkedReader(r) @@ -488,7 +491,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req writeErrorResponse(w, r, s3Error, r.URL.Path) return } - objInfo, err = objectAPI.PutObject(bucket, object, size, reader, metadata) + objInfo, err = objectAPI.PutObject(bucket, object, size, reader, metadata, sha256sum) case authTypeSignedV2, authTypePresignedV2: s3Error := isReqAuthenticatedV2(r) if s3Error != ErrNone { @@ -496,12 +499,18 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req writeErrorResponse(w, r, s3Error, r.URL.Path) return } - objInfo, err = objectAPI.PutObject(bucket, object, size, r.Body, metadata) + objInfo, err = objectAPI.PutObject(bucket, object, size, r.Body, metadata, sha256sum) case authTypePresigned, authTypeSigned: - // Initialize signature verifier. - reader := newSignVerify(r) + if s3Error := reqSignatureV4Verify(r); s3Error != ErrNone { + errorIf(errSignatureMismatch, dumpRequest(r)) + writeErrorResponse(w, r, s3Error, r.URL.Path) + return + } + if !skipContentSha256Cksum(r) { + sha256sum = r.Header.Get("X-Amz-Content-Sha256") + } // Create object. - objInfo, err = objectAPI.PutObject(bucket, object, size, reader, metadata) + objInfo, err = objectAPI.PutObject(bucket, object, size, r.Body, metadata, sha256sum) } if err != nil { errorIf(err, "Unable to create an object.") @@ -642,6 +651,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http var partMD5 string incomingMD5 := hex.EncodeToString(md5Bytes) + sha256sum := "" switch rAuthType { default: // For all unknown auth types return error. @@ -654,7 +664,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http return } // No need to verify signature, anonymous request access is already allowed. - partMD5, err = objectAPI.PutObjectPart(bucket, object, uploadID, partID, size, r.Body, incomingMD5) + partMD5, err = objectAPI.PutObjectPart(bucket, object, uploadID, partID, size, r.Body, incomingMD5, sha256sum) case authTypeStreamingSigned: // Initialize stream signature verifier. reader, s3Error := newSignV4ChunkedReader(r) @@ -663,7 +673,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http writeErrorResponse(w, r, s3Error, r.URL.Path) return } - partMD5, err = objectAPI.PutObjectPart(bucket, object, uploadID, partID, size, reader, incomingMD5) + partMD5, err = objectAPI.PutObjectPart(bucket, object, uploadID, partID, size, reader, incomingMD5, sha256sum) case authTypeSignedV2, authTypePresignedV2: s3Error := isReqAuthenticatedV2(r) if s3Error != ErrNone { @@ -671,11 +681,18 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http writeErrorResponse(w, r, s3Error, r.URL.Path) return } - partMD5, err = objectAPI.PutObjectPart(bucket, object, uploadID, partID, size, r.Body, incomingMD5) + partMD5, err = objectAPI.PutObjectPart(bucket, object, uploadID, partID, size, r.Body, incomingMD5, sha256sum) case authTypePresigned, authTypeSigned: - // Initialize signature verifier. - reader := newSignVerify(r) - partMD5, err = objectAPI.PutObjectPart(bucket, object, uploadID, partID, size, reader, incomingMD5) + if s3Error := reqSignatureV4Verify(r); s3Error != ErrNone { + errorIf(errSignatureMismatch, dumpRequest(r)) + writeErrorResponse(w, r, s3Error, r.URL.Path) + return + } + + if !skipContentSha256Cksum(r) { + sha256sum = r.Header.Get("X-Amz-Content-Sha256") + } + partMD5, err = objectAPI.PutObjectPart(bucket, object, uploadID, partID, size, r.Body, incomingMD5, sha256sum) } if err != nil { errorIf(err, "Unable to create object part.") diff --git a/cmd/object-handlers_test.go b/cmd/object-handlers_test.go index c82f8b8d6..abf7f9f08 100644 --- a/cmd/object-handlers_test.go +++ b/cmd/object-handlers_test.go @@ -71,10 +71,11 @@ func testAPIGetOjectHandler(obj ObjectLayer, instanceType, bucketName string, ap // case - 1. {bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)}, } + sha256sum := "" // iterate through the above set of inputs and upload the object. for i, input := range putObjectInputs { // uploading the object. - _, err := obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData) + _, err := obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData, sha256sum) // if object upload fails stop the test. if err != nil { t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) @@ -430,10 +431,11 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, // case - 1. {bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)}, } + sha256sum := "" // iterate through the above set of inputs and upload the object. for i, input := range putObjectInputs { // uploading the object. - _, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData) + _, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData, sha256sum) // if object upload fails stop the test. if err != nil { t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) @@ -683,7 +685,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s } // Iterating over creatPartCases to generate multipart chunks. for _, part := range parts { - _, err = obj.PutObjectPart(part.bucketName, part.objName, part.uploadID, part.PartID, part.intputDataSize, bytes.NewBufferString(part.inputReaderData), part.inputMd5) + _, err = obj.PutObjectPart(part.bucketName, part.objName, part.uploadID, part.PartID, part.intputDataSize, bytes.NewBufferString(part.inputReaderData), part.inputMd5, "") if err != nil { t.Fatalf("%s : %s", instanceType, err) } @@ -905,7 +907,7 @@ func testAPIDeleteOjectHandler(obj ObjectLayer, instanceType, bucketName string, // iterate through the above set of inputs and upload the object. for i, input := range putObjectInputs { // uploading the object. - _, err := obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData) + _, err := obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData, "") // if object upload fails stop the test. if err != nil { t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) @@ -1211,7 +1213,7 @@ func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName strin NoAPIErr := APIError{} MissingContent := getAPIError(ErrMissingContentLength) EntityTooLarge := getAPIError(ErrEntityTooLarge) - BadSigning := getAPIError(ErrContentSHA256Mismatch) + BadSigning := getAPIError(ErrSignatureDoesNotMatch) BadChecksum := getAPIError(ErrInvalidDigest) InvalidPart := getAPIError(ErrInvalidPart) InvalidMaxParts := getAPIError(ErrInvalidMaxParts) @@ -1248,7 +1250,8 @@ func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName strin case TooBigObject: tReq.ContentLength = maxObjectSize + 1 case BadSignature: - tReq.Header.Set("x-amz-content-sha256", "somethingElse") + // Mangle signature + tReq.Header.Set("authorization", tReq.Header.Get("authorization")+"a") case BadMD5: tReq.Header.Set("Content-MD5", "badmd5") } diff --git a/cmd/object-interface.go b/cmd/object-interface.go index 943357667..9f0afd9aa 100644 --- a/cmd/object-interface.go +++ b/cmd/object-interface.go @@ -36,14 +36,14 @@ type ObjectLayer interface { // Object operations. GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) GetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) - PutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string) (objInto ObjectInfo, err error) + PutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInto ObjectInfo, err error) DeleteObject(bucket, object string) error HealObject(bucket, object string) error // Multipart operations. ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) NewMultipartUpload(bucket, object string, metadata map[string]string) (uploadID string, err error) - PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (md5 string, err error) + PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (md5 string, err error) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error) AbortMultipartUpload(bucket, object, uploadID string) error CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []completePart) (md5 string, err error) diff --git a/cmd/object_api_suite_test.go b/cmd/object_api_suite_test.go index e53a2151f..d1203bcc9 100644 --- a/cmd/object_api_suite_test.go +++ b/cmd/object_api_suite_test.go @@ -109,7 +109,7 @@ func testMultipartObjectCreation(obj ObjectLayer, instanceType string, c TestErr expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil)) var calculatedMD5sum string - calculatedMD5sum, err = obj.PutObjectPart("bucket", "key", uploadID, i, int64(len(data)), bytes.NewBuffer(data), expectedMD5Sumhex) + calculatedMD5sum, err = obj.PutObjectPart("bucket", "key", uploadID, i, int64(len(data)), bytes.NewBuffer(data), expectedMD5Sumhex, "") if err != nil { c.Errorf("%s: %s", instanceType, err) } @@ -158,7 +158,7 @@ func testMultipartObjectAbort(obj ObjectLayer, instanceType string, c TestErrHan metadata["md5"] = expectedMD5Sumhex var calculatedMD5sum string - calculatedMD5sum, err = obj.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex) + calculatedMD5sum, err = obj.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex, "") if err != nil { c.Fatalf("%s: %s", instanceType, err) } @@ -201,7 +201,7 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, c TestErrH metadata := make(map[string]string) metadata["md5Sum"] = expectedMD5Sumhex var objInfo ObjectInfo - objInfo, err = obj.PutObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), metadata) + objInfo, err = obj.PutObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), metadata, "") if err != nil { c.Fatalf("%s: %s", instanceType, err) } @@ -254,7 +254,7 @@ func testPaging(obj ObjectLayer, instanceType string, c TestErrHandler) { // check before paging occurs. for i := 0; i < 5; i++ { key := "obj" + strconv.Itoa(i) - _, err = obj.PutObject("bucket", key, int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil) + _, err = obj.PutObject("bucket", key, int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "") if err != nil { c.Fatalf("%s: %s", instanceType, err) } @@ -274,7 +274,7 @@ func testPaging(obj ObjectLayer, instanceType string, c TestErrHandler) { // check after paging occurs pages work. for i := 6; i <= 10; i++ { key := "obj" + strconv.Itoa(i) - _, err = obj.PutObject("bucket", key, int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil) + _, err = obj.PutObject("bucket", key, int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "") if err != nil { c.Fatalf("%s: %s", instanceType, err) } @@ -291,11 +291,11 @@ func testPaging(obj ObjectLayer, instanceType string, c TestErrHandler) { } // check paging with prefix at end returns less objects. { - _, err = obj.PutObject("bucket", "newPrefix", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil) + _, err = obj.PutObject("bucket", "newPrefix", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "") if err != nil { c.Fatalf("%s: %s", instanceType, err) } - _, err = obj.PutObject("bucket", "newPrefix2", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil) + _, err = obj.PutObject("bucket", "newPrefix2", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "") if err != nil { c.Fatalf("%s: %s", instanceType, err) } @@ -333,11 +333,11 @@ func testPaging(obj ObjectLayer, instanceType string, c TestErrHandler) { // check delimited results with delimiter and prefix. { - _, err = obj.PutObject("bucket", "this/is/delimited", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil) + _, err = obj.PutObject("bucket", "this/is/delimited", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "") if err != nil { c.Fatalf("%s: %s", instanceType, err) } - _, err = obj.PutObject("bucket", "this/is/also/a/delimited/file", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil) + _, err = obj.PutObject("bucket", "this/is/also/a/delimited/file", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "") if err != nil { c.Fatalf("%s: %s", instanceType, err) } @@ -446,14 +446,14 @@ func testObjectOverwriteWorks(obj ObjectLayer, instanceType string, c TestErrHan c.Fatalf("%s: %s", instanceType, err) } - _, err = obj.PutObject("bucket", "object", int64(len("The list of parts was not in ascending order. The parts list must be specified in order by part number.")), bytes.NewBufferString("The list of parts was not in ascending order. The parts list must be specified in order by part number."), nil) + _, err = obj.PutObject("bucket", "object", int64(len("The list of parts was not in ascending order. The parts list must be specified in order by part number.")), bytes.NewBufferString("The list of parts was not in ascending order. The parts list must be specified in order by part number."), nil, "") if err != nil { c.Fatalf("%s: %s", instanceType, err) } uploadContent := "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed." length := int64(len(uploadContent)) - _, err = obj.PutObject("bucket", "object", length, bytes.NewBufferString(uploadContent), nil) + _, err = obj.PutObject("bucket", "object", length, bytes.NewBufferString(uploadContent), nil, "") if err != nil { c.Fatalf("%s: %s", instanceType, err) } @@ -475,7 +475,7 @@ func (s *ObjectLayerAPISuite) TestNonExistantBucketOperations(c *C) { // Tests validate that bucket operation on non-existent bucket fails. func testNonExistantBucketOperations(obj ObjectLayer, instanceType string, c TestErrHandler) { - _, err := obj.PutObject("bucket1", "object", int64(len("one")), bytes.NewBufferString("one"), nil) + _, err := obj.PutObject("bucket1", "object", int64(len("one")), bytes.NewBufferString("one"), nil, "") if err == nil { c.Fatal("Expected error but found nil") } @@ -522,7 +522,7 @@ func testPutObject(obj ObjectLayer, instanceType string, c TestErrHandler) { } var bytesBuffer1 bytes.Buffer - _, err = obj.PutObject("bucket", "object", length, readerEOF, nil) + _, err = obj.PutObject("bucket", "object", length, readerEOF, nil, "") if err != nil { c.Fatalf("%s: %s", instanceType, err) } @@ -535,7 +535,7 @@ func testPutObject(obj ObjectLayer, instanceType string, c TestErrHandler) { } var bytesBuffer2 bytes.Buffer - _, err = obj.PutObject("bucket", "object", length, readerNoEOF, nil) + _, err = obj.PutObject("bucket", "object", length, readerNoEOF, nil, "") if err != nil { c.Fatalf("%s: %s", instanceType, err) } @@ -563,7 +563,7 @@ func testPutObjectInSubdir(obj ObjectLayer, instanceType string, c TestErrHandle uploadContent := `The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.` length := int64(len(uploadContent)) - _, err = obj.PutObject("bucket", "dir1/dir2/object", length, bytes.NewBufferString(uploadContent), nil) + _, err = obj.PutObject("bucket", "dir1/dir2/object", length, bytes.NewBufferString(uploadContent), nil, "") if err != nil { c.Fatalf("%s: %s", instanceType, err) } @@ -737,7 +737,7 @@ func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string, _, err = obj.PutObject("bucket", "dir1/dir3/object", int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")), - bytes.NewBufferString("One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tag might not have matched the part's entity tag."), nil) + bytes.NewBufferString("One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tag might not have matched the part's entity tag."), nil, "") if err != nil { c.Fatalf("%s: %s", instanceType, err) @@ -790,7 +790,7 @@ func testContentType(obj ObjectLayer, instanceType string, c TestErrHandler) { } uploadContent := "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed." // Test empty. - _, err = obj.PutObject("bucket", "minio.png", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil) + _, err = obj.PutObject("bucket", "minio.png", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "") if err != nil { c.Fatalf("%s: %s", instanceType, err) } diff --git a/cmd/server_test.go b/cmd/server_test.go index 41ada0864..e69a728a8 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -1091,8 +1091,7 @@ func (s *TestSuiteCommon) TestSHA256Mismatch(c *C) { // Body is on purpose set to nil so that we get payload generated for empty bytes. // Create new HTTP request with incorrect secretKey to generate an incorrect signature. - secretKey := s.secretKey + "a" - request, err = newTestSignedRequestV4("PUT", getPutObjectURL(s.endPoint, bucketName, objName), 0, nil, s.accessKey, secretKey) + request, err = newTestSignedRequestV4("PUT", getPutObjectURL(s.endPoint, bucketName, objName), 0, nil, s.accessKey, s.secretKey) c.Assert(request.Header.Get("x-amz-content-sha256"), Equals, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") // Set the body to generate signature mismatch. request.Body = ioutil.NopCloser(bytes.NewReader([]byte("Hello, World"))) diff --git a/cmd/web-handlers.go b/cmd/web-handlers.go index 3a8887de7..976c171be 100644 --- a/cmd/web-handlers.go +++ b/cmd/web-handlers.go @@ -416,7 +416,8 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) { writeWebErrorResponse(w, errors.New("Server not initialized")) return } - if _, err := objectAPI.PutObject(bucket, object, -1, r.Body, metadata); err != nil { + sha256sum := "" + if _, err := objectAPI.PutObject(bucket, object, -1, r.Body, metadata, sha256sum); err != nil { writeWebErrorResponse(w, err) return } diff --git a/cmd/web-handlers_test.go b/cmd/web-handlers_test.go index 168573d0e..78d2fb7bf 100644 --- a/cmd/web-handlers_test.go +++ b/cmd/web-handlers_test.go @@ -351,7 +351,7 @@ func testListObjectsWebHandler(obj ObjectLayer, instanceType string, t TestErrHa data := bytes.Repeat([]byte("a"), objectSize) - _, err = obj.PutObject(bucketName, objectName, int64(len(data)), bytes.NewReader(data), map[string]string{"md5Sum": "c9a34cfc85d982698c6ac89f76071abd"}) + _, err = obj.PutObject(bucketName, objectName, int64(len(data)), bytes.NewReader(data), map[string]string{"md5Sum": "c9a34cfc85d982698c6ac89f76071abd"}, "") if err != nil { t.Fatalf("Was not able to upload an object, %v", err) @@ -422,7 +422,7 @@ func testRemoveObjectWebHandler(obj ObjectLayer, instanceType string, t TestErrH data := bytes.Repeat([]byte("a"), objectSize) - _, err = obj.PutObject(bucketName, objectName, int64(len(data)), bytes.NewReader(data), map[string]string{"md5Sum": "c9a34cfc85d982698c6ac89f76071abd"}) + _, err = obj.PutObject(bucketName, objectName, int64(len(data)), bytes.NewReader(data), map[string]string{"md5Sum": "c9a34cfc85d982698c6ac89f76071abd"}, "") if err != nil { t.Fatalf("Was not able to upload an object, %v", err) @@ -696,7 +696,7 @@ func testDownloadWebHandler(obj ObjectLayer, instanceType string, t TestErrHandl } content := []byte("temporary file's content") - _, err = obj.PutObject(bucketName, objectName, int64(len(content)), bytes.NewReader(content), map[string]string{"md5Sum": "01ce59706106fe5e02e7f55fffda7f34"}) + _, err = obj.PutObject(bucketName, objectName, int64(len(content)), bytes.NewReader(content), map[string]string{"md5Sum": "01ce59706106fe5e02e7f55fffda7f34"}, "") if err != nil { t.Fatalf("Was not able to upload an object, %v", err) } @@ -755,7 +755,7 @@ func testWebPresignedGetHandler(obj ObjectLayer, instanceType string, t TestErrH } data := bytes.Repeat([]byte("a"), objectSize) - _, err = obj.PutObject(bucketName, objectName, int64(len(data)), bytes.NewReader(data), map[string]string{"md5Sum": "c9a34cfc85d982698c6ac89f76071abd"}) + _, err = obj.PutObject(bucketName, objectName, int64(len(data)), bytes.NewReader(data), map[string]string{"md5Sum": "c9a34cfc85d982698c6ac89f76071abd"}, "") if err != nil { t.Fatalf("Was not able to upload an object, %v", err) } diff --git a/cmd/xl-v1-multipart.go b/cmd/xl-v1-multipart.go index 33413c060..fd74bb9c8 100644 --- a/cmd/xl-v1-multipart.go +++ b/cmd/xl-v1-multipart.go @@ -18,8 +18,10 @@ package cmd import ( "crypto/md5" + "crypto/sha256" "encoding/hex" "fmt" + "hash" "io" "io/ioutil" "path" @@ -333,7 +335,7 @@ func (xl xlObjects) NewMultipartUpload(bucket, object string, meta map[string]st // of the multipart transaction. // // Implements S3 compatible Upload Part API. -func (xl xlObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, error) { +func (xl xlObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (string, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { return "", traceError(BucketNameInvalid{Bucket: bucket}) @@ -384,10 +386,21 @@ func (xl xlObjects) PutObjectPart(bucket, object, uploadID string, partID int, s tmpSuffix := getUUID() tmpPartPath := path.Join(tmpMetaPrefix, tmpSuffix) + lreader := data + // Initialize md5 writer. md5Writer := md5.New() - lreader := data + writers := []io.Writer{md5Writer} + + var sha256Writer hash.Hash + if sha256sum != "" { + sha256Writer = sha256.New() + writers = append(writers, sha256Writer) + } + + mw := io.MultiWriter(writers...) + // Limit the reader to its provided size > 0. if size > 0 { // This is done so that we can avoid erroneous clients sending @@ -396,7 +409,7 @@ func (xl xlObjects) PutObjectPart(bucket, object, uploadID string, partID int, s } // else we read till EOF. // Construct a tee reader for md5sum. - teeReader := io.TeeReader(lreader, md5Writer) + teeReader := io.TeeReader(lreader, mw) // Erasure code data and write across all disks. sizeWritten, checkSums, err := erasureCreateFile(onlineDisks, minioMetaBucket, tmpPartPath, teeReader, xlMeta.Erasure.BlockSize, xl.dataBlocks, xl.parityBlocks, bitRotAlgo, xl.writeQuorum) @@ -436,7 +449,18 @@ func (xl xlObjects) PutObjectPart(bucket, object, uploadID string, partID int, s } } + if sha256sum != "" { + newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil)) + if newSHA256sum != sha256sum { + // SHA256 mismatch, delete the temporary object. + xl.deleteObject(minioMetaBucket, tmpPartPath) + return "", traceError(SHA256Mismatch{}) + } + } + // get a random ID for lock instrumentation. + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. opsID = getOpsID() nsMutex.Lock(minioMetaBucket, uploadIDPath, opsID) diff --git a/cmd/xl-v1-object.go b/cmd/xl-v1-object.go index b3ad00981..5a9f853c2 100644 --- a/cmd/xl-v1-object.go +++ b/cmd/xl-v1-object.go @@ -18,7 +18,9 @@ package cmd import ( "crypto/md5" + "crypto/sha256" "encoding/hex" + "hash" "io" "path" "strings" @@ -490,7 +492,7 @@ func renameObject(disks []StorageAPI, srcBucket, srcObject, dstBucket, dstObject // until EOF, erasure codes the data across all disk and additionally // writes `xl.json` which carries the necessary metadata for future // object operations. -func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) { +func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { return ObjectInfo{}, traceError(BucketNameInvalid{Bucket: bucket}) @@ -515,10 +517,17 @@ func (xl xlObjects) PutObject(bucket string, object string, size int64, data io. minioMetaTmpBucket := path.Join(minioMetaBucket, tmpMetaPrefix) tempObj := uniqueID - var mw io.Writer // Initialize md5 writer. md5Writer := md5.New() + writers := []io.Writer{md5Writer} + + var sha256Writer hash.Hash + if sha256sum != "" { + sha256Writer = sha256.New() + writers = append(writers, sha256Writer) + } + // Proceed to set the cache. var newBuffer io.WriteCloser @@ -531,17 +540,17 @@ func (xl xlObjects) PutObject(bucket string, object string, size int64, data io. newBuffer, err = xl.objCache.Create(path.Join(bucket, object), size) if err == nil { // Create a multi writer to write to both memory and client response. - mw = io.MultiWriter(newBuffer, md5Writer) + writers = append(writers, newBuffer) } // Ignore error if cache is full, proceed to write the object. if err != nil && err != objcache.ErrCacheFull { // For any other error return here. return ObjectInfo{}, toObjectErr(traceError(err), bucket, object) } - } else { - mw = md5Writer } + mw := io.MultiWriter(writers...) + // Limit the reader to its provided size if specified. var limitDataReader io.Reader if size > 0 { @@ -621,7 +630,18 @@ func (xl xlObjects) PutObject(bucket string, object string, size int64, data io. } } + if sha256sum != "" { + newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil)) + if newSHA256sum != sha256sum { + // SHA256 mismatch, delete the temporary object. + xl.deleteObject(minioMetaBucket, tempObj) + return ObjectInfo{}, traceError(SHA256Mismatch{}) + } + } + // get a random ID for lock instrumentation. + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. opsID := getOpsID() // Lock the object. diff --git a/cmd/xl-v1-object_test.go b/cmd/xl-v1-object_test.go index aa978539a..6c8957740 100644 --- a/cmd/xl-v1-object_test.go +++ b/cmd/xl-v1-object_test.go @@ -50,12 +50,12 @@ func TestRepeatPutObjectPart(t *testing.T) { md5Writer := md5.New() md5Writer.Write(fiveMBBytes) md5Hex := hex.EncodeToString(md5Writer.Sum(nil)) - _, err = objLayer.PutObjectPart("bucket1", "mpartObj1", uploadID, 1, 5*1024*1024, bytes.NewReader(fiveMBBytes), md5Hex) + _, err = objLayer.PutObjectPart("bucket1", "mpartObj1", uploadID, 1, 5*1024*1024, bytes.NewReader(fiveMBBytes), md5Hex, "") if err != nil { t.Fatal(err) } // PutObjectPart should succeed even if part already exists. ref: https://github.com/minio/minio/issues/1930 - _, err = objLayer.PutObjectPart("bucket1", "mpartObj1", uploadID, 1, 5*1024*1024, bytes.NewReader(fiveMBBytes), md5Hex) + _, err = objLayer.PutObjectPart("bucket1", "mpartObj1", uploadID, 1, 5*1024*1024, bytes.NewReader(fiveMBBytes), md5Hex, "") if err != nil { t.Fatal(err) } @@ -89,7 +89,7 @@ func TestXLDeleteObjectBasic(t *testing.T) { } // Create object "obj" under bucket "bucket" for Test 7 to pass - _, err = xl.PutObject("bucket", "obj", int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil) + _, err = xl.PutObject("bucket", "obj", int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, "") if err != nil { t.Fatalf("XL Object upload failed: %s", err) } @@ -125,7 +125,7 @@ func TestXLDeleteObjectDiskNotFound(t *testing.T) { bucket := "bucket" object := "object" // Create object "obj" under bucket "bucket". - _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil) + _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, "") if err != nil { t.Fatal(err) } @@ -140,7 +140,7 @@ func TestXLDeleteObjectDiskNotFound(t *testing.T) { } // Create "obj" under "bucket". - _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil) + _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, "") if err != nil { t.Fatal(err) } @@ -175,7 +175,7 @@ func TestGetObjectNoQuorum(t *testing.T) { bucket := "bucket" object := "object" // Create "object" under "bucket". - _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil) + _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, "") if err != nil { t.Fatal(err) } @@ -227,7 +227,7 @@ func TestPutObjectNoQuorum(t *testing.T) { bucket := "bucket" object := "object" // Create "object" under "bucket". - _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil) + _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, "") if err != nil { t.Fatal(err) } @@ -250,7 +250,7 @@ func TestPutObjectNoQuorum(t *testing.T) { } } // Upload new content to same object "object" - _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil) + _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, "") err = errorCause(err) if err != toObjectErr(errXLWriteQuorum, bucket, object) { t.Errorf("Expected putObject to fail with %v, but failed with %v", toObjectErr(errXLWriteQuorum, bucket, object), err)