From e781959d5bbdf6a14346ed0a0bcfe1631df00e26 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 10 Mar 2016 14:22:28 -0800 Subject: [PATCH] vendor: Add minio-go vendor updates. --- routers.go | 2 +- .../github.com/minio/minio-go/CONTRIBUTING.md | 4 +- .../minio/minio-go/api-error-response.go | 77 +- vendor/github.com/minio/minio-go/api-get.go | 50 +- vendor/github.com/minio/minio-go/api-list.go | 37 +- .../minio/minio-go/api-presigned.go | 2 +- .../minio/minio-go/api-put-bucket.go | 13 +- .../minio/minio-go/api-put-object-file.go | 4 +- .../minio-go/api-put-object-multipart.go | 46 +- .../minio/minio-go/api-put-object-progress.go | 2 +- .../minio/minio-go/api-put-object-readat.go | 7 +- .../minio/minio-go/api-put-object.go | 39 +- .../github.com/minio/minio-go/api-remove.go | 44 +- vendor/github.com/minio/minio-go/api-stat.go | 50 +- vendor/github.com/minio/minio-go/api.go | 172 ++- .../minio/minio-go/api_functional_v2_test.go | 1073 ---------------- .../minio/minio-go/api_functional_v4_test.go | 1101 ----------------- .../minio/minio-go/api_unit_test.go | 599 --------- .../github.com/minio/minio-go/bucket-cache.go | 19 +- .../github.com/minio/minio-go/hook-reader.go | 20 +- vendor/github.com/minio/minio-go/retry.go | 95 ++ vendor/vendor.json | 4 +- 22 files changed, 392 insertions(+), 3068 deletions(-) delete mode 100644 vendor/github.com/minio/minio-go/api_functional_v2_test.go delete mode 100644 vendor/github.com/minio/minio-go/api_functional_v4_test.go delete mode 100644 vendor/github.com/minio/minio-go/api_unit_test.go create mode 100644 vendor/github.com/minio/minio-go/retry.go diff --git a/routers.go b/routers.go index 78023bb42..a36b5995c 100644 --- a/routers.go +++ b/routers.go @@ -52,7 +52,7 @@ type webAPI struct { // Once true log all incoming request. AccessLog bool // Minio client instance. - Client minio.CloudStorageClient + Client *minio.Client // private params. apiAddress string // api destination address. diff --git a/vendor/github.com/minio/minio-go/CONTRIBUTING.md b/vendor/github.com/minio/minio-go/CONTRIBUTING.md index 3a938e856..8b1ee86c6 100644 --- a/vendor/github.com/minio/minio-go/CONTRIBUTING.md +++ b/vendor/github.com/minio/minio-go/CONTRIBUTING.md @@ -14,7 +14,9 @@ - Have test cases for the new code. If you have questions about how to do it, please ask in your pull request. - Run `go fmt` - Squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request. - - Make sure `go test -short -race ./...` and `go build` completes. + - Make sure `go test -race ./...` and `go build` completes. + NOTE: go test runs functional tests and requires you to have a AWS S3 account. Set them as environment variables + ``ACCESS_KEY`` and ``SECRET_KEY``. To run shorter version of the tests please use ``go test -short -race ./...`` * Read [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project - `minio-go` project is strictly conformant with Golang style diff --git a/vendor/github.com/minio/minio-go/api-error-response.go b/vendor/github.com/minio/minio-go/api-error-response.go index 647165112..e6789aff5 100644 --- a/vendor/github.com/minio/minio-go/api-error-response.go +++ b/vendor/github.com/minio/minio-go/api-error-response.go @@ -47,7 +47,7 @@ type ErrorResponse struct { // Region where the bucket is located. This header is returned // only in HEAD bucket and ListObjects response. - AmzBucketRegion string + Region string } // ToErrorResponse - Returns parsed ErrorResponse struct from body and @@ -98,65 +98,54 @@ func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string) case http.StatusNotFound: if objectName == "" { errResp = ErrorResponse{ - Code: "NoSuchBucket", - Message: "The specified bucket does not exist.", - BucketName: bucketName, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"), + Code: "NoSuchBucket", + Message: "The specified bucket does not exist.", + BucketName: bucketName, + RequestID: resp.Header.Get("x-amz-request-id"), + HostID: resp.Header.Get("x-amz-id-2"), + Region: resp.Header.Get("x-amz-bucket-region"), } } else { errResp = ErrorResponse{ - Code: "NoSuchKey", - Message: "The specified key does not exist.", - BucketName: bucketName, - Key: objectName, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"), + Code: "NoSuchKey", + Message: "The specified key does not exist.", + BucketName: bucketName, + Key: objectName, + RequestID: resp.Header.Get("x-amz-request-id"), + HostID: resp.Header.Get("x-amz-id-2"), + Region: resp.Header.Get("x-amz-bucket-region"), } } case http.StatusForbidden: errResp = ErrorResponse{ - Code: "AccessDenied", - Message: "Access Denied.", - BucketName: bucketName, - Key: objectName, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"), + Code: "AccessDenied", + Message: "Access Denied.", + BucketName: bucketName, + Key: objectName, + RequestID: resp.Header.Get("x-amz-request-id"), + HostID: resp.Header.Get("x-amz-id-2"), + Region: resp.Header.Get("x-amz-bucket-region"), } case http.StatusConflict: errResp = ErrorResponse{ - Code: "Conflict", - Message: "Bucket not empty.", - BucketName: bucketName, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"), + Code: "Conflict", + Message: "Bucket not empty.", + BucketName: bucketName, + RequestID: resp.Header.Get("x-amz-request-id"), + HostID: resp.Header.Get("x-amz-id-2"), + Region: resp.Header.Get("x-amz-bucket-region"), } default: errResp = ErrorResponse{ - Code: resp.Status, - Message: resp.Status, - BucketName: bucketName, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"), + Code: resp.Status, + Message: resp.Status, + BucketName: bucketName, + RequestID: resp.Header.Get("x-amz-request-id"), + HostID: resp.Header.Get("x-amz-id-2"), + Region: resp.Header.Get("x-amz-bucket-region"), } } } - - // AccessDenied without a signature mismatch code, usually means - // that the bucket policy has certain restrictions where some API - // operations are not allowed. Handle this case so that top level - // callers can interpret this easily and fall back if needed to a - // lower functionality call. Read each individual API specific - // code for such fallbacks. - if errResp.Code == "AccessDenied" && errResp.Message == "Access Denied" { - errResp.Code = "NotImplemented" - errResp.Message = "Operation is not allowed according to your bucket policy." - } return errResp } diff --git a/vendor/github.com/minio/minio-go/api-get.go b/vendor/github.com/minio/minio-go/api-get.go index 56d44c9f5..3362a2283 100644 --- a/vendor/github.com/minio/minio-go/api-get.go +++ b/vendor/github.com/minio/minio-go/api-get.go @@ -1,5 +1,5 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,7 +20,6 @@ import ( "errors" "fmt" "io" - "math" "net/http" "net/url" "strings" @@ -48,17 +47,11 @@ func (c Client) GetBucketACL(bucketName string) (BucketACL, error) { urlValues := make(url.Values) urlValues.Set("acl", "") - // Instantiate a new request. - req, err := c.newRequest("GET", requestMetadata{ + // Execute GET acl on bucketName. + resp, err := c.executeMethod("GET", requestMetadata{ bucketName: bucketName, queryValues: urlValues, }) - if err != nil { - return "", err - } - - // Initiate the request. - resp, err := c.do(req) defer closeResponse(resp) if err != nil { return "", err @@ -83,12 +76,12 @@ func (c Client) GetBucketACL(bucketName string) (BucketACL, error) { if !isGoogleEndpoint(c.endpointURL) { if policy.AccessControlList.Grant == nil { errorResponse := ErrorResponse{ - Code: "InternalError", - Message: "Access control Grant list is empty. " + reportIssue, - BucketName: bucketName, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"), + Code: "InternalError", + Message: "Access control Grant list is empty. " + reportIssue, + BucketName: bucketName, + RequestID: resp.Header.Get("x-amz-request-id"), + HostID: resp.Header.Get("x-amz-id-2"), + Region: resp.Header.Get("x-amz-bucket-region"), } return "", errorResponse } @@ -177,7 +170,11 @@ func (c Client) GetObject(bucketName, objectName string) (*Object, error) { // Get shortest length. // NOTE: Last remaining bytes are usually smaller than // req.Buffer size. Use that as the final length. - length := math.Min(float64(len(req.Buffer)), float64(objectInfo.Size-req.Offset)) + // Don't use Math.min() here to avoid converting int64 to float64 + length := int64(len(req.Buffer)) + if objectInfo.Size-req.Offset < length { + length = objectInfo.Size - req.Offset + } httpReader, _, err := c.getObject(bucketName, objectName, req.Offset, int64(length)) if err != nil { resCh <- readResponse{ @@ -484,8 +481,8 @@ func (c Client) getObject(bucketName, objectName string, offset, length int64) ( customHeader.Set("Range", fmt.Sprintf("bytes=%d", length)) } - // Instantiate a new request. - req, err := c.newRequest("GET", requestMetadata{ + // Execute GET on objectName. + resp, err := c.executeMethod("GET", requestMetadata{ bucketName: bucketName, objectName: objectName, customHeader: customHeader, @@ -493,11 +490,6 @@ func (c Client) getObject(bucketName, objectName string, offset, length int64) ( if err != nil { return nil, ObjectInfo{}, err } - // Execute the request. - resp, err := c.do(req) - if err != nil { - return nil, ObjectInfo{}, err - } if resp != nil { if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { return nil, ObjectInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) @@ -513,11 +505,11 @@ func (c Client) getObject(bucketName, objectName string, offset, length int64) ( if err != nil { msg := "Last-Modified time format not recognized. " + reportIssue return nil, ObjectInfo{}, ErrorResponse{ - Code: "InternalError", - Message: msg, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"), + Code: "InternalError", + Message: msg, + RequestID: resp.Header.Get("x-amz-request-id"), + HostID: resp.Header.Get("x-amz-id-2"), + Region: resp.Header.Get("x-amz-bucket-region"), } } // Get content-type. diff --git a/vendor/github.com/minio/minio-go/api-list.go b/vendor/github.com/minio/minio-go/api-list.go index c7cbac7d1..b17a51acc 100644 --- a/vendor/github.com/minio/minio-go/api-list.go +++ b/vendor/github.com/minio/minio-go/api-list.go @@ -1,5 +1,5 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,13 +34,8 @@ import ( // } // func (c Client) ListBuckets() ([]BucketInfo, error) { - // Instantiate a new request. - req, err := c.newRequest("GET", requestMetadata{}) - if err != nil { - return nil, err - } - // Initiate the request. - resp, err := c.do(req) + // Execute GET on service. + resp, err := c.executeMethod("GET", requestMetadata{}) defer closeResponse(resp) if err != nil { return nil, err @@ -206,16 +201,11 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit // Set max keys. urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) - // Initialize a new request. - req, err := c.newRequest("GET", requestMetadata{ + // Execute GET on bucket to list objects. + resp, err := c.executeMethod("GET", requestMetadata{ bucketName: bucketName, queryValues: urlValues, }) - if err != nil { - return listBucketResult{}, err - } - // Execute list buckets. - resp, err := c.do(req) defer closeResponse(resp) if err != nil { return listBucketResult{}, err @@ -388,16 +378,11 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, // Set max-uploads. urlValues.Set("max-uploads", fmt.Sprintf("%d", maxUploads)) - // Instantiate a new request. - req, err := c.newRequest("GET", requestMetadata{ + // Execute GET on bucketName to list multipart uploads. + resp, err := c.executeMethod("GET", requestMetadata{ bucketName: bucketName, queryValues: urlValues, }) - if err != nil { - return listMultipartUploadsResult{}, err - } - // Execute list multipart uploads request. - resp, err := c.do(req) defer closeResponse(resp) if err != nil { return listMultipartUploadsResult{}, err @@ -510,16 +495,12 @@ func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, pa // Set max parts. urlValues.Set("max-parts", fmt.Sprintf("%d", maxParts)) - req, err := c.newRequest("GET", requestMetadata{ + // Execute GET on objectName to get list of parts. + resp, err := c.executeMethod("GET", requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, }) - if err != nil { - return listObjectPartsResult{}, err - } - // Exectue list object parts. - resp, err := c.do(req) defer closeResponse(resp) if err != nil { return listObjectPartsResult{}, err diff --git a/vendor/github.com/minio/minio-go/api-presigned.go b/vendor/github.com/minio/minio-go/api-presigned.go index 1a2a34752..0d3285253 100644 --- a/vendor/github.com/minio/minio-go/api-presigned.go +++ b/vendor/github.com/minio/minio-go/api-presigned.go @@ -1,5 +1,5 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/vendor/github.com/minio/minio-go/api-put-bucket.go b/vendor/github.com/minio/minio-go/api-put-bucket.go index 986099e34..115112461 100644 --- a/vendor/github.com/minio/minio-go/api-put-bucket.go +++ b/vendor/github.com/minio/minio-go/api-put-bucket.go @@ -1,5 +1,5 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -74,7 +74,7 @@ func (c Client) MakeBucket(bucketName string, acl BucketACL, location string) er } } - // Save the location into cache on a successfull makeBucket response. + // Save the location into cache on a successful makeBucket response. c.bucketLocCache.Set(bucketName, location) // Return. @@ -180,19 +180,16 @@ func (c Client) SetBucketACL(bucketName string, acl BucketACL) error { customHeader.Set("x-amz-acl", "private") } - // Instantiate a new request. - req, err := c.newRequest("PUT", requestMetadata{ + // Execute PUT bucket. + resp, err := c.executeMethod("PUT", requestMetadata{ bucketName: bucketName, queryValues: urlValues, customHeader: customHeader, }) + defer closeResponse(resp) if err != nil { return err } - - // Initiate the request. - resp, err := c.do(req) - defer closeResponse(resp) if err != nil { return err } diff --git a/vendor/github.com/minio/minio-go/api-put-object-file.go b/vendor/github.com/minio/minio-go/api-put-object-file.go index d212dfb87..d4772c53a 100644 --- a/vendor/github.com/minio/minio-go/api-put-object-file.go +++ b/vendor/github.com/minio/minio-go/api-put-object-file.go @@ -1,5 +1,5 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -187,7 +187,7 @@ func (c Client) putObjectMultipartFromFile(bucketName, objectName string, fileRe }, partsInfo) { // Proceed to upload the part. var objPart objectPart - objPart, err = c.uploadPart(bucketName, objectName, uploadID, ioutil.NopCloser(reader), partNumber, + objPart, err = c.uploadPart(bucketName, objectName, uploadID, reader, partNumber, md5Sum, sha256Sum, prtSize) if err != nil { return totalUploadedSize, err diff --git a/vendor/github.com/minio/minio-go/api-put-object-multipart.go b/vendor/github.com/minio/minio-go/api-put-object-multipart.go index ee0019165..8b3db1039 100644 --- a/vendor/github.com/minio/minio-go/api-put-object-multipart.go +++ b/vendor/github.com/minio/minio-go/api-put-object-multipart.go @@ -1,5 +1,5 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -124,7 +124,7 @@ func (c Client) putObjectMultipartStream(bucketName, objectName string, reader i var reader io.Reader // Update progress reader appropriately to the latest offset // as we read from the source. - reader = newHook(tmpBuffer, progress) + reader = newHook(bytes.NewReader(tmpBuffer.Bytes()), progress) // Verify if part should be uploaded. if shouldUploadPart(objectPart{ @@ -134,8 +134,7 @@ func (c Client) putObjectMultipartStream(bucketName, objectName string, reader i }, partsInfo) { // Proceed to upload the part. var objPart objectPart - objPart, err = c.uploadPart(bucketName, objectName, uploadID, ioutil.NopCloser(reader), partNumber, - md5Sum, sha256Sum, prtSize) + objPart, err = c.uploadPart(bucketName, objectName, uploadID, reader, partNumber, md5Sum, sha256Sum, prtSize) if err != nil { // Reset the temporary buffer upon any error. tmpBuffer.Reset() @@ -230,14 +229,8 @@ func (c Client) initiateMultipartUpload(bucketName, objectName, contentType stri customHeader: customHeader, } - // Instantiate the request. - req, err := c.newRequest("POST", reqMetadata) - if err != nil { - return initiateMultipartUploadResult{}, err - } - - // Execute the request. - resp, err := c.do(req) + // Execute POST on an objectName to initiate multipart upload. + resp, err := c.executeMethod("POST", reqMetadata) defer closeResponse(resp) if err != nil { return initiateMultipartUploadResult{}, err @@ -257,7 +250,7 @@ func (c Client) initiateMultipartUpload(bucketName, objectName, contentType stri } // uploadPart - Uploads a part in a multipart upload. -func (c Client) uploadPart(bucketName, objectName, uploadID string, reader io.ReadCloser, partNumber int, md5Sum, sha256Sum []byte, size int64) (objectPart, error) { +func (c Client) uploadPart(bucketName, objectName, uploadID string, reader io.Reader, partNumber int, md5Sum, sha256Sum []byte, size int64) (objectPart, error) { // Input validation. if err := isValidBucketName(bucketName); err != nil { return objectPart{}, err @@ -295,13 +288,8 @@ func (c Client) uploadPart(bucketName, objectName, uploadID string, reader io.Re contentSHA256Bytes: sha256Sum, } - // Instantiate a request. - req, err := c.newRequest("PUT", reqMetadata) - if err != nil { - return objectPart{}, err - } - // Execute the request. - resp, err := c.do(req) + // Execute PUT on each part. + resp, err := c.executeMethod("PUT", reqMetadata) defer closeResponse(resp) if err != nil { return objectPart{}, err @@ -342,24 +330,18 @@ func (c Client) completeMultipartUpload(bucketName, objectName, uploadID string, } // Instantiate all the complete multipart buffer. - completeMultipartUploadBuffer := bytes.NewBuffer(completeMultipartUploadBytes) + completeMultipartUploadBuffer := bytes.NewReader(completeMultipartUploadBytes) reqMetadata := requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, - contentBody: ioutil.NopCloser(completeMultipartUploadBuffer), - contentLength: int64(completeMultipartUploadBuffer.Len()), - contentSHA256Bytes: sum256(completeMultipartUploadBuffer.Bytes()), + contentBody: completeMultipartUploadBuffer, + contentLength: int64(len(completeMultipartUploadBytes)), + contentSHA256Bytes: sum256(completeMultipartUploadBytes), } - // Instantiate the request. - req, err := c.newRequest("POST", reqMetadata) - if err != nil { - return completeMultipartUploadResult{}, err - } - - // Execute the request. - resp, err := c.do(req) + // Execute POST to complete multipart upload for an objectName. + resp, err := c.executeMethod("POST", reqMetadata) defer closeResponse(resp) if err != nil { return completeMultipartUploadResult{}, err diff --git a/vendor/github.com/minio/minio-go/api-put-object-progress.go b/vendor/github.com/minio/minio-go/api-put-object-progress.go index ae4425d49..ebbc380c3 100644 --- a/vendor/github.com/minio/minio-go/api-put-object-progress.go +++ b/vendor/github.com/minio/minio-go/api-put-object-progress.go @@ -91,7 +91,7 @@ func (c Client) PutObjectWithProgress(bucketName, objectName string, reader io.R errResp := ToErrorResponse(err) // Verify if multipart functionality is not available, if not // fall back to single PutObject operation. - if errResp.Code == "NotImplemented" { + if errResp.Code == "AccessDenied" && errResp.Message == "Access Denied." { // Verify if size of reader is greater than '5GiB'. if size > maxSinglePutObjectSize { return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) diff --git a/vendor/github.com/minio/minio-go/api-put-object-readat.go b/vendor/github.com/minio/minio-go/api-put-object-readat.go index ddb1ab3dc..b02499f7d 100644 --- a/vendor/github.com/minio/minio-go/api-put-object-readat.go +++ b/vendor/github.com/minio/minio-go/api-put-object-readat.go @@ -1,5 +1,5 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -155,12 +155,11 @@ func (c Client) putObjectMultipartFromReadAt(bucketName, objectName string, read var reader io.Reader // Update progress reader appropriately to the latest offset // as we read from the source. - reader = newHook(tmpBuffer, progress) + reader = newHook(bytes.NewReader(tmpBuffer.Bytes()), progress) // Proceed to upload the part. var objPart objectPart - objPart, err = c.uploadPart(bucketName, objectName, uploadID, ioutil.NopCloser(reader), - partNumber, md5Sum, sha256Sum, prtSize) + objPart, err = c.uploadPart(bucketName, objectName, uploadID, reader, partNumber, md5Sum, sha256Sum, prtSize) if err != nil { // Reset the buffer upon any error. tmpBuffer.Reset() diff --git a/vendor/github.com/minio/minio-go/api-put-object.go b/vendor/github.com/minio/minio-go/api-put-object.go index a09e658f4..1586580ed 100644 --- a/vendor/github.com/minio/minio-go/api-put-object.go +++ b/vendor/github.com/minio/minio-go/api-put-object.go @@ -1,5 +1,5 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -146,11 +146,11 @@ func (c Client) putObjectNoChecksum(bucketName, objectName string, reader io.Rea // Update progress reader appropriately to the latest offset as we // read from the source. - reader = newHook(reader, progress) + readSeeker := newHook(reader, progress) // This function does not calculate sha256 and md5sum for payload. // Execute put object. - st, err := c.putObjectDo(bucketName, objectName, ioutil.NopCloser(reader), nil, nil, size, contentType) + st, err := c.putObjectDo(bucketName, objectName, readSeeker, nil, nil, size, contentType) if err != nil { return 0, err } @@ -178,12 +178,12 @@ func (c Client) putObjectSingle(bucketName, objectName string, reader io.Reader, size = maxSinglePutObjectSize } var md5Sum, sha256Sum []byte - var readCloser io.ReadCloser if size <= minPartSize { // Initialize a new temporary buffer. tmpBuffer := new(bytes.Buffer) md5Sum, sha256Sum, size, err = c.hashCopyN(tmpBuffer, reader, size) - readCloser = ioutil.NopCloser(tmpBuffer) + reader = bytes.NewReader(tmpBuffer.Bytes()) + tmpBuffer.Reset() } else { // Initialize a new temporary file. var tmpFile *tempFile @@ -191,12 +191,13 @@ func (c Client) putObjectSingle(bucketName, objectName string, reader io.Reader, if err != nil { return 0, err } + defer tmpFile.Close() md5Sum, sha256Sum, size, err = c.hashCopyN(tmpFile, reader, size) // Seek back to beginning of the temporary file. if _, err = tmpFile.Seek(0, 0); err != nil { return 0, err } - readCloser = tmpFile + reader = tmpFile } // Return error if its not io.EOF. if err != nil { @@ -204,26 +205,26 @@ func (c Client) putObjectSingle(bucketName, objectName string, reader io.Reader, return 0, err } } - // Progress the reader to the size. - if progress != nil { - if _, err = io.CopyN(ioutil.Discard, progress, size); err != nil { - return size, err - } - } // Execute put object. - st, err := c.putObjectDo(bucketName, objectName, readCloser, md5Sum, sha256Sum, size, contentType) + st, err := c.putObjectDo(bucketName, objectName, reader, md5Sum, sha256Sum, size, contentType) if err != nil { return 0, err } if st.Size != size { return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName) } + // Progress the reader to the size if putObjectDo is successful. + if progress != nil { + if _, err = io.CopyN(ioutil.Discard, progress, size); err != nil { + return size, err + } + } return size, nil } // putObjectDo - executes the put object http operation. // NOTE: You must have WRITE permissions on a bucket to add an object to it. -func (c Client) putObjectDo(bucketName, objectName string, reader io.ReadCloser, md5Sum []byte, sha256Sum []byte, size int64, contentType string) (ObjectInfo, error) { +func (c Client) putObjectDo(bucketName, objectName string, reader io.Reader, md5Sum []byte, sha256Sum []byte, size int64, contentType string) (ObjectInfo, error) { // Input validation. if err := isValidBucketName(bucketName); err != nil { return ObjectInfo{}, err @@ -258,13 +259,9 @@ func (c Client) putObjectDo(bucketName, objectName string, reader io.ReadCloser, contentMD5Bytes: md5Sum, contentSHA256Bytes: sha256Sum, } - // Initiate new request. - req, err := c.newRequest("PUT", reqMetadata) - if err != nil { - return ObjectInfo{}, err - } - // Execute the request. - resp, err := c.do(req) + + // Execute PUT an objectName. + resp, err := c.executeMethod("PUT", reqMetadata) defer closeResponse(resp) if err != nil { return ObjectInfo{}, err diff --git a/vendor/github.com/minio/minio-go/api-remove.go b/vendor/github.com/minio/minio-go/api-remove.go index 8f59c15e6..46d47abf1 100644 --- a/vendor/github.com/minio/minio-go/api-remove.go +++ b/vendor/github.com/minio/minio-go/api-remove.go @@ -1,5 +1,5 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,15 +30,10 @@ func (c Client) RemoveBucket(bucketName string) error { if err := isValidBucketName(bucketName); err != nil { return err } - // Instantiate a new request. - req, err := c.newRequest("DELETE", requestMetadata{ + // Execute DELETE on bucket. + resp, err := c.executeMethod("DELETE", requestMetadata{ bucketName: bucketName, }) - if err != nil { - return err - } - // Initiate the request. - resp, err := c.do(req) defer closeResponse(resp) if err != nil { return err @@ -64,16 +59,11 @@ func (c Client) RemoveObject(bucketName, objectName string) error { if err := isValidObjectName(objectName); err != nil { return err } - // Instantiate the request. - req, err := c.newRequest("DELETE", requestMetadata{ + // Execute DELETE on objectName. + resp, err := c.executeMethod("DELETE", requestMetadata{ bucketName: bucketName, objectName: objectName, }) - if err != nil { - return err - } - // Initiate the request. - resp, err := c.do(req) defer closeResponse(resp) if err != nil { return err @@ -124,18 +114,12 @@ func (c Client) abortMultipartUpload(bucketName, objectName, uploadID string) er urlValues := make(url.Values) urlValues.Set("uploadId", uploadID) - // Instantiate a new DELETE request. - req, err := c.newRequest("DELETE", requestMetadata{ + // Execute DELETE on multipart upload. + resp, err := c.executeMethod("DELETE", requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, }) - if err != nil { - return err - } - - // Initiate the request. - resp, err := c.do(req) defer closeResponse(resp) if err != nil { return err @@ -149,13 +133,13 @@ func (c Client) abortMultipartUpload(bucketName, objectName, uploadID string) er // This is needed specifically for abort and it cannot // be converged into default case. errorResponse = ErrorResponse{ - Code: "NoSuchUpload", - Message: "The specified multipart upload does not exist.", - BucketName: bucketName, - Key: objectName, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"), + Code: "NoSuchUpload", + Message: "The specified multipart upload does not exist.", + BucketName: bucketName, + Key: objectName, + RequestID: resp.Header.Get("x-amz-request-id"), + HostID: resp.Header.Get("x-amz-id-2"), + Region: resp.Header.Get("x-amz-bucket-region"), } default: return httpRespToErrorResponse(resp, bucketName, objectName) diff --git a/vendor/github.com/minio/minio-go/api-stat.go b/vendor/github.com/minio/minio-go/api-stat.go index 20f66e8fc..b5db7fedc 100644 --- a/vendor/github.com/minio/minio-go/api-stat.go +++ b/vendor/github.com/minio/minio-go/api-stat.go @@ -1,5 +1,5 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,15 +29,11 @@ func (c Client) BucketExists(bucketName string) error { if err := isValidBucketName(bucketName); err != nil { return err } - // Instantiate a new request. - req, err := c.newRequest("HEAD", requestMetadata{ + + // Execute HEAD on bucketName. + resp, err := c.executeMethod("HEAD", requestMetadata{ bucketName: bucketName, }) - if err != nil { - return err - } - // Initiate the request. - resp, err := c.do(req) defer closeResponse(resp) if err != nil { return err @@ -59,16 +55,12 @@ func (c Client) StatObject(bucketName, objectName string) (ObjectInfo, error) { if err := isValidObjectName(objectName); err != nil { return ObjectInfo{}, err } - // Instantiate a new request. - req, err := c.newRequest("HEAD", requestMetadata{ + + // Execute HEAD on objectName. + resp, err := c.executeMethod("HEAD", requestMetadata{ bucketName: bucketName, objectName: objectName, }) - if err != nil { - return ObjectInfo{}, err - } - // Initiate the request. - resp, err := c.do(req) defer closeResponse(resp) if err != nil { return ObjectInfo{}, err @@ -87,26 +79,26 @@ func (c Client) StatObject(bucketName, objectName string) (ObjectInfo, error) { size, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) if err != nil { return ObjectInfo{}, ErrorResponse{ - Code: "InternalError", - Message: "Content-Length is invalid. " + reportIssue, - BucketName: bucketName, - Key: objectName, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"), + Code: "InternalError", + Message: "Content-Length is invalid. " + reportIssue, + BucketName: bucketName, + Key: objectName, + RequestID: resp.Header.Get("x-amz-request-id"), + HostID: resp.Header.Get("x-amz-id-2"), + Region: resp.Header.Get("x-amz-bucket-region"), } } // Parse Last-Modified has http time format. date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified")) if err != nil { return ObjectInfo{}, ErrorResponse{ - Code: "InternalError", - Message: "Last-Modified time format is invalid. " + reportIssue, - BucketName: bucketName, - Key: objectName, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"), + Code: "InternalError", + Message: "Last-Modified time format is invalid. " + reportIssue, + BucketName: bucketName, + Key: objectName, + RequestID: resp.Header.Get("x-amz-request-id"), + HostID: resp.Header.Get("x-amz-id-2"), + Region: resp.Header.Get("x-amz-bucket-region"), } } // Fetch content type if any present. diff --git a/vendor/github.com/minio/minio-go/api.go b/vendor/github.com/minio/minio-go/api.go index d52d641bb..abeeb3b05 100644 --- a/vendor/github.com/minio/minio-go/api.go +++ b/vendor/github.com/minio/minio-go/api.go @@ -1,5 +1,5 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ import ( "encoding/hex" "fmt" "io" + "io/ioutil" "net/http" "net/http/httputil" "net/url" @@ -64,7 +65,7 @@ type Client struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "0.2.5" + libraryVersion = "1.0.0" ) // User Agent should always following the below style. @@ -78,7 +79,7 @@ const ( // NewV2 - instantiate minio client with Amazon S3 signature version // '2' compatibility. -func NewV2(endpoint string, accessKeyID, secretAccessKey string, insecure bool) (CloudStorageClient, error) { +func NewV2(endpoint string, accessKeyID, secretAccessKey string, insecure bool) (*Client, error) { clnt, err := privateNew(endpoint, accessKeyID, secretAccessKey, insecure) if err != nil { return nil, err @@ -90,7 +91,7 @@ func NewV2(endpoint string, accessKeyID, secretAccessKey string, insecure bool) // NewV4 - instantiate minio client with Amazon S3 signature version // '4' compatibility. -func NewV4(endpoint string, accessKeyID, secretAccessKey string, insecure bool) (CloudStorageClient, error) { +func NewV4(endpoint string, accessKeyID, secretAccessKey string, insecure bool) (*Client, error) { clnt, err := privateNew(endpoint, accessKeyID, secretAccessKey, insecure) if err != nil { return nil, err @@ -102,7 +103,7 @@ func NewV4(endpoint string, accessKeyID, secretAccessKey string, insecure bool) // New - instantiate minio client Client, adds automatic verification // of signature. -func New(endpoint string, accessKeyID, secretAccessKey string, insecure bool) (CloudStorageClient, error) { +func New(endpoint string, accessKeyID, secretAccessKey string, insecure bool) (*Client, error) { clnt, err := privateNew(endpoint, accessKeyID, secretAccessKey, insecure) if err != nil { return nil, err @@ -112,7 +113,7 @@ func New(endpoint string, accessKeyID, secretAccessKey string, insecure bool) (C if isGoogleEndpoint(clnt.endpointURL) { clnt.signature = SignatureV2 } - // If Amazon S3 set to signature v2. + // If Amazon S3 set to signature v2.n if isAmazonEndpoint(clnt.endpointURL) { clnt.signature = SignatureV4 } @@ -138,7 +139,14 @@ func privateNew(endpoint, accessKeyID, secretAccessKey string, insecure bool) (* clnt.endpointURL = endpointURL // Instantiate http client and bucket location cache. - clnt.httpClient = &http.Client{} + clnt.httpClient = &http.Client{ + // Setting a sensible time out of 2minutes to wait for response + // headers. Request is pro-actively cancelled after 2minutes + // if no response was received from server. + Timeout: 2 * time.Minute, + Transport: http.DefaultTransport, + } + clnt.bucketLocCache = newBucketLocationCache() // Return. @@ -214,7 +222,7 @@ type requestMetadata struct { // Generated by our internal code. bucketLocation string - contentBody io.ReadCloser + contentBody io.Reader contentLength int64 contentSHA256Bytes []byte contentMD5Bytes []byte @@ -292,7 +300,7 @@ func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error { // to zero. Keep this workaround until the above bug is fixed. if resp.ContentLength == 0 { var buffer bytes.Buffer - if err := resp.Header.Write(&buffer); err != nil { + if err = resp.Header.Write(&buffer); err != nil { return err } respTrace = buffer.Bytes() @@ -322,17 +330,28 @@ func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error { // do - execute http request. func (c Client) do(req *http.Request) (*http.Response, error) { - // execute the request. + // do the request. resp, err := c.httpClient.Do(req) if err != nil { // Handle this specifically for now until future Golang // versions fix this issue properly. urlErr, ok := err.(*url.Error) if ok && strings.Contains(urlErr.Err.Error(), "EOF") { - return nil, fmt.Errorf("Connection closed by foreign host %s. Retry again.", urlErr.URL) + return nil, &url.Error{ + Op: urlErr.Op, + URL: urlErr.URL, + Err: fmt.Errorf("Connection closed by foreign host %s. Retry again.", urlErr.URL), + } } - return resp, err + return nil, err } + + // Response cannot be non-nil, report if its the case. + if resp == nil { + msg := "Response is empty. " + reportIssue + return nil, ErrInvalidArgument(msg) + } + // If trace is enabled, dump http request and response. if c.isTraceEnabled { err = c.dumpHTTP(req, resp) @@ -343,6 +362,85 @@ func (c Client) do(req *http.Request) (*http.Response, error) { return resp, nil } +// List of success status. +var successStatus = []int{ + http.StatusOK, + http.StatusNoContent, + http.StatusPartialContent, +} + +// executeMethod - instantiates a given method, and retries the +// request upon any error up to maxRetries attempts in a binomially +// delayed manner using a standard back off algorithm. +func (c Client) executeMethod(method string, metadata requestMetadata) (res *http.Response, err error) { + var isRetryable bool // Indicates if request can be retried. + var bodySeeker io.Seeker // Extracted seeker from io.Reader. + if metadata.contentBody != nil { + // Check if body is seekable then it is retryable. + bodySeeker, isRetryable = metadata.contentBody.(io.Seeker) + } + + // Retry executes the following function body if request has an + // error until maxRetries have been exhausted, retry attempts are + // performed after waiting for a given period of time in a + // binomial fashion. + for range newRetryTimer(MaxRetry, time.Second, time.Second*30, MaxJitter) { + if isRetryable { + // Seek back to beginning for each attempt. + if _, err = bodySeeker.Seek(0, 0); err != nil { + // If seek failed, no need to retry. + return nil, err + } + } + + // Instantiate a new request. + var req *http.Request + req, err = c.newRequest(method, metadata) + if err != nil { + errResponse := ToErrorResponse(err) + if isS3CodeRetryable(errResponse.Code) { + continue // Retry. + } + return nil, err + } + + // Initiate the request. + res, err = c.do(req) + if err != nil { + // For supported network errors verify. + if isNetErrorRetryable(err) { + continue // Retry. + } + // For other errors, return here no need to retry. + return nil, err + } + + // For any known successful http status, return quickly. + for _, httpStatus := range successStatus { + if httpStatus == res.StatusCode { + return res, nil + } + } + + // For errors verify if its retryable otherwise fail quickly. + errResponse := ToErrorResponse(httpRespToErrorResponse(res, metadata.bucketName, metadata.objectName)) + // Bucket region if set in error response, we can retry the + // request with the new region. + if errResponse.Region != "" { + c.bucketLocCache.Set(metadata.bucketName, errResponse.Region) + continue // Retry. + } + + // Verify if error response code is retryable. + if isS3CodeRetryable(errResponse.Code) { + continue // Retry. + } + // For all other cases break out of the retry loop. + break + } + return res, err +} + // newRequest - instantiate a new HTTP request for a given method. func (c Client) newRequest(method string, metadata requestMetadata) (req *http.Request, err error) { // If no method is supplied default to 'POST'. @@ -391,10 +489,13 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R // Set content body if available. if metadata.contentBody != nil { - req.Body = metadata.contentBody + req.Body = ioutil.NopCloser(metadata.contentBody) } - // set UserAgent for the request. + // set 'Expect' header for the request. + req.Header.Set("Expect", "100-continue") + + // set 'User-Agent' header for the request. c.setUserAgent(req) // Set all headers. @@ -493,46 +594,3 @@ func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, que return u, nil } - -// CloudStorageClient - Cloud Storage Client interface. -type CloudStorageClient interface { - // Bucket Read/Write/Stat operations. - MakeBucket(bucketName string, cannedACL BucketACL, location string) error - BucketExists(bucketName string) error - RemoveBucket(bucketName string) error - SetBucketACL(bucketName string, cannedACL BucketACL) error - GetBucketACL(bucketName string) (BucketACL, error) - - ListBuckets() ([]BucketInfo, error) - ListObjects(bucket, prefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo - ListIncompleteUploads(bucket, prefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo - - // Object Read/Write/Stat operations. - GetObject(bucketName, objectName string) (reader *Object, err error) - PutObject(bucketName, objectName string, reader io.Reader, contentType string) (n int64, err error) - StatObject(bucketName, objectName string) (ObjectInfo, error) - RemoveObject(bucketName, objectName string) error - RemoveIncompleteUpload(bucketName, objectName string) error - - // File to Object API. - FPutObject(bucketName, objectName, filePath, contentType string) (n int64, err error) - FGetObject(bucketName, objectName, filePath string) error - - // PutObjectWithProgress for progress. - PutObjectWithProgress(bucketName, objectName string, reader io.Reader, contentType string, progress io.Reader) (n int64, err error) - - // Presigned operations. - PresignedGetObject(bucketName, objectName string, expires time.Duration, reqParams url.Values) (presignedURL string, err error) - PresignedPutObject(bucketName, objectName string, expires time.Duration) (presignedURL string, err error) - PresignedPostPolicy(*PostPolicy) (formData map[string]string, err error) - - // Application info. - SetAppInfo(appName, appVersion string) - - // Set custom transport. - SetCustomTransport(customTransport http.RoundTripper) - - // HTTP tracing methods. - TraceOn(traceOutput io.Writer) - TraceOff() -} diff --git a/vendor/github.com/minio/minio-go/api_functional_v2_test.go b/vendor/github.com/minio/minio-go/api_functional_v2_test.go deleted file mode 100644 index a3f3411a5..000000000 --- a/vendor/github.com/minio/minio-go/api_functional_v2_test.go +++ /dev/null @@ -1,1073 +0,0 @@ -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio_test - -import ( - "bytes" - crand "crypto/rand" - "errors" - "io" - "io/ioutil" - "math/rand" - "net/http" - "net/url" - "os" - "testing" - "time" - - "github.com/minio/minio-go" -) - -// Tests bucket re-create errors. -func TestMakeBucketErrorV2(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // Make a new bucket in 'eu-west-1'. - if err = c.MakeBucket(bucketName, "private", "eu-west-1"); err != nil { - t.Fatal("Error:", err, bucketName) - } - if err = c.MakeBucket(bucketName, "private", "eu-west-1"); err == nil { - t.Fatal("Error: make bucket should should fail for", bucketName) - } - // Verify valid error response from server. - if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && - minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { - t.Fatal("Error: Invalid error returned by server", err) - } - if err = c.RemoveBucket(bucketName); err != nil { - t.Fatal("Error:", err, bucketName) - } -} - -// Test get object reader to not throw error on being closed twice. -func TestGetObjectClosedTwiceV2(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // Make a new bucket. - err = c.MakeBucket(bucketName, "private", "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K - buf := make([]byte, rand.Intn(1<<20)+32*1024) - - _, err = io.ReadFull(crand.Reader, buf) - if err != nil { - t.Fatal("Error:", err) - } - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano())) - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - if err := r.Close(); err != nil { - t.Fatal("Error:", err) - } - if err := r.Close(); err == nil { - t.Fatal("Error: object is already closed, should return error") - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests removing partially uploaded objects. -func TestRemovePartiallyUploadedV2(t *testing.T) { - if testing.Short() { - t.Skip("skipping function tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // make a new bucket. - err = c.MakeBucket(bucketName, "private", "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - reader, writer := io.Pipe() - go func() { - i := 0 - for i < 25 { - _, err = io.CopyN(writer, crand.Reader, 128*1024) - if err != nil { - t.Fatal("Error:", err, bucketName) - } - i++ - } - writer.CloseWithError(errors.New("Proactively closed to be verified later.")) - }() - - objectName := bucketName + "-resumable" - _, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream") - if err == nil { - t.Fatal("Error: PutObject should fail.") - } - if err.Error() != "Proactively closed to be verified later." { - t.Fatal("Error:", err) - } - err = c.RemoveIncompleteUpload(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests resumable put object cloud to cloud. -func TestResumbalePutObjectV2(t *testing.T) { - // By passing 'go test -short' skips these tests. - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // Make a new bucket. - err = c.MakeBucket(bucketName, "private", "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Create a temporary file. - file, err := ioutil.TempFile(os.TempDir(), "resumable") - if err != nil { - t.Fatal("Error:", err) - } - - // Copy 11MiB worth of random data. - n, err := io.CopyN(file, crand.Reader, 11*1024*1024) - if err != nil { - t.Fatal("Error:", err) - } - if n != int64(11*1024*1024) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) - } - - // Close the file pro-actively for windows. - if err = file.Close(); err != nil { - t.Fatal("Error:", err) - } - - // New object name. - objectName := bucketName + "-resumable" - - // Upload the file. - n, err = c.FPutObject(bucketName, objectName, file.Name(), "application/octet-stream") - if err != nil { - t.Fatal("Error:", err) - } - if n != int64(11*1024*1024) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) - } - - // Get the uploaded object. - reader, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err) - } - - // Upload now cloud to cloud. - n, err = c.PutObject(bucketName, objectName+"-put", reader, "application/octest-stream") - if err != nil { - t.Fatal("Error:", err) - } - - // Get object info. - objInfo, err := reader.Stat() - if err != nil { - t.Fatal("Error:", err) - } - if n != objInfo.Size { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", objInfo.Size, n) - } - - // Remove all temp files, objects and bucket. - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - - err = c.RemoveObject(bucketName, objectName+"-put") - if err != nil { - t.Fatal("Error: ", err) - } - - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } - - err = os.Remove(file.Name()) - if err != nil { - t.Fatal("Error:", err) - } - -} - -// Tests resumable file based put object multipart upload. -func TestResumableFPutObjectV2(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // make a new bucket. - err = c.MakeBucket(bucketName, "private", "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - file, err := ioutil.TempFile(os.TempDir(), "resumable") - if err != nil { - t.Fatal("Error:", err) - } - - n, err := io.CopyN(file, crand.Reader, 11*1024*1024) - if err != nil { - t.Fatal("Error:", err) - } - if n != int64(11*1024*1024) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) - } - - objectName := bucketName + "-resumable" - - n, err = c.FPutObject(bucketName, objectName, file.Name(), "application/octet-stream") - if err != nil { - t.Fatal("Error:", err) - } - if n != int64(11*1024*1024) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) - } - - // Close the file pro-actively for windows. - file.Close() - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } - - err = os.Remove(file.Name()) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests various bucket supported formats. -func TestMakeBucketRegionsV2(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(bucketName, "private", "eu-west-1"); err != nil { - t.Fatal("Error:", err, bucketName) - } - - if err = c.RemoveBucket(bucketName); err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Make a new bucket with '.' in its name, in 'us-west-2'. This - // request is internally staged into a path style instead of - // virtual host style. - if err = c.MakeBucket(bucketName+".withperiod", "private", "us-west-2"); err != nil { - t.Fatal("Error:", err, bucketName+".withperiod") - } - - // Remove the newly created bucket. - if err = c.RemoveBucket(bucketName + ".withperiod"); err != nil { - t.Fatal("Error:", err, bucketName+".withperiod") - } -} - -// Tests resumable put object multipart upload. -func TestResumablePutObjectV2(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // make a new bucket. - err = c.MakeBucket(bucketName, "private", "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // generate 11MB - buf := make([]byte, 11*1024*1024) - - _, err = io.ReadFull(crand.Reader, buf) - if err != nil { - t.Fatal("Error:", err) - } - - objectName := bucketName + "-resumable" - reader := bytes.NewReader(buf) - n, err := c.PutObject(bucketName, objectName, reader, "application/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests get object ReaderSeeker interface methods. -func TestGetObjectReadSeekFunctionalV2(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // Make a new bucket. - err = c.MakeBucket(bucketName, "private", "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K - buf := make([]byte, rand.Intn(1<<20)+32*1024) - - _, err = io.ReadFull(crand.Reader, buf) - if err != nil { - t.Fatal("Error:", err) - } - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano())) - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - - offset := int64(2048) - n, err = r.Seek(offset, 0) - if err != nil { - t.Fatal("Error:", err, offset) - } - if n != offset { - t.Fatalf("Error: number of bytes seeked does not match, want %v, got %v\n", - offset, n) - } - n, err = r.Seek(0, 1) - if err != nil { - t.Fatal("Error:", err) - } - if n != offset { - t.Fatalf("Error: number of current seek does not match, want %v, got %v\n", - offset, n) - } - _, err = r.Seek(offset, 2) - if err == nil { - t.Fatal("Error: seek on positive offset for whence '2' should error out") - } - n, err = r.Seek(-offset, 2) - if err != nil { - t.Fatal("Error:", err) - } - if n != 0 { - t.Fatalf("Error: number of bytes seeked back does not match, want 0, got %v\n", n) - } - var buffer bytes.Buffer - if _, err = io.CopyN(&buffer, r, st.Size); err != nil { - t.Fatal("Error:", err) - } - if !bytes.Equal(buf, buffer.Bytes()) { - t.Fatal("Error: Incorrect read bytes v/s original buffer.") - } - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests get object ReaderAt interface methods. -func TestGetObjectReadAtFunctionalV2(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // Make a new bucket. - err = c.MakeBucket(bucketName, "private", "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K - buf := make([]byte, rand.Intn(1<<20)+32*1024) - - _, err = io.ReadFull(crand.Reader, buf) - if err != nil { - t.Fatal("Error:", err) - } - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano())) - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - - offset := int64(2048) - - // Read directly - buf2 := make([]byte, 512) - buf3 := make([]byte, 512) - buf4 := make([]byte, 512) - - m, err := r.ReadAt(buf2, offset) - if err != nil { - t.Fatal("Error:", err, st.Size, len(buf2), offset) - } - if m != len(buf2) { - t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf2)) - } - if !bytes.Equal(buf2, buf[offset:offset+512]) { - t.Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - offset += 512 - m, err = r.ReadAt(buf3, offset) - if err != nil { - t.Fatal("Error:", err, st.Size, len(buf3), offset) - } - if m != len(buf3) { - t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf3)) - } - if !bytes.Equal(buf3, buf[offset:offset+512]) { - t.Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - offset += 512 - m, err = r.ReadAt(buf4, offset) - if err != nil { - t.Fatal("Error:", err, st.Size, len(buf4), offset) - } - if m != len(buf4) { - t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf4)) - } - if !bytes.Equal(buf4, buf[offset:offset+512]) { - t.Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - - buf5 := make([]byte, n) - // Read the whole object. - m, err = r.ReadAt(buf5, 0) - if err != nil { - if err != io.EOF { - t.Fatal("Error:", err, len(buf5)) - } - } - if m != len(buf5) { - t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf5)) - } - if !bytes.Equal(buf, buf5) { - t.Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") - } - - buf6 := make([]byte, n+1) - // Read the whole object and beyond. - _, err = r.ReadAt(buf6, 0) - if err != nil { - if err != io.EOF { - t.Fatal("Error:", err, len(buf6)) - } - } - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests comprehensive list of all methods. -func TestFunctionalV2(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - c, err := minio.NewV2( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable to debug - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // Make a new bucket. - err = c.MakeBucket(bucketName, "private", "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate a random file name. - fileName := randString(60, rand.NewSource(time.Now().UnixNano())) - file, err := os.Create(fileName) - if err != nil { - t.Fatal("Error:", err) - } - for i := 0; i < 3; i++ { - buf := make([]byte, rand.Intn(1<<19)) - _, err = file.Write(buf) - if err != nil { - t.Fatal("Error:", err) - } - } - file.Close() - - // Verify if bucket exits and you have access. - err = c.BucketExists(bucketName) - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Make the bucket 'public read/write'. - err = c.SetBucketACL(bucketName, "public-read-write") - if err != nil { - t.Fatal("Error:", err) - } - - // Get the previously set acl. - acl, err := c.GetBucketACL(bucketName) - if err != nil { - t.Fatal("Error:", err) - } - - // ACL must be 'public read/write'. - if acl != minio.BucketACL("public-read-write") { - t.Fatal("Error:", acl) - } - - // List all buckets. - buckets, err := c.ListBuckets() - if len(buckets) == 0 { - t.Fatal("Error: list buckets cannot be empty", buckets) - } - if err != nil { - t.Fatal("Error:", err) - } - - // Verify if previously created bucket is listed in list buckets. - bucketFound := false - for _, bucket := range buckets { - if bucket.Name == bucketName { - bucketFound = true - } - } - - // If bucket not found error out. - if !bucketFound { - t.Fatal("Error: bucket ", bucketName, "not found") - } - - objectName := bucketName + "unique" - - // Generate data - buf := make([]byte, rand.Intn(1<<19)) - _, err = io.ReadFull(crand.Reader, buf) - if err != nil { - t.Fatal("Error: ", err) - } - - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "") - if err != nil { - t.Fatal("Error: ", err) - } - if n != int64(len(buf)) { - t.Fatal("Error: bad length ", n, len(buf)) - } - - n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName+"-nolength") - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Instantiate a done channel to close all listing. - doneCh := make(chan struct{}) - defer close(doneCh) - - objFound := false - isRecursive := true // Recursive is true. - for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { - if obj.Key == objectName { - objFound = true - break - } - } - if !objFound { - t.Fatal("Error: object " + objectName + " not found.") - } - - incompObjNotFound := true - for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { - if objIncompl.Key != "" { - incompObjNotFound = false - break - } - } - if !incompObjNotFound { - t.Fatal("Error: unexpected dangling incomplete upload found.") - } - - newReader, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - - newReadBytes, err := ioutil.ReadAll(newReader) - if err != nil { - t.Fatal("Error: ", err) - } - - if !bytes.Equal(newReadBytes, buf) { - t.Fatal("Error: bytes mismatch.") - } - - err = c.FGetObject(bucketName, objectName, fileName+"-f") - if err != nil { - t.Fatal("Error: ", err) - } - - // Generate presigned GET object url. - presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) - if err != nil { - t.Fatal("Error: ", err) - } - // Verify if presigned url works. - resp, err := http.Get(presignedGetURL) - if err != nil { - t.Fatal("Error: ", err) - } - if resp.StatusCode != http.StatusOK { - t.Fatal("Error: ", resp.Status) - } - newPresignedBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal("Error: ", err) - } - if !bytes.Equal(newPresignedBytes, buf) { - t.Fatal("Error: bytes mismatch.") - } - - // Set request parameters. - reqParams := make(url.Values) - reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") - // Generate presigned GET object url. - presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) - if err != nil { - t.Fatal("Error: ", err) - } - // Verify if presigned url works. - resp, err = http.Get(presignedGetURL) - if err != nil { - t.Fatal("Error: ", err) - } - if resp.StatusCode != http.StatusOK { - t.Fatal("Error: ", resp.Status) - } - newPresignedBytes, err = ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal("Error: ", err) - } - if !bytes.Equal(newPresignedBytes, buf) { - t.Fatal("Error: bytes mismatch for presigned GET url.") - } - // Verify content disposition. - if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { - t.Fatalf("Error: wrong Content-Disposition received %s", resp.Header.Get("Content-Disposition")) - } - - presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) - if err != nil { - t.Fatal("Error: ", err) - } - buf = make([]byte, rand.Intn(1<<20)) - _, err = io.ReadFull(crand.Reader, buf) - if err != nil { - t.Fatal("Error: ", err) - } - req, err := http.NewRequest("PUT", presignedPutURL, bytes.NewReader(buf)) - if err != nil { - t.Fatal("Error: ", err) - } - httpClient := &http.Client{} - resp, err = httpClient.Do(req) - if err != nil { - t.Fatal("Error: ", err) - } - - newReader, err = c.GetObject(bucketName, objectName+"-presigned") - if err != nil { - t.Fatal("Error: ", err) - } - - newReadBytes, err = ioutil.ReadAll(newReader) - if err != nil { - t.Fatal("Error: ", err) - } - - if !bytes.Equal(newReadBytes, buf) { - t.Fatal("Error: bytes mismatch.") - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveObject(bucketName, objectName+"-f") - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveObject(bucketName, objectName+"-nolength") - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveObject(bucketName, objectName+"-presigned") - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } - err = c.RemoveBucket(bucketName) - if err == nil { - t.Fatal("Error:") - } - if err.Error() != "The specified bucket does not exist" { - t.Fatal("Error: ", err) - } - if err = os.Remove(fileName); err != nil { - t.Fatal("Error: ", err) - } - if err = os.Remove(fileName + "-f"); err != nil { - t.Fatal("Error: ", err) - } -} diff --git a/vendor/github.com/minio/minio-go/api_functional_v4_test.go b/vendor/github.com/minio/minio-go/api_functional_v4_test.go deleted file mode 100644 index 583ab79b9..000000000 --- a/vendor/github.com/minio/minio-go/api_functional_v4_test.go +++ /dev/null @@ -1,1101 +0,0 @@ -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio_test - -import ( - "bytes" - crand "crypto/rand" - "errors" - "io" - "io/ioutil" - "math/rand" - "net/http" - "net/url" - "os" - "testing" - "time" - - "github.com/minio/minio-go" -) - -const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" -const ( - letterIdxBits = 6 // 6 bits to represent a letter index - letterIdxMask = 1<= 0; { - if remain == 0 { - cache, remain = src.Int63(), letterIdxMax - } - if idx := int(cache & letterIdxMask); idx < len(letterBytes) { - b[i] = letterBytes[idx] - i-- - } - cache >>= letterIdxBits - remain-- - } - return string(b[0:30]) -} - -// Tests bucket re-create errors. -func TestMakeBucketError(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(bucketName, "private", "eu-central-1"); err != nil { - t.Fatal("Error:", err, bucketName) - } - if err = c.MakeBucket(bucketName, "private", "eu-central-1"); err == nil { - t.Fatal("Error: make bucket should should fail for", bucketName) - } - // Verify valid error response from server. - if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && - minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { - t.Fatal("Error: Invalid error returned by server", err) - } - if err = c.RemoveBucket(bucketName); err != nil { - t.Fatal("Error:", err, bucketName) - } -} - -// Tests various bucket supported formats. -func TestMakeBucketRegions(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(bucketName, "private", "eu-central-1"); err != nil { - t.Fatal("Error:", err, bucketName) - } - - if err = c.RemoveBucket(bucketName); err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Make a new bucket with '.' in its name, in 'us-west-2'. This - // request is internally staged into a path style instead of - // virtual host style. - if err = c.MakeBucket(bucketName+".withperiod", "private", "us-west-2"); err != nil { - t.Fatal("Error:", err, bucketName+".withperiod") - } - - // Remove the newly created bucket. - if err = c.RemoveBucket(bucketName + ".withperiod"); err != nil { - t.Fatal("Error:", err, bucketName+".withperiod") - } -} - -// Test get object reader to not throw error on being closed twice. -func TestGetObjectClosedTwice(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // Make a new bucket. - err = c.MakeBucket(bucketName, "private", "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K - buf := make([]byte, rand.Intn(1<<20)+32*1024) - - _, err = io.ReadFull(crand.Reader, buf) - if err != nil { - t.Fatal("Error:", err) - } - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano())) - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - if err := r.Close(); err != nil { - t.Fatal("Error:", err) - } - if err := r.Close(); err == nil { - t.Fatal("Error: object is already closed, should return error") - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests removing partially uploaded objects. -func TestRemovePartiallyUploaded(t *testing.T) { - if testing.Short() { - t.Skip("skipping function tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // Make a new bucket. - err = c.MakeBucket(bucketName, "private", "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - reader, writer := io.Pipe() - go func() { - i := 0 - for i < 25 { - _, err = io.CopyN(writer, crand.Reader, 128*1024) - if err != nil { - t.Fatal("Error:", err, bucketName) - } - i++ - } - err := writer.CloseWithError(errors.New("Proactively closed to be verified later.")) - if err != nil { - t.Fatal("Error:", err) - } - }() - - objectName := bucketName + "-resumable" - _, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream") - if err == nil { - t.Fatal("Error: PutObject should fail.") - } - if err.Error() != "Proactively closed to be verified later." { - t.Fatal("Error:", err) - } - err = c.RemoveIncompleteUpload(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests resumable put object cloud to cloud. -func TestResumbalePutObject(t *testing.T) { - // By passing 'go test -short' skips these tests. - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // Make a new bucket. - err = c.MakeBucket(bucketName, "private", "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Create a temporary file. - file, err := ioutil.TempFile(os.TempDir(), "resumable") - if err != nil { - t.Fatal("Error:", err) - } - - // Copy 11MiB worth of random data. - n, err := io.CopyN(file, crand.Reader, 11*1024*1024) - if err != nil { - t.Fatal("Error:", err) - } - if n != int64(11*1024*1024) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) - } - - // Close the file pro-actively for windows. - if err = file.Close(); err != nil { - t.Fatal("Error:", err) - } - - // New object name. - objectName := bucketName + "-resumable" - - // Upload the file. - n, err = c.FPutObject(bucketName, objectName, file.Name(), "application/octet-stream") - if err != nil { - t.Fatal("Error:", err) - } - if n != int64(11*1024*1024) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) - } - - // Get the uploaded object. - reader, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err) - } - - // Upload now cloud to cloud. - n, err = c.PutObject(bucketName, objectName+"-put", reader, "application/octest-stream") - if err != nil { - t.Fatal("Error:", err) - } - - // Get object info. - objInfo, err := reader.Stat() - if err != nil { - t.Fatal("Error:", err) - } - if n != objInfo.Size { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", objInfo.Size, n) - } - - // Remove all temp files, objects and bucket. - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - - err = c.RemoveObject(bucketName, objectName+"-put") - if err != nil { - t.Fatal("Error: ", err) - } - - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } - - err = os.Remove(file.Name()) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests resumable file based put object multipart upload. -func TestResumableFPutObject(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // Make a new bucket. - err = c.MakeBucket(bucketName, "private", "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - file, err := ioutil.TempFile(os.TempDir(), "resumable") - if err != nil { - t.Fatal("Error:", err) - } - - n, err := io.CopyN(file, crand.Reader, 11*1024*1024) - if err != nil { - t.Fatal("Error:", err) - } - if n != int64(11*1024*1024) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) - } - - // Close the file pro-actively for windows. - err = file.Close() - if err != nil { - t.Fatal("Error:", err) - } - - objectName := bucketName + "-resumable" - - n, err = c.FPutObject(bucketName, objectName, file.Name(), "application/octet-stream") - if err != nil { - t.Fatal("Error:", err) - } - if n != int64(11*1024*1024) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } - - err = os.Remove(file.Name()) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests resumable put object multipart upload. -func TestResumablePutObject(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // Make a new bucket. - err = c.MakeBucket(bucketName, "private", "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate 11MB - buf := make([]byte, 11*1024*1024) - - _, err = io.ReadFull(crand.Reader, buf) - if err != nil { - t.Fatal("Error:", err) - } - - objectName := bucketName + "-resumable" - reader := bytes.NewReader(buf) - n, err := c.PutObject(bucketName, objectName, reader, "application/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests get object ReaderSeeker interface methods. -func TestGetObjectReadSeekFunctional(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // Make a new bucket. - err = c.MakeBucket(bucketName, "private", "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K - buf := make([]byte, rand.Intn(1<<20)+32*1024) - - _, err = io.ReadFull(crand.Reader, buf) - if err != nil { - t.Fatal("Error:", err) - } - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano())) - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - - offset := int64(2048) - n, err = r.Seek(offset, 0) - if err != nil { - t.Fatal("Error:", err, offset) - } - if n != offset { - t.Fatalf("Error: number of bytes seeked does not match, want %v, got %v\n", - offset, n) - } - n, err = r.Seek(0, 1) - if err != nil { - t.Fatal("Error:", err) - } - if n != offset { - t.Fatalf("Error: number of current seek does not match, want %v, got %v\n", - offset, n) - } - _, err = r.Seek(offset, 2) - if err == nil { - t.Fatal("Error: seek on positive offset for whence '2' should error out") - } - n, err = r.Seek(-offset, 2) - if err != nil { - t.Fatal("Error:", err) - } - if n != 0 { - t.Fatalf("Error: number of bytes seeked back does not match, want 0, got %v\n", n) - } - var buffer bytes.Buffer - if _, err = io.CopyN(&buffer, r, st.Size); err != nil { - t.Fatal("Error:", err) - } - if !bytes.Equal(buf, buffer.Bytes()) { - t.Fatal("Error: Incorrect read bytes v/s original buffer.") - } - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests get object ReaderAt interface methods. -func TestGetObjectReadAtFunctional(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // Make a new bucket. - err = c.MakeBucket(bucketName, "private", "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K - buf := make([]byte, rand.Intn(1<<20)+32*1024) - - _, err = io.ReadFull(crand.Reader, buf) - if err != nil { - t.Fatal("Error:", err) - } - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano())) - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - - offset := int64(2048) - - // read directly - buf2 := make([]byte, 512) - buf3 := make([]byte, 512) - buf4 := make([]byte, 512) - - m, err := r.ReadAt(buf2, offset) - if err != nil { - t.Fatal("Error:", err, st.Size, len(buf2), offset) - } - if m != len(buf2) { - t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf2)) - } - if !bytes.Equal(buf2, buf[offset:offset+512]) { - t.Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - offset += 512 - m, err = r.ReadAt(buf3, offset) - if err != nil { - t.Fatal("Error:", err, st.Size, len(buf3), offset) - } - if m != len(buf3) { - t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf3)) - } - if !bytes.Equal(buf3, buf[offset:offset+512]) { - t.Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - offset += 512 - m, err = r.ReadAt(buf4, offset) - if err != nil { - t.Fatal("Error:", err, st.Size, len(buf4), offset) - } - if m != len(buf4) { - t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf4)) - } - if !bytes.Equal(buf4, buf[offset:offset+512]) { - t.Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - - buf5 := make([]byte, n) - // Read the whole object. - m, err = r.ReadAt(buf5, 0) - if err != nil { - if err != io.EOF { - t.Fatal("Error:", err, len(buf5)) - } - } - if m != len(buf5) { - t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf5)) - } - if !bytes.Equal(buf, buf5) { - t.Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") - } - - buf6 := make([]byte, n+1) - // Read the whole object and beyond. - _, err = r.ReadAt(buf6, 0) - if err != nil { - if err != io.EOF { - t.Fatal("Error:", err, len(buf6)) - } - } - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } -} - -// Tests comprehensive list of all methods. -func TestFunctional(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - c, err := minio.New( - "s3.amazonaws.com", - os.Getenv("ACCESS_KEY"), - os.Getenv("SECRET_KEY"), - false, - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable to debug - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) - - // Make a new bucket. - err = c.MakeBucket(bucketName, "private", "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate a random file name. - fileName := randString(60, rand.NewSource(time.Now().UnixNano())) - file, err := os.Create(fileName) - if err != nil { - t.Fatal("Error:", err) - } - for i := 0; i < 3; i++ { - buf := make([]byte, rand.Intn(1<<19)) - _, err = file.Write(buf) - if err != nil { - t.Fatal("Error:", err) - } - } - file.Close() - - // Verify if bucket exits and you have access. - err = c.BucketExists(bucketName) - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Make the bucket 'public read/write'. - err = c.SetBucketACL(bucketName, "public-read-write") - if err != nil { - t.Fatal("Error:", err) - } - - // Get the previously set acl. - acl, err := c.GetBucketACL(bucketName) - if err != nil { - t.Fatal("Error:", err) - } - - // ACL must be 'public read/write'. - if acl != minio.BucketACL("public-read-write") { - t.Fatal("Error:", acl) - } - - // List all buckets. - buckets, err := c.ListBuckets() - if len(buckets) == 0 { - t.Fatal("Error: list buckets cannot be empty", buckets) - } - if err != nil { - t.Fatal("Error:", err) - } - - // Verify if previously created bucket is listed in list buckets. - bucketFound := false - for _, bucket := range buckets { - if bucket.Name == bucketName { - bucketFound = true - } - } - - // If bucket not found error out. - if !bucketFound { - t.Fatal("Error: bucket ", bucketName, "not found") - } - - objectName := bucketName + "unique" - - // Generate data - buf := make([]byte, rand.Intn(1<<19)) - _, err = io.ReadFull(crand.Reader, buf) - if err != nil { - t.Fatal("Error: ", err) - } - - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "") - if err != nil { - t.Fatal("Error: ", err) - } - if n != int64(len(buf)) { - t.Fatal("Error: bad length ", n, len(buf)) - } - - n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - t.Fatal("Error:", err, bucketName, objectName+"-nolength") - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Instantiate a done channel to close all listing. - doneCh := make(chan struct{}) - defer close(doneCh) - - objFound := false - isRecursive := true // Recursive is true. - for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { - if obj.Key == objectName { - objFound = true - break - } - } - if !objFound { - t.Fatal("Error: object " + objectName + " not found.") - } - - incompObjNotFound := true - for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { - if objIncompl.Key != "" { - incompObjNotFound = false - break - } - } - if !incompObjNotFound { - t.Fatal("Error: unexpected dangling incomplete upload found.") - } - - newReader, err := c.GetObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - - newReadBytes, err := ioutil.ReadAll(newReader) - if err != nil { - t.Fatal("Error: ", err) - } - - if !bytes.Equal(newReadBytes, buf) { - t.Fatal("Error: bytes mismatch.") - } - - err = c.FGetObject(bucketName, objectName, fileName+"-f") - if err != nil { - t.Fatal("Error: ", err) - } - - // Generate presigned GET object url. - presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) - if err != nil { - t.Fatal("Error: ", err) - } - - // Verify if presigned url works. - resp, err := http.Get(presignedGetURL) - if err != nil { - t.Fatal("Error: ", err) - } - if resp.StatusCode != http.StatusOK { - t.Fatal("Error: ", resp.Status) - } - newPresignedBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal("Error: ", err) - } - if !bytes.Equal(newPresignedBytes, buf) { - t.Fatal("Error: bytes mismatch.") - } - - // Set request parameters. - reqParams := make(url.Values) - reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") - presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) - if err != nil { - t.Fatal("Error: ", err) - } - // Verify if presigned url works. - resp, err = http.Get(presignedGetURL) - if err != nil { - t.Fatal("Error: ", err) - } - if resp.StatusCode != http.StatusOK { - t.Fatal("Error: ", resp.Status) - } - newPresignedBytes, err = ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal("Error: ", err) - } - if !bytes.Equal(newPresignedBytes, buf) { - t.Fatal("Error: bytes mismatch for presigned GET URL.") - } - if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { - t.Fatalf("Error: wrong Content-Disposition received %s", resp.Header.Get("Content-Disposition")) - } - - presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) - if err != nil { - t.Fatal("Error: ", err) - } - buf = make([]byte, rand.Intn(1<<20)) - _, err = io.ReadFull(crand.Reader, buf) - if err != nil { - t.Fatal("Error: ", err) - } - req, err := http.NewRequest("PUT", presignedPutURL, bytes.NewReader(buf)) - if err != nil { - t.Fatal("Error: ", err) - } - httpClient := &http.Client{} - resp, err = httpClient.Do(req) - if err != nil { - t.Fatal("Error: ", err) - } - - newReader, err = c.GetObject(bucketName, objectName+"-presigned") - if err != nil { - t.Fatal("Error: ", err) - } - - newReadBytes, err = ioutil.ReadAll(newReader) - if err != nil { - t.Fatal("Error: ", err) - } - - if !bytes.Equal(newReadBytes, buf) { - t.Fatal("Error: bytes mismatch.") - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveObject(bucketName, objectName+"-f") - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveObject(bucketName, objectName+"-nolength") - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveObject(bucketName, objectName+"-presigned") - if err != nil { - t.Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - t.Fatal("Error:", err) - } - err = c.RemoveBucket(bucketName) - if err == nil { - t.Fatal("Error:") - } - if err.Error() != "The specified bucket does not exist" { - t.Fatal("Error: ", err) - } - if err = os.Remove(fileName); err != nil { - t.Fatal("Error: ", err) - } - if err = os.Remove(fileName + "-f"); err != nil { - t.Fatal("Error: ", err) - } -} diff --git a/vendor/github.com/minio/minio-go/api_unit_test.go b/vendor/github.com/minio/minio-go/api_unit_test.go deleted file mode 100644 index cbae12fe2..000000000 --- a/vendor/github.com/minio/minio-go/api_unit_test.go +++ /dev/null @@ -1,599 +0,0 @@ -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - "os" - "strings" - "testing" -) - -type customReader struct{} - -func (c *customReader) Read(p []byte) (n int, err error) { - return 0, nil -} - -func (c *customReader) Size() (n int64) { - return 10 -} - -// Tests getReaderSize() for various Reader types. -func TestGetReaderSize(t *testing.T) { - var reader io.Reader - size, err := getReaderSize(reader) - if err != nil { - t.Fatal("Error:", err) - } - if size != -1 { - t.Fatal("Reader shouldn't have any length.") - } - - bytesReader := bytes.NewReader([]byte("Hello World")) - size, err = getReaderSize(bytesReader) - if err != nil { - t.Fatal("Error:", err) - } - if size != int64(len("Hello World")) { - t.Fatalf("Reader length doesn't match got: %v, want: %v", size, len("Hello World")) - } - - size, err = getReaderSize(new(customReader)) - if err != nil { - t.Fatal("Error:", err) - } - if size != int64(10) { - t.Fatalf("Reader length doesn't match got: %v, want: %v", size, 10) - } - - stringsReader := strings.NewReader("Hello World") - size, err = getReaderSize(stringsReader) - if err != nil { - t.Fatal("Error:", err) - } - if size != int64(len("Hello World")) { - t.Fatalf("Reader length doesn't match got: %v, want: %v", size, len("Hello World")) - } - - // Create request channel. - reqCh := make(chan readRequest) - // Create response channel. - resCh := make(chan readResponse) - // Create done channel. - doneCh := make(chan struct{}) - // objectInfo. - objectInfo := ObjectInfo{Size: 10} - objectReader := newObject(reqCh, resCh, doneCh, objectInfo) - defer objectReader.Close() - - size, err = getReaderSize(objectReader) - if err != nil { - t.Fatal("Error:", err) - } - if size != int64(10) { - t.Fatalf("Reader length doesn't match got: %v, want: %v", size, 10) - } - - fileReader, err := ioutil.TempFile(os.TempDir(), "prefix") - if err != nil { - t.Fatal("Error:", err) - } - defer fileReader.Close() - defer os.RemoveAll(fileReader.Name()) - - size, err = getReaderSize(fileReader) - if err != nil { - t.Fatal("Error:", err) - } - if size == -1 { - t.Fatal("Reader length for file cannot be -1.") - } - - // Verify for standard input, output and error file descriptors. - size, err = getReaderSize(os.Stdin) - if err != nil { - t.Fatal("Error:", err) - } - if size != -1 { - t.Fatal("Stdin should have length of -1.") - } - size, err = getReaderSize(os.Stdout) - if err != nil { - t.Fatal("Error:", err) - } - if size != -1 { - t.Fatal("Stdout should have length of -1.") - } - size, err = getReaderSize(os.Stderr) - if err != nil { - t.Fatal("Error:", err) - } - if size != -1 { - t.Fatal("Stderr should have length of -1.") - } - file, err := os.Open(os.TempDir()) - if err != nil { - t.Fatal("Error:", err) - } - defer file.Close() - _, err = getReaderSize(file) - if err == nil { - t.Fatal("Input file as directory should throw an error.") - } -} - -// Tests valid hosts for location. -func TestValidBucketLocation(t *testing.T) { - s3Hosts := []struct { - bucketLocation string - endpoint string - }{ - {"us-east-1", "s3.amazonaws.com"}, - {"unknown", "s3.amazonaws.com"}, - {"ap-southeast-1", "s3-ap-southeast-1.amazonaws.com"}, - } - for _, s3Host := range s3Hosts { - endpoint := getS3Endpoint(s3Host.bucketLocation) - if endpoint != s3Host.endpoint { - t.Fatal("Error: invalid bucket location", endpoint) - } - } -} - -// Tests valid bucket names. -func TestBucketNames(t *testing.T) { - buckets := []struct { - name string - valid error - }{ - {".mybucket", ErrInvalidBucketName("Bucket name cannot start or end with a '.' dot.")}, - {"mybucket.", ErrInvalidBucketName("Bucket name cannot start or end with a '.' dot.")}, - {"mybucket-", ErrInvalidBucketName("Bucket name contains invalid characters.")}, - {"my", ErrInvalidBucketName("Bucket name cannot be smaller than 3 characters.")}, - {"", ErrInvalidBucketName("Bucket name cannot be empty.")}, - {"my..bucket", ErrInvalidBucketName("Bucket name cannot have successive periods.")}, - {"my.bucket.com", nil}, - {"my-bucket", nil}, - {"123my-bucket", nil}, - } - - for _, b := range buckets { - err := isValidBucketName(b.name) - if err != b.valid { - t.Fatal("Error:", err) - } - } -} - -// Tests temp file. -func TestTempFile(t *testing.T) { - tmpFile, err := newTempFile("testing") - if err != nil { - t.Fatal("Error:", err) - } - fileName := tmpFile.Name() - // Closing temporary file purges the file. - err = tmpFile.Close() - if err != nil { - t.Fatal("Error:", err) - } - st, err := os.Stat(fileName) - if err != nil && !os.IsNotExist(err) { - t.Fatal("Error:", err) - } - if err == nil && st != nil { - t.Fatal("Error: file should be deleted and should not exist.") - } -} - -// Tests url encoding. -func TestEncodeURL2Path(t *testing.T) { - type urlStrings struct { - objName string - encodedObjName string - } - - bucketName := "bucketName" - want := []urlStrings{ - { - objName: "本語", - encodedObjName: "%E6%9C%AC%E8%AA%9E", - }, - { - objName: "本語.1", - encodedObjName: "%E6%9C%AC%E8%AA%9E.1", - }, - { - objName: ">123>3123123", - encodedObjName: "%3E123%3E3123123", - }, - { - objName: "test 1 2.txt", - encodedObjName: "test%201%202.txt", - }, - { - objName: "test++ 1.txt", - encodedObjName: "test%2B%2B%201.txt", - }, - } - - for _, o := range want { - u, err := url.Parse(fmt.Sprintf("https://%s.s3.amazonaws.com/%s", bucketName, o.objName)) - if err != nil { - t.Fatal("Error:", err) - } - urlPath := "/" + bucketName + "/" + o.encodedObjName - if urlPath != encodeURL2Path(u) { - t.Fatal("Error") - } - } -} - -// Tests error response structure. -func TestErrorResponse(t *testing.T) { - var err error - err = ErrorResponse{ - Code: "Testing", - } - errResp := ToErrorResponse(err) - if errResp.Code != "Testing" { - t.Fatal("Type conversion failed, we have an empty struct.") - } - - // Test http response decoding. - var httpResponse *http.Response - // Set empty variables - httpResponse = nil - var bucketName, objectName string - - // Should fail with invalid argument. - err = httpRespToErrorResponse(httpResponse, bucketName, objectName) - errResp = ToErrorResponse(err) - if errResp.Code != "InvalidArgument" { - t.Fatal("Empty response input should return invalid argument.") - } -} - -// Tests signature calculation. -func TestSignatureCalculation(t *testing.T) { - req, err := http.NewRequest("GET", "https://s3.amazonaws.com", nil) - if err != nil { - t.Fatal("Error:", err) - } - req = signV4(*req, "", "", "us-east-1") - if req.Header.Get("Authorization") != "" { - t.Fatal("Error: anonymous credentials should not have Authorization header.") - } - - req = preSignV4(*req, "", "", "us-east-1", 0) - if strings.Contains(req.URL.RawQuery, "X-Amz-Signature") { - t.Fatal("Error: anonymous credentials should not have Signature query resource.") - } - - req = signV2(*req, "", "") - if req.Header.Get("Authorization") != "" { - t.Fatal("Error: anonymous credentials should not have Authorization header.") - } - - req = preSignV2(*req, "", "", 0) - if strings.Contains(req.URL.RawQuery, "Signature") { - t.Fatal("Error: anonymous credentials should not have Signature query resource.") - } - - req = signV4(*req, "ACCESS-KEY", "SECRET-KEY", "us-east-1") - if req.Header.Get("Authorization") == "" { - t.Fatal("Error: normal credentials should have Authorization header.") - } - - req = preSignV4(*req, "ACCESS-KEY", "SECRET-KEY", "us-east-1", 0) - if !strings.Contains(req.URL.RawQuery, "X-Amz-Signature") { - t.Fatal("Error: normal credentials should have Signature query resource.") - } - - req = signV2(*req, "ACCESS-KEY", "SECRET-KEY") - if req.Header.Get("Authorization") == "" { - t.Fatal("Error: normal credentials should have Authorization header.") - } - - req = preSignV2(*req, "ACCESS-KEY", "SECRET-KEY", 0) - if !strings.Contains(req.URL.RawQuery, "Signature") { - t.Fatal("Error: normal credentials should not have Signature query resource.") - } -} - -// Tests signature type. -func TestSignatureType(t *testing.T) { - clnt := Client{} - if !clnt.signature.isV4() { - t.Fatal("Error") - } - clnt.signature = SignatureV2 - if !clnt.signature.isV2() { - t.Fatal("Error") - } - if clnt.signature.isV4() { - t.Fatal("Error") - } - clnt.signature = SignatureV4 - if !clnt.signature.isV4() { - t.Fatal("Error") - } -} - -// Tests bucket acl types. -func TestBucketACLTypes(t *testing.T) { - want := map[string]bool{ - "private": true, - "public-read": true, - "public-read-write": true, - "authenticated-read": true, - "invalid": false, - } - for acl, ok := range want { - if BucketACL(acl).isValidBucketACL() != ok { - t.Fatal("Error") - } - } -} - -// Tests optimal part size. -func TestPartSize(t *testing.T) { - totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(5000000000000000000) - if err == nil { - t.Fatal("Error: should fail") - } - totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(5497558138880) - if err != nil { - t.Fatal("Error: ", err) - } - if totalPartsCount != 9987 { - t.Fatalf("Error: expecting total parts count of 9987: got %v instead", totalPartsCount) - } - if partSize != 550502400 { - t.Fatalf("Error: expecting part size of 550502400: got %v instead", partSize) - } - if lastPartSize != 241172480 { - t.Fatalf("Error: expecting last part size of 241172480: got %v instead", lastPartSize) - } - totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(5000000000) - if err != nil { - t.Fatal("Error:", err) - } - if partSize != minPartSize { - t.Fatalf("Error: expecting part size of %v: got %v instead", minPartSize, partSize) - } - totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(-1) - if err != nil { - t.Fatal("Error:", err) - } - if totalPartsCount != 9987 { - t.Fatalf("Error: expecting total parts count of 9987: got %v instead", totalPartsCount) - } - if partSize != 550502400 { - t.Fatalf("Error: expecting part size of 550502400: got %v instead", partSize) - } - if lastPartSize != 241172480 { - t.Fatalf("Error: expecting last part size of 241172480: got %v instead", lastPartSize) - } -} - -// Tests query values to URL encoding. -func TestQueryURLEncoding(t *testing.T) { - urlValues := make(url.Values) - urlValues.Set("prefix", "test@1123") - urlValues.Set("delimiter", "/") - urlValues.Set("marker", "%%%@$$$") - - queryStr := queryEncode(urlValues) - if !strings.Contains(queryStr, "test%401123") { - t.Fatalf("Error: @ should be encoded as %s, invalid query string %s", "test%401123", queryStr) - } - if !strings.Contains(queryStr, "%25%25%25%40%24%24%24") { - t.Fatalf("Error: %s should be encoded as %s, invalid query string %s", "%%%@$$$", "%25%25%25%40%24%24%24", queryStr) - } -} - -// Tests url encoding. -func TestURLEncoding(t *testing.T) { - type urlStrings struct { - name string - encodedName string - } - - want := []urlStrings{ - { - name: "bigfile-1._%", - encodedName: "bigfile-1._%25", - }, - { - name: "本語", - encodedName: "%E6%9C%AC%E8%AA%9E", - }, - { - name: "本語.1", - encodedName: "%E6%9C%AC%E8%AA%9E.1", - }, - { - name: ">123>3123123", - encodedName: "%3E123%3E3123123", - }, - { - name: "test 1 2.txt", - encodedName: "test%201%202.txt", - }, - { - name: "test++ 1.txt", - encodedName: "test%2B%2B%201.txt", - }, - } - - for _, u := range want { - if u.encodedName != urlEncodePath(u.name) { - t.Fatal("Error") - } - } -} - -// Tests constructing valid endpoint url. -func TestGetEndpointURL(t *testing.T) { - if _, err := getEndpointURL("s3.amazonaws.com", false); err != nil { - t.Fatal("Error:", err) - } - if _, err := getEndpointURL("192.168.1.1", false); err != nil { - t.Fatal("Error:", err) - } - if _, err := getEndpointURL("13333.123123.-", false); err == nil { - t.Fatal("Error") - } - if _, err := getEndpointURL("s3.aamzza.-", false); err == nil { - t.Fatal("Error") - } - if _, err := getEndpointURL("s3.amazonaws.com:443", false); err == nil { - t.Fatal("Error") - } -} - -// Tests valid ip address. -func TestValidIPAddr(t *testing.T) { - type validIP struct { - ip string - valid bool - } - - want := []validIP{ - { - ip: "192.168.1.1", - valid: true, - }, - { - ip: "192.1.8", - valid: false, - }, - { - ip: "..192.", - valid: false, - }, - { - ip: "192.168.1.1.1", - valid: false, - }, - } - for _, w := range want { - valid := isValidIP(w.ip) - if valid != w.valid { - t.Fatal("Error") - } - } -} - -// Tests valid endpoint domain. -func TestValidEndpointDomain(t *testing.T) { - type validEndpoint struct { - endpointDomain string - valid bool - } - - want := []validEndpoint{ - { - endpointDomain: "s3.amazonaws.com", - valid: true, - }, - { - endpointDomain: "s3.amazonaws.com_", - valid: false, - }, - { - endpointDomain: "%$$$", - valid: false, - }, - { - endpointDomain: "s3.amz.test.com", - valid: true, - }, - { - endpointDomain: "s3.%%", - valid: false, - }, - { - endpointDomain: "localhost", - valid: true, - }, - { - endpointDomain: "-localhost", - valid: false, - }, - { - endpointDomain: "", - valid: false, - }, - { - endpointDomain: "\n \t", - valid: false, - }, - { - endpointDomain: " ", - valid: false, - }, - } - for _, w := range want { - valid := isValidDomain(w.endpointDomain) - if valid != w.valid { - t.Fatal("Error:", w.endpointDomain) - } - } -} - -// Tests valid endpoint url. -func TestValidEndpointURL(t *testing.T) { - type validURL struct { - url string - valid bool - } - want := []validURL{ - { - url: "https://s3.amazonaws.com", - valid: true, - }, - { - url: "https://s3.amazonaws.com/bucket/object", - valid: false, - }, - { - url: "192.168.1.1", - valid: false, - }, - } - for _, w := range want { - u, err := url.Parse(w.url) - if err != nil { - t.Fatal("Error:", err) - } - valid := false - if err := isValidEndpointURL(u); err == nil { - valid = true - } - if valid != w.valid { - t.Fatal("Error") - } - } -} diff --git a/vendor/github.com/minio/minio-go/bucket-cache.go b/vendor/github.com/minio/minio-go/bucket-cache.go index c0b4f0cb8..7111c8f41 100644 --- a/vendor/github.com/minio/minio-go/bucket-cache.go +++ b/vendor/github.com/minio/minio-go/bucket-cache.go @@ -20,7 +20,7 @@ import ( "encoding/hex" "net/http" "net/url" - "path/filepath" + "path" "sync" ) @@ -90,7 +90,20 @@ func (c Client) getBucketLocation(bucketName string) (string, error) { } if resp != nil { if resp.StatusCode != http.StatusOK { - return "", httpRespToErrorResponse(resp, bucketName, "") + err = httpRespToErrorResponse(resp, bucketName, "") + errResp := ToErrorResponse(err) + // AccessDenied without a signature mismatch code, + // usually means that the bucket policy has certain + // restrictions where some API operations are not + // allowed. Handle this case so that top level callers can + // interpret this easily and fall back if needed to a + // lower functionality call. Read each individual API + // specific code for such fallbacks. + if errResp.Code == "AccessDenied" && errResp.Message == "Access Denied" { + // In this case return as "us-east-1" and let the call fail. + return "us-east-1", nil + } + return "", err } } @@ -127,7 +140,7 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro // Set get bucket location always as path style. targetURL := c.endpointURL - targetURL.Path = filepath.Join(bucketName, "") + "/" + targetURL.Path = path.Join(bucketName, "") + "/" targetURL.RawQuery = urlValues.Encode() // Get a new HTTP request for the method. diff --git a/vendor/github.com/minio/minio-go/hook-reader.go b/vendor/github.com/minio/minio-go/hook-reader.go index 043425f23..bc9ece049 100644 --- a/vendor/github.com/minio/minio-go/hook-reader.go +++ b/vendor/github.com/minio/minio-go/hook-reader.go @@ -1,5 +1,5 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,6 +27,22 @@ type hookReader struct { hook io.Reader } +// Seek implements io.Seeker. Seeks source first, and if necessary +// seeks hook if Seek method is appropriately found. +func (hr *hookReader) Seek(offset int64, whence int) (n int64, err error) { + // Verify for source has embedded Seeker, use it. + sourceSeeker, ok := hr.source.(io.Seeker) + if ok { + return sourceSeeker.Seek(offset, whence) + } + // Verify if hook has embedded Seeker, use it. + hookSeeker, ok := hr.hook.(io.Seeker) + if ok { + return hookSeeker.Seek(offset, whence) + } + return n, nil +} + // Read implements io.Reader. Always reads from the source, the return // value 'n' number of bytes are reported through the hook. Returns // error for all non io.EOF conditions. @@ -44,7 +60,7 @@ func (hr *hookReader) Read(b []byte) (n int, err error) { return n, err } -// newHook returns a io.Reader which implements hookReader that +// newHook returns a io.ReadSeeker which implements hookReader that // reports the data read from the source to the hook. func newHook(source, hook io.Reader) io.Reader { if hook == nil { diff --git a/vendor/github.com/minio/minio-go/retry.go b/vendor/github.com/minio/minio-go/retry.go new file mode 100644 index 000000000..73981ce47 --- /dev/null +++ b/vendor/github.com/minio/minio-go/retry.go @@ -0,0 +1,95 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "math/rand" + "net" + "time" +) + +// MaxRetry is the maximum number of retries before stopping. +var MaxRetry = 5 + +// MaxJitter will randomize over the full exponential backoff time +const MaxJitter = 1.0 + +// NoJitter disables the use of jitter for randomizing the exponential backoff time +const NoJitter = 0.0 + +// newRetryTimer creates a timer with exponentially increasing delays +// until the maximum retry attempts are reached. +func newRetryTimer(maxRetry int, unit time.Duration, cap time.Duration, jitter float64) <-chan int { + attemptCh := make(chan int) + + // Seed random function with current unix nano time. + rand.Seed(time.Now().UTC().UnixNano()) + + go func() { + defer close(attemptCh) + for i := 0; i < maxRetry; i++ { + attemptCh <- i + 1 // Attempts start from 1. + // Grow the interval at an exponential rate, + // starting at unit and capping at cap + time.Sleep(exponentialBackoffWait(unit, i, cap, jitter)) + } + }() + return attemptCh +} + +// isNetErrorRetryable - is network error retryable. +func isNetErrorRetryable(err error) bool { + switch err.(type) { + case *net.DNSError, *net.OpError, net.UnknownNetworkError: + return true + } + return false +} + +// computes the exponential backoff duration according to +// https://www.awsarchitectureblog.com/2015/03/backoff.html +func exponentialBackoffWait(base time.Duration, attempt int, cap time.Duration, jitter float64) time.Duration { + // normalize jitter to the range [0, 1.0] + if jitter < NoJitter { + jitter = NoJitter + } + if jitter > MaxJitter { + jitter = MaxJitter + } + //sleep = random_between(0, min(cap, base * 2 ** attempt)) + sleep := base * time.Duration(1< cap { + sleep = cap + } + if jitter != NoJitter { + sleep -= time.Duration(rand.Float64() * float64(sleep) * jitter) + } + return sleep +} + +// isS3CodeRetryable - is s3 error code retryable. +func isS3CodeRetryable(s3Code string) bool { + switch s3Code { + case "RequestError", "RequestTimeout", "Throttling", "ThrottlingException": + fallthrough + case "RequestLimitExceeded", "RequestThrottled", "InternalError": + fallthrough + case "ExpiredToken", "ExpiredTokenException": + return true + } + return false +} diff --git a/vendor/vendor.json b/vendor/vendor.json index f5567308a..80435254e 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -89,8 +89,8 @@ }, { "path": "github.com/minio/minio-go", - "revision": "280f16a52008d3ebba1bd64398b9b082e6738386", - "revisionTime": "2016-02-07T03:45:25-08:00" + "revision": "a4c6c439feb53e1aad083f0a3f0083a047092c17", + "revisionTime": "2016-03-10T10:12:11-08:00" }, { "path": "github.com/minio/miniobrowser",