From bdb1a907201fd3b56018d9046a8e9b143cd2e6e6 Mon Sep 17 00:00:00 2001 From: kannappanr Date: Wed, 14 Mar 2018 17:31:14 -0700 Subject: [PATCH] Add context support for gateway b2, manta, gcs --- cmd/gateway/b2/gateway-b2.go | 92 +++++++++++++++--------------- cmd/gateway/gcs/gateway-gcs.go | 52 ++++++++--------- cmd/gateway/manta/gateway-manta.go | 43 ++++++-------- 3 files changed, 88 insertions(+), 99 deletions(-) diff --git a/cmd/gateway/b2/gateway-b2.go b/cmd/gateway/b2/gateway-b2.go index 5e1fa6d82..57eb97574 100644 --- a/cmd/gateway/b2/gateway-b2.go +++ b/cmd/gateway/b2/gateway-b2.go @@ -199,18 +199,18 @@ func b2ToObjectError(err error, params ...string) error { // Shutdown saves any gateway metadata to disk // if necessary and reload upon next restart. -func (l *b2Objects) Shutdown() error { +func (l *b2Objects) Shutdown(ctx context.Context) error { // TODO return nil } // StorageInfo is not relevant to B2 backend. -func (l *b2Objects) StorageInfo() (si minio.StorageInfo) { +func (l *b2Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) { return si } // MakeBucket creates a new container on B2 backend. -func (l *b2Objects) MakeBucketWithLocation(bucket, location string) error { +func (l *b2Objects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error { // location is ignored for B2 backend. // All buckets are set to private by default. @@ -218,7 +218,7 @@ func (l *b2Objects) MakeBucketWithLocation(bucket, location string) error { return b2ToObjectError(errors.Trace(err), bucket) } -func (l *b2Objects) reAuthorizeAccount() error { +func (l *b2Objects) reAuthorizeAccount(ctx context.Context) error { client, err := b2.AuthorizeAccount(l.ctx, l.creds.AccessKey, l.creds.SecretKey, b2.Transport(minio.NewCustomHTTPTransport())) if err != nil { return err @@ -233,18 +233,18 @@ func (l *b2Objects) reAuthorizeAccount() error { // the account and updates the B2 client safely. Once successfully // authorized performs the call again and returns list of buckets. // For any errors which are not actionable we return an error. -func (l *b2Objects) listBuckets(err error) ([]*b2.Bucket, error) { +func (l *b2Objects) listBuckets(ctx context.Context, err error) ([]*b2.Bucket, error) { if err != nil { if b2.Action(err) != b2.ReAuthenticate { return nil, err } - if rerr := l.reAuthorizeAccount(); rerr != nil { + if rerr := l.reAuthorizeAccount(ctx); rerr != nil { return nil, rerr } } bktList, lerr := l.b2Client.ListBuckets(l.ctx) if lerr != nil { - return l.listBuckets(lerr) + return l.listBuckets(ctx, lerr) } return bktList, nil } @@ -253,8 +253,8 @@ func (l *b2Objects) listBuckets(err error) ([]*b2.Bucket, error) { // for performing an API operation. B2 API doesn't // provide a direct way to access the bucket so we need // to employ following technique. -func (l *b2Objects) Bucket(bucket string) (*b2.Bucket, error) { - bktList, err := l.listBuckets(nil) +func (l *b2Objects) Bucket(ctx context.Context, bucket string) (*b2.Bucket, error) { + bktList, err := l.listBuckets(ctx, nil) if err != nil { return nil, b2ToObjectError(errors.Trace(err), bucket) } @@ -267,8 +267,8 @@ func (l *b2Objects) Bucket(bucket string) (*b2.Bucket, error) { } // GetBucketInfo gets bucket metadata.. -func (l *b2Objects) GetBucketInfo(bucket string) (bi minio.BucketInfo, err error) { - if _, err = l.Bucket(bucket); err != nil { +func (l *b2Objects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, err error) { + if _, err = l.Bucket(ctx, bucket); err != nil { return bi, err } return minio.BucketInfo{ @@ -278,8 +278,8 @@ func (l *b2Objects) GetBucketInfo(bucket string) (bi minio.BucketInfo, err error } // ListBuckets lists all B2 buckets -func (l *b2Objects) ListBuckets() ([]minio.BucketInfo, error) { - bktList, err := l.listBuckets(nil) +func (l *b2Objects) ListBuckets(ctx context.Context) ([]minio.BucketInfo, error) { + bktList, err := l.listBuckets(ctx, nil) if err != nil { return nil, err } @@ -294,8 +294,8 @@ func (l *b2Objects) ListBuckets() ([]minio.BucketInfo, error) { } // DeleteBucket deletes a bucket on B2 -func (l *b2Objects) DeleteBucket(bucket string) error { - bkt, err := l.Bucket(bucket) +func (l *b2Objects) DeleteBucket(ctx context.Context, bucket string) error { + bkt, err := l.Bucket(ctx, bucket) if err != nil { return err } @@ -304,8 +304,8 @@ func (l *b2Objects) DeleteBucket(bucket string) error { } // ListObjects lists all objects in B2 bucket filtered by prefix, returns upto at max 1000 entries at a time. -func (l *b2Objects) ListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) { - bkt, err := l.Bucket(bucket) +func (l *b2Objects) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) { + bkt, err := l.Bucket(ctx, bucket) if err != nil { return loi, err } @@ -335,10 +335,10 @@ func (l *b2Objects) ListObjects(bucket string, prefix string, marker string, del } // ListObjectsV2 lists all objects in B2 bucket filtered by prefix, returns upto max 1000 entries at a time. -func (l *b2Objects) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, +func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) { // fetchOwner, startAfter are not supported and unused. - bkt, err := l.Bucket(bucket) + bkt, err := l.Bucket(ctx, bucket) if err != nil { return loi, err } @@ -374,8 +374,8 @@ func (l *b2Objects) ListObjectsV2(bucket, prefix, continuationToken, delimiter s // // startOffset indicates the starting read location of the object. // length indicates the total length of the object. -func (l *b2Objects) GetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string) error { - bkt, err := l.Bucket(bucket) +func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string) error { + bkt, err := l.Bucket(ctx, bucket) if err != nil { return err } @@ -389,8 +389,8 @@ func (l *b2Objects) GetObject(bucket string, object string, startOffset int64, l } // GetObjectInfo reads object info and replies back ObjectInfo -func (l *b2Objects) GetObjectInfo(bucket string, object string) (objInfo minio.ObjectInfo, err error) { - bkt, err := l.Bucket(bucket) +func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object string) (objInfo minio.ObjectInfo, err error) { + bkt, err := l.Bucket(ctx, bucket) if err != nil { return objInfo, err } @@ -478,8 +478,8 @@ func (nb *Reader) Read(p []byte) (int, error) { } // PutObject uploads the single upload to B2 backend by using *b2_upload_file* API, uploads upto 5GiB. -func (l *b2Objects) PutObject(bucket string, object string, data *h2.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { - bkt, err := l.Bucket(bucket) +func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string, data *h2.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { + bkt, err := l.Bucket(ctx, bucket) if err != nil { return objInfo, err } @@ -517,8 +517,8 @@ func (l *b2Objects) PutObject(bucket string, object string, data *h2.Reader, met } // DeleteObject deletes a blob in bucket -func (l *b2Objects) DeleteObject(bucket string, object string) error { - bkt, err := l.Bucket(bucket) +func (l *b2Objects) DeleteObject(ctx context.Context, bucket string, object string) error { + bkt, err := l.Bucket(ctx, bucket) if err != nil { return err } @@ -533,11 +533,11 @@ func (l *b2Objects) DeleteObject(bucket string, object string) error { } // ListMultipartUploads lists all multipart uploads. -func (l *b2Objects) ListMultipartUploads(bucket string, prefix string, keyMarker string, uploadIDMarker string, +func (l *b2Objects) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, err error) { // keyMarker, prefix, delimiter are all ignored, Backblaze B2 doesn't support any // of these parameters only equivalent parameter is uploadIDMarker. - bkt, err := l.Bucket(bucket) + bkt, err := l.Bucket(ctx, bucket) if err != nil { return lmi, err } @@ -572,9 +572,9 @@ func (l *b2Objects) ListMultipartUploads(bucket string, prefix string, keyMarker // Each large file must consist of at least 2 parts, and all of the parts except the // last one must be at least 5MB in size. The last part must contain at least one byte. // For more information - https://www.backblaze.com/b2/docs/large_files.html -func (l *b2Objects) NewMultipartUpload(bucket string, object string, metadata map[string]string) (string, error) { +func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string) (string, error) { var uploadID string - bkt, err := l.Bucket(bucket) + bkt, err := l.Bucket(ctx, bucket) if err != nil { return uploadID, err } @@ -590,8 +590,8 @@ func (l *b2Objects) NewMultipartUpload(bucket string, object string, metadata ma } // PutObjectPart puts a part of object in bucket, uses B2's LargeFile upload API. -func (l *b2Objects) PutObjectPart(bucket string, object string, uploadID string, partID int, data *h2.Reader) (pi minio.PartInfo, err error) { - bkt, err := l.Bucket(bucket) +func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *h2.Reader) (pi minio.PartInfo, err error) { + bkt, err := l.Bucket(ctx, bucket) if err != nil { return pi, err } @@ -616,8 +616,8 @@ func (l *b2Objects) PutObjectPart(bucket string, object string, uploadID string, } // ListObjectParts returns all object parts for specified object in specified bucket, uses B2's LargeFile upload API. -func (l *b2Objects) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi minio.ListPartsInfo, err error) { - bkt, err := l.Bucket(bucket) +func (l *b2Objects) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi minio.ListPartsInfo, err error) { + bkt, err := l.Bucket(ctx, bucket) if err != nil { return lpi, err } @@ -649,8 +649,8 @@ func (l *b2Objects) ListObjectParts(bucket string, object string, uploadID strin } // AbortMultipartUpload aborts a on going multipart upload, uses B2's LargeFile upload API. -func (l *b2Objects) AbortMultipartUpload(bucket string, object string, uploadID string) error { - bkt, err := l.Bucket(bucket) +func (l *b2Objects) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string) error { + bkt, err := l.Bucket(ctx, bucket) if err != nil { return err } @@ -659,8 +659,8 @@ func (l *b2Objects) AbortMultipartUpload(bucket string, object string, uploadID } // CompleteMultipartUpload completes ongoing multipart upload and finalizes object, uses B2's LargeFile upload API. -func (l *b2Objects) CompleteMultipartUpload(bucket string, object string, uploadID string, uploadedParts []minio.CompletePart) (oi minio.ObjectInfo, err error) { - bkt, err := l.Bucket(bucket) +func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart) (oi minio.ObjectInfo, err error) { + bkt, err := l.Bucket(ctx, bucket) if err != nil { return oi, err } @@ -680,14 +680,14 @@ func (l *b2Objects) CompleteMultipartUpload(bucket string, object string, upload return oi, b2ToObjectError(errors.Trace(err), bucket, object, uploadID) } - return l.GetObjectInfo(bucket, object) + return l.GetObjectInfo(ctx, bucket, object) } // SetBucketPolicy - B2 supports 2 types of bucket policies: // bucketType.AllPublic - bucketTypeReadOnly means that anybody can download the files is the bucket; // bucketType.AllPrivate - bucketTypePrivate means that you need an authorization token to download them. // Default is AllPrivate for all buckets. -func (l *b2Objects) SetBucketPolicy(bucket string, policyInfo policy.BucketAccessPolicy) error { +func (l *b2Objects) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error { var policies []minio.BucketAccessPolicy for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket, "") { @@ -706,7 +706,7 @@ func (l *b2Objects) SetBucketPolicy(bucket string, policyInfo policy.BucketAcces if policies[0].Policy != policy.BucketPolicyReadOnly { return errors.Trace(minio.NotImplemented{}) } - bkt, err := l.Bucket(bucket) + bkt, err := l.Bucket(ctx, bucket) if err != nil { return err } @@ -717,9 +717,9 @@ func (l *b2Objects) SetBucketPolicy(bucket string, policyInfo policy.BucketAcces // GetBucketPolicy, returns the current bucketType from B2 backend and convert // it into S3 compatible bucket policy info. -func (l *b2Objects) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, error) { +func (l *b2Objects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) { policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"} - bkt, err := l.Bucket(bucket) + bkt, err := l.Bucket(ctx, bucket) if err != nil { return policyInfo, err } @@ -734,8 +734,8 @@ func (l *b2Objects) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, e } // DeleteBucketPolicy - resets the bucketType of bucket on B2 to 'allPrivate'. -func (l *b2Objects) DeleteBucketPolicy(bucket string) error { - bkt, err := l.Bucket(bucket) +func (l *b2Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error { + bkt, err := l.Bucket(ctx, bucket) if err != nil { return err } diff --git a/cmd/gateway/gcs/gateway-gcs.go b/cmd/gateway/gcs/gateway-gcs.go index 535ae28bc..208767be2 100644 --- a/cmd/gateway/gcs/gateway-gcs.go +++ b/cmd/gateway/gcs/gateway-gcs.go @@ -401,17 +401,17 @@ func (l *gcsGateway) CleanupGCSMinioSysTmp() { // Shutdown - save any gateway metadata to disk // if necessary and reload upon next restart. -func (l *gcsGateway) Shutdown() error { +func (l *gcsGateway) Shutdown(ctx context.Context) error { return nil } // StorageInfo - Not relevant to GCS backend. -func (l *gcsGateway) StorageInfo() minio.StorageInfo { +func (l *gcsGateway) StorageInfo(ctx context.Context) minio.StorageInfo { return minio.StorageInfo{} } // MakeBucketWithLocation - Create a new container on GCS backend. -func (l *gcsGateway) MakeBucketWithLocation(bucket, location string) error { +func (l *gcsGateway) MakeBucketWithLocation(ctx context.Context, bucket, location string) error { bkt := l.client.Bucket(bucket) // we'll default to the us multi-region in case of us-east-1 @@ -427,7 +427,7 @@ func (l *gcsGateway) MakeBucketWithLocation(bucket, location string) error { } // GetBucketInfo - Get bucket metadata.. -func (l *gcsGateway) GetBucketInfo(bucket string) (minio.BucketInfo, error) { +func (l *gcsGateway) GetBucketInfo(ctx context.Context, bucket string) (minio.BucketInfo, error) { attrs, err := l.client.Bucket(bucket).Attrs(l.ctx) if err != nil { return minio.BucketInfo{}, gcsToObjectError(errors.Trace(err), bucket) @@ -440,7 +440,7 @@ func (l *gcsGateway) GetBucketInfo(bucket string) (minio.BucketInfo, error) { } // ListBuckets lists all buckets under your project-id on GCS. -func (l *gcsGateway) ListBuckets() (buckets []minio.BucketInfo, err error) { +func (l *gcsGateway) ListBuckets(ctx context.Context) (buckets []minio.BucketInfo, err error) { it := l.client.Buckets(l.ctx, l.projectID) // Iterate and capture all the buckets. @@ -464,7 +464,7 @@ func (l *gcsGateway) ListBuckets() (buckets []minio.BucketInfo, err error) { } // DeleteBucket delete a bucket on GCS. -func (l *gcsGateway) DeleteBucket(bucket string) error { +func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error { itObject := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{ Delimiter: "/", Versions: false, @@ -537,7 +537,7 @@ func isGCSMarker(marker string) bool { } // ListObjects - lists all blobs in GCS bucket filtered by prefix -func (l *gcsGateway) ListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (minio.ListObjectsInfo, error) { +func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (minio.ListObjectsInfo, error) { it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{ Delimiter: delimiter, Prefix: prefix, @@ -640,7 +640,7 @@ func (l *gcsGateway) ListObjects(bucket string, prefix string, marker string, de } // ListObjectsV2 - lists all blobs in GCS bucket filtered by prefix -func (l *gcsGateway) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (minio.ListObjectsV2Info, error) { +func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (minio.ListObjectsV2Info, error) { it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{ Delimiter: delimiter, Prefix: prefix, @@ -713,7 +713,7 @@ func (l *gcsGateway) ListObjectsV2(bucket, prefix, continuationToken, delimiter // // startOffset indicates the starting read location of the object. // length indicates the total length of the object. -func (l *gcsGateway) GetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string) error { +func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string) error { // if we want to mimic S3 behavior exactly, we need to verify if bucket exists first, // otherwise gcs will just return object not exist in case of non-existing bucket if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil { @@ -751,7 +751,7 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo { } // GetObjectInfo - reads object info and replies back ObjectInfo -func (l *gcsGateway) GetObjectInfo(bucket string, object string) (minio.ObjectInfo, error) { +func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object string) (minio.ObjectInfo, error) { // if we want to mimic S3 behavior exactly, we need to verify if bucket exists first, // otherwise gcs will just return object not exist in case of non-existing bucket if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil { @@ -767,7 +767,7 @@ func (l *gcsGateway) GetObjectInfo(bucket string, object string) (minio.ObjectIn } // PutObject - Create a new object with the incoming data, -func (l *gcsGateway) PutObject(bucket string, key string, data *hash.Reader, metadata map[string]string) (minio.ObjectInfo, error) { +func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, data *hash.Reader, metadata map[string]string) (minio.ObjectInfo, error) { // if we want to mimic S3 behavior exactly, we need to verify if bucket exists first, // otherwise gcs will just return object not exist in case of non-existing bucket if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil { @@ -800,7 +800,7 @@ func (l *gcsGateway) PutObject(bucket string, key string, data *hash.Reader, met } // CopyObject - Copies a blob from source container to destination container. -func (l *gcsGateway) CopyObject(srcBucket string, srcObject string, destBucket string, destObject string, +func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject string, destBucket string, destObject string, srcInfo minio.ObjectInfo) (minio.ObjectInfo, error) { src := l.client.Bucket(srcBucket).Object(srcObject) @@ -818,7 +818,7 @@ func (l *gcsGateway) CopyObject(srcBucket string, srcObject string, destBucket s } // DeleteObject - Deletes a blob in bucket -func (l *gcsGateway) DeleteObject(bucket string, object string) error { +func (l *gcsGateway) DeleteObject(ctx context.Context, bucket string, object string) error { err := l.client.Bucket(bucket).Object(object).Delete(l.ctx) if err != nil { return gcsToObjectError(errors.Trace(err), bucket, object) @@ -828,7 +828,7 @@ func (l *gcsGateway) DeleteObject(bucket string, object string) error { } // NewMultipartUpload - upload object in multiple parts -func (l *gcsGateway) NewMultipartUpload(bucket string, key string, metadata map[string]string) (uploadID string, err error) { +func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key string, metadata map[string]string) (uploadID string, err error) { // generate new uploadid uploadID = minio.MustGetUUID() @@ -853,7 +853,7 @@ func (l *gcsGateway) NewMultipartUpload(bucket string, key string, metadata map[ } // ListMultipartUploads - lists all multipart uploads. -func (l *gcsGateway) ListMultipartUploads(bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (minio.ListMultipartsInfo, error) { +func (l *gcsGateway) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (minio.ListMultipartsInfo, error) { return minio.ListMultipartsInfo{ KeyMarker: keyMarker, UploadIDMarker: uploadIDMarker, @@ -865,14 +865,14 @@ func (l *gcsGateway) ListMultipartUploads(bucket string, prefix string, keyMarke // Checks if minio.sys.tmp/multipart/v1//gcs.json exists, returns // an object layer compatible error upon any error. -func (l *gcsGateway) checkUploadIDExists(bucket string, key string, uploadID string) error { +func (l *gcsGateway) checkUploadIDExists(ctx context.Context, bucket string, key string, uploadID string) error { _, err := l.client.Bucket(bucket).Object(gcsMultipartMetaName(uploadID)).Attrs(l.ctx) return gcsToObjectError(errors.Trace(err), bucket, key, uploadID) } // PutObjectPart puts a part of object in bucket -func (l *gcsGateway) PutObjectPart(bucket string, key string, uploadID string, partNumber int, data *hash.Reader) (minio.PartInfo, error) { - if err := l.checkUploadIDExists(bucket, key, uploadID); err != nil { +func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key string, uploadID string, partNumber int, data *hash.Reader) (minio.PartInfo, error) { + if err := l.checkUploadIDExists(ctx, bucket, key, uploadID); err != nil { return minio.PartInfo{}, err } etag := data.MD5HexString() @@ -902,8 +902,8 @@ func (l *gcsGateway) PutObjectPart(bucket string, key string, uploadID string, p } // ListObjectParts returns all object parts for specified object in specified bucket -func (l *gcsGateway) ListObjectParts(bucket string, key string, uploadID string, partNumberMarker int, maxParts int) (minio.ListPartsInfo, error) { - return minio.ListPartsInfo{}, l.checkUploadIDExists(bucket, key, uploadID) +func (l *gcsGateway) ListObjectParts(ctx context.Context, bucket string, key string, uploadID string, partNumberMarker int, maxParts int) (minio.ListPartsInfo, error) { + return minio.ListPartsInfo{}, l.checkUploadIDExists(ctx, bucket, key, uploadID) } // Called by AbortMultipartUpload and CompleteMultipartUpload for cleaning up. @@ -931,8 +931,8 @@ func (l *gcsGateway) cleanupMultipartUpload(bucket, key, uploadID string) error } // AbortMultipartUpload aborts a ongoing multipart upload -func (l *gcsGateway) AbortMultipartUpload(bucket string, key string, uploadID string) error { - if err := l.checkUploadIDExists(bucket, key, uploadID); err != nil { +func (l *gcsGateway) AbortMultipartUpload(ctx context.Context, bucket string, key string, uploadID string) error { + if err := l.checkUploadIDExists(ctx, bucket, key, uploadID); err != nil { return err } return l.cleanupMultipartUpload(bucket, key, uploadID) @@ -946,7 +946,7 @@ func (l *gcsGateway) AbortMultipartUpload(bucket string, key string, uploadID st // to the number of components you can compose per second. This rate counts both the // components being appended to a composite object as well as the components being // copied when the composite object of which they are a part is copied. -func (l *gcsGateway) CompleteMultipartUpload(bucket string, key string, uploadID string, uploadedParts []minio.CompletePart) (minio.ObjectInfo, error) { +func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string, key string, uploadID string, uploadedParts []minio.CompletePart) (minio.ObjectInfo, error) { meta := gcsMultipartMetaName(uploadID) object := l.client.Bucket(bucket).Object(meta) @@ -1046,7 +1046,7 @@ func (l *gcsGateway) CompleteMultipartUpload(bucket string, key string, uploadID } // SetBucketPolicy - Set policy on bucket -func (l *gcsGateway) SetBucketPolicy(bucket string, policyInfo policy.BucketAccessPolicy) error { +func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error { var policies []minio.BucketAccessPolicy for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket, "") { @@ -1091,7 +1091,7 @@ func (l *gcsGateway) SetBucketPolicy(bucket string, policyInfo policy.BucketAcce } // GetBucketPolicy - Get policy on bucket -func (l *gcsGateway) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, error) { +func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) { rules, err := l.client.Bucket(bucket).ACL().List(l.ctx) if err != nil { return policy.BucketAccessPolicy{}, gcsToObjectError(errors.Trace(err), bucket) @@ -1116,7 +1116,7 @@ func (l *gcsGateway) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, } // DeleteBucketPolicy - Delete all policies on bucket -func (l *gcsGateway) DeleteBucketPolicy(bucket string) error { +func (l *gcsGateway) DeleteBucketPolicy(ctx context.Context, bucket string) error { // This only removes the storage.AllUsers policies if err := l.client.Bucket(bucket).ACL().Delete(l.ctx, storage.AllUsers); err != nil { return gcsToObjectError(errors.Trace(err), bucket) diff --git a/cmd/gateway/manta/gateway-manta.go b/cmd/gateway/manta/gateway-manta.go index 88e0f5f3f..94af73ed1 100644 --- a/cmd/gateway/manta/gateway-manta.go +++ b/cmd/gateway/manta/gateway-manta.go @@ -220,12 +220,12 @@ type tritonObjects struct { // Shutdown - save any gateway metadata to disk // if necessary and reload upon next restart. -func (t *tritonObjects) Shutdown() error { +func (t *tritonObjects) Shutdown(ctx context.Context) error { return nil } // StorageInfo - Not relevant to Triton backend. -func (t *tritonObjects) StorageInfo() (si minio.StorageInfo) { +func (t *tritonObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo) { return si } @@ -236,8 +236,7 @@ func (t *tritonObjects) StorageInfo() (si minio.StorageInfo) { // MakeBucketWithLocation - Create a new directory within manta. // // https://apidocs.joyent.com/manta/api.html#PutDirectory -func (t *tritonObjects) MakeBucketWithLocation(bucket, location string) error { - ctx := context.Background() +func (t *tritonObjects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error { err := t.client.Dir().Put(ctx, &storage.PutDirectoryInput{ DirectoryName: path.Join(mantaRoot, bucket), }) @@ -250,9 +249,8 @@ func (t *tritonObjects) MakeBucketWithLocation(bucket, location string) error { // GetBucketInfo - Get directory metadata.. // // https://apidocs.joyent.com/manta/api.html#GetObject -func (t *tritonObjects) GetBucketInfo(bucket string) (bi minio.BucketInfo, e error) { +func (t *tritonObjects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, e error) { var info minio.BucketInfo - ctx := context.Background() resp, err := t.client.Objects().Get(ctx, &storage.GetObjectInput{ ObjectPath: path.Join(mantaRoot, bucket), }) @@ -270,8 +268,7 @@ func (t *tritonObjects) GetBucketInfo(bucket string) (bi minio.BucketInfo, e err // ListDirectories. // // https://apidocs.joyent.com/manta/api.html#ListDirectory -func (t *tritonObjects) ListBuckets() (buckets []minio.BucketInfo, err error) { - ctx := context.Background() +func (t *tritonObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketInfo, err error) { dirs, err := t.client.Dir().List(ctx, &storage.ListDirectoryInput{ DirectoryName: path.Join(mantaRoot), }) @@ -295,8 +292,7 @@ func (t *tritonObjects) ListBuckets() (buckets []minio.BucketInfo, err error) { // DeleteDirectory. // // https://apidocs.joyent.com/manta/api.html#DeleteDirectory -func (t *tritonObjects) DeleteBucket(bucket string) error { - ctx := context.Background() +func (t *tritonObjects) DeleteBucket(ctx context.Context, bucket string) error { return t.client.Dir().Delete(ctx, &storage.DeleteDirectoryInput{ DirectoryName: path.Join(mantaRoot, bucket), }) @@ -310,13 +306,12 @@ func (t *tritonObjects) DeleteBucket(bucket string) error { // and marker, uses Manta equivalent ListDirectory. // // https://apidocs.joyent.com/manta/api.html#ListDirectory -func (t *tritonObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result minio.ListObjectsInfo, err error) { +func (t *tritonObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result minio.ListObjectsInfo, err error) { var ( dirName string objs *storage.ListDirectoryOutput input *storage.ListDirectoryInput - ctx = context.Background() pathBase = path.Base(prefix) ) @@ -393,13 +388,12 @@ func (t *tritonObjects) ListObjects(bucket, prefix, marker, delimiter string, ma // and continuationToken, uses Manta equivalent ListDirectory. // // https://apidocs.joyent.com/manta/api.html#ListDirectory -func (t *tritonObjects) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result minio.ListObjectsV2Info, err error) { +func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result minio.ListObjectsV2Info, err error) { var ( dirName string objs *storage.ListDirectoryOutput input *storage.ListDirectoryInput - ctx = context.Background() pathBase = path.Base(prefix) ) @@ -467,13 +461,12 @@ func (t *tritonObjects) ListObjectsV2(bucket, prefix, continuationToken, delimit // indicates the total length of the object. // // https://apidocs.joyent.com/manta/api.html#GetObject -func (t *tritonObjects) GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error { +func (t *tritonObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error { // Start offset cannot be negative. if startOffset < 0 { return errors.Trace(fmt.Errorf("Unexpected error")) } - ctx := context.Background() output, err := t.client.Objects().Get(ctx, &storage.GetObjectInput{ ObjectPath: path.Join(mantaRoot, bucket, object), }) @@ -499,8 +492,7 @@ func (t *tritonObjects) GetObject(bucket, object string, startOffset int64, leng // uses Triton equivalent GetBlobProperties. // // https://apidocs.joyent.com/manta/api.html#GetObject -func (t *tritonObjects) GetObjectInfo(bucket, object string) (objInfo minio.ObjectInfo, err error) { - ctx := context.Background() +func (t *tritonObjects) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo minio.ObjectInfo, err error) { info, err := t.client.Objects().GetInfo(ctx, &storage.GetInfoInput{ ObjectPath: path.Join(mantaRoot, bucket, object), }) @@ -538,8 +530,7 @@ func (d dummySeeker) Seek(offset int64, whence int) (int64, error) { // CreateBlockBlobFromReader. // // https://apidocs.joyent.com/manta/api.html#PutObject -func (t *tritonObjects) PutObject(bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { - ctx := context.Background() +func (t *tritonObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { if err = t.client.Objects().Put(ctx, &storage.PutObjectInput{ ContentLength: uint64(data.Size()), ObjectPath: path.Join(mantaRoot, bucket, object), @@ -552,19 +543,18 @@ func (t *tritonObjects) PutObject(bucket, object string, data *hash.Reader, meta return objInfo, errors.Trace(err) } if err = data.Verify(); err != nil { - t.DeleteObject(bucket, object) + t.DeleteObject(ctx, bucket, object) return objInfo, errors.Trace(err) } - return t.GetObjectInfo(bucket, object) + return t.GetObjectInfo(ctx, bucket, object) } // CopyObject - Copies a blob from source container to destination container. // Uses Manta Snaplinks API. // // https://apidocs.joyent.com/manta/api.html#PutSnapLink -func (t *tritonObjects) CopyObject(srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) { - ctx := context.Background() +func (t *tritonObjects) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) { if err = t.client.SnapLinks().Put(ctx, &storage.PutSnapLinkInput{ SourcePath: path.Join(mantaRoot, srcBucket, srcObject), LinkPath: path.Join(mantaRoot, destBucket, destObject), @@ -572,14 +562,13 @@ func (t *tritonObjects) CopyObject(srcBucket, srcObject, destBucket, destObject return objInfo, errors.Trace(err) } - return t.GetObjectInfo(destBucket, destObject) + return t.GetObjectInfo(ctx, destBucket, destObject) } // DeleteObject - Delete a blob in Manta, uses Triton equivalent DeleteBlob API. // // https://apidocs.joyent.com/manta/api.html#DeleteObject -func (t *tritonObjects) DeleteObject(bucket, object string) error { - ctx := context.Background() +func (t *tritonObjects) DeleteObject(ctx context.Context, bucket, object string) error { if err := t.client.Objects().Delete(ctx, &storage.DeleteObjectInput{ ObjectPath: path.Join(mantaRoot, bucket, object), }); err != nil {