package filestate import ( "context" "io" "path" "path/filepath" "github.com/pkg/errors" "github.com/pulumi/pulumi/pkg/util/logging" "gocloud.dev/blob" ) // Bucket is a wrapper around an underlying gocloud blob.Bucket. It ensures that we pass all paths // to it normalized to forward-slash form like it requires. type Bucket interface { Copy(ctx context.Context, dstKey, srcKey string, opts *blob.CopyOptions) (err error) Delete(ctx context.Context, key string) (err error) List(opts *blob.ListOptions) *blob.ListIterator SignedURL(ctx context.Context, key string, opts *blob.SignedURLOptions) (string, error) ReadAll(ctx context.Context, key string) (_ []byte, err error) WriteAll(ctx context.Context, key string, p []byte, opts *blob.WriterOptions) (err error) Exists(ctx context.Context, key string) (bool, error) } // wrappedBucket encapsulates a true gocloud blob.Bucket, but ensures that all paths we send to it // are appropriately normalized to use forward slashes as required by it. Without this, we may use // filepath.join which can make paths like `c:\temp\etc`. gocloud's fileblob then converts those // backslashes to the hex string __0x5c__, breaking things on windows completely. type wrappedBucket struct { bucket *blob.Bucket } func (b *wrappedBucket) Copy(ctx context.Context, dstKey, srcKey string, opts *blob.CopyOptions) (err error) { return b.bucket.Copy(ctx, filepath.ToSlash(dstKey), filepath.ToSlash(srcKey), opts) } func (b *wrappedBucket) Delete(ctx context.Context, key string) (err error) { return b.bucket.Delete(ctx, filepath.ToSlash(key)) } func (b *wrappedBucket) List(opts *blob.ListOptions) *blob.ListIterator { return b.bucket.List(opts) } func (b *wrappedBucket) SignedURL(ctx context.Context, key string, opts *blob.SignedURLOptions) (string, error) { return b.bucket.SignedURL(ctx, filepath.ToSlash(key), opts) } func (b *wrappedBucket) ReadAll(ctx context.Context, key string) (_ []byte, err error) { return b.bucket.ReadAll(ctx, filepath.ToSlash(key)) } func (b *wrappedBucket) WriteAll(ctx context.Context, key string, p []byte, opts *blob.WriterOptions) (err error) { return b.bucket.WriteAll(ctx, filepath.ToSlash(key), p, opts) } func (b *wrappedBucket) Exists(ctx context.Context, key string) (bool, error) { return b.bucket.Exists(ctx, filepath.ToSlash(key)) } // listBucket returns a list of all files in the bucket within a given directory. go-cloud sorts the results by key func listBucket(bucket Bucket, dir string) ([]*blob.ListObject, error) { bucketIter := bucket.List(&blob.ListOptions{ Delimiter: "/", Prefix: dir + "/", }) files := []*blob.ListObject{} ctx := context.TODO() for { file, err := bucketIter.Next(ctx) if err == io.EOF { break } if err != nil { return nil, errors.Wrap(err, "could not list bucket") } files = append(files, file) } return files, nil } // objectName returns the filename of a ListObject (an object from a bucket) func objectName(obj *blob.ListObject) string { _, filename := path.Split(obj.Key) return filename } // removeAllByPrefix deletes all objects with a given prefix (i.e. filepath) func removeAllByPrefix(bucket Bucket, dir string) error { files, err := listBucket(bucket, dir) if err != nil { return errors.Wrap(err, "unable to list bucket objects for removal") } for _, file := range files { err = bucket.Delete(context.TODO(), file.Key) if err != nil { logging.V(5).Infof("error deleting object: %v (%v) skipping", file.Key, err) } } return nil } // renameObject renames an object in a bucket. the rename requires a download/upload of the object // due to a go-cloud API limitation func renameObject(bucket Bucket, source string, dest string) error { err := bucket.Copy(context.TODO(), dest, source, nil) if err != nil { return errors.Wrapf(err, "copying %s to %s", source, dest) } err = bucket.Delete(context.TODO(), source) if err != nil { logging.V(5).Infof("error deleting source object after rename: %v (%v) skipping", source, err) } return nil }