maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Switch to kurin/blazer from minio/blazer fork for b2 gateway (#7879)

Browse source
This commit is contained in:
mzukowski-reef 2019-07-09 17:14:02 +02:00 committed by Harshavardhana
parent 8e09374cb8
commit 9d49688c87
4 changed files with 131 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
cmd/gateway/b2/gateway-b2.go
View file

@ -29,7 +29,7 @@ import (
"sync"
"time"
b2 "github.com/minio/blazer/base"
b2 "github.com/kurin/blazer/base"
"github.com/minio/cli"
miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
"github.com/minio/minio/cmd/logger"
@ -147,6 +147,29 @@ func b2ToObjectError(err error, params ...string) error {
if err == nil {
return nil
}
code, msgCode, msg := b2.MsgCode(err)
if code == 0 {
// We don't interpret non B2 errors. B2 errors have statusCode
// to help us convert them to S3 object errors.
return err
}
objErr := b2MsgCodeToObjectError(code, msgCode, msg, params...)
if objErr == nil {
return err
}
return objErr
}
func b2MsgCodeToObjectError(code int, msgCode string, msg string, params ...string) error {
// Following code is a non-exhaustive check to convert
// B2 errors into S3 compatible errors.
//
// For a more complete information - https://www.backblaze.com/b2/docs/
var err error
bucket := ""
object := ""
uploadID := ""
@ -160,18 +183,7 @@ func b2ToObjectError(err error, params ...string) error {
uploadID = params[2]
}
// Following code is a non-exhaustive check to convert
// B2 errors into S3 compatible errors.
//
// For a more complete information - https://www.backblaze.com/b2/docs/
statusCode, code, msg := b2.Code(err)
if statusCode == 0 {
// We don't interpret non B2 errors. B2 errors have statusCode
// to help us convert them to S3 object errors.
return err
}
switch code {
switch msgCode {
case "duplicate_bucket_name":
err = minio.BucketAlreadyOwnedByYou{Bucket: bucket}
case "bad_request":
@ -343,7 +355,7 @@ func (l *b2Objects) ListObjects(ctx context.Context, bucket string, prefix strin
Name: file.Name,
ModTime: file.Timestamp,
Size: file.Size,
ETag: minio.ToS3ETag(file.Info.ID),
ETag: minio.ToS3ETag(file.ID),
ContentType: file.Info.ContentType,
UserDefined: file.Info.Info,
})
@ -386,7 +398,7 @@ func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat
Name: file.Name,
ModTime: file.Timestamp,
Size: file.Size,
ETag: minio.ToS3ETag(file.Info.ID),
ETag: minio.ToS3ETag(file.ID),
ContentType: file.Info.ContentType,
UserDefined: file.Info.Info,
})
@ -462,7 +474,7 @@ func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object str
return minio.ObjectInfo{
Bucket: bucket,
Name: object,
ETag: minio.ToS3ETag(fi.ID),
ETag: minio.ToS3ETag(f.ID),
Size: fi.Size,
ModTime: fi.Timestamp,
ContentType: fi.ContentType,
@ -569,7 +581,7 @@ func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string,
return minio.ObjectInfo{
Bucket: bucket,
Name: object,
ETag: minio.ToS3ETag(fi.ID),
ETag: minio.ToS3ETag(f.ID),
Size: fi.Size,
ModTime: fi.Timestamp,
ContentType: fi.ContentType,
@ -617,7 +629,7 @@ func (l *b2Objects) ListMultipartUploads(ctx context.Context, bucket string, pre
if maxUploads > 100 {
maxUploads = 100
}
largeFiles, nextMarker, err := bkt.ListUnfinishedLargeFiles(l.ctx, uploadIDMarker, maxUploads)
largeFiles, nextMarker, err := bkt.ListUnfinishedLargeFiles(l.ctx, maxUploads, uploadIDMarker)
if err != nil {
logger.LogIf(ctx, err)
return lmi, b2ToObjectError(err, bucket)
@ -677,7 +689,7 @@ func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object str
}
hr := newB2Reader(data, data.Size())
sha1, err := fc.UploadPart(l.ctx, hr, sha1AtEOF, int(hr.Size()), partID)
_, err = fc.UploadPart(l.ctx, hr, sha1AtEOF, int(hr.Size()), partID)
if err != nil {
logger.LogIf(ctx, err)
return pi, b2ToObjectError(err, bucket, object, uploadID)
@ -686,7 +698,7 @@ func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object str
return minio.PartInfo{
PartNumber: partID,
LastModified: minio.UTCNow(),
ETag: minio.ToS3ETag(sha1),
ETag: minio.ToS3ETag(fmt.Sprintf("%x", hr.sha1Hash.Sum(nil))),
Size: data.Size(),
}, nil
}

161
cmd/gateway/b2/gateway-b2_test.go
View file

@ -20,8 +20,6 @@ import (
"fmt"
"testing"
b2 "github.com/minio/blazer/base"
minio "github.com/minio/minio/cmd"
)
@ -41,71 +39,6 @@ func TestB2ObjectError(t *testing.T) {
{
[]string{}, fmt.Errorf("Non B2 Error"), fmt.Errorf("Non B2 Error"),
},
{
[]string{"bucket"}, b2.Error{
StatusCode: 1,
Code: "duplicate_bucket_name",
}, minio.BucketAlreadyOwnedByYou{
Bucket: "bucket",
},
},
{
[]string{"bucket"}, b2.Error{
StatusCode: 1,
Code: "bad_request",
}, minio.BucketNotFound{
Bucket: "bucket",
},
},
{
[]string{"bucket", "object"}, b2.Error{
StatusCode: 1,
Code: "bad_request",
}, minio.ObjectNameInvalid{
Bucket: "bucket",
Object: "object",
},
},
{
[]string{"bucket"}, b2.Error{
StatusCode: 1,
Code: "bad_bucket_id",
}, minio.BucketNotFound{Bucket: "bucket"},
},
{
[]string{"bucket", "object"}, b2.Error{
StatusCode: 1,
Code: "file_not_present",
}, minio.ObjectNotFound{
Bucket: "bucket",
Object: "object",
},
},
{
[]string{"bucket", "object"}, b2.Error{
StatusCode: 1,
Code: "not_found",
}, minio.ObjectNotFound{
Bucket: "bucket",
Object: "object",
},
},
{
[]string{"bucket"}, b2.Error{
StatusCode: 1,
Code: "cannot_delete_non_empty_bucket",
}, minio.BucketNotEmpty{
Bucket: "bucket",
},
},
{
[]string{"bucket", "object", "uploadID"}, b2.Error{
StatusCode: 1,
Message: "No active upload for",
}, minio.InvalidUploadID{
UploadID: "uploadID",
},
},
}
for i, testCase := range testCases {
@ -117,3 +50,97 @@ func TestB2ObjectError(t *testing.T) {
}
}
}
// Test b2 msg code to object error.
func TestB2MsgCodeToObjectError(t *testing.T) {
testCases := []struct {
params []string
code int
msgCode string
msg string
expectedErr error
}{
{
[]string{"bucket"},
1,
"duplicate_bucket_name",
"",
minio.BucketAlreadyOwnedByYou{
Bucket: "bucket",
},
},
{
[]string{"bucket"},
1,
"bad_request",
"",
minio.BucketNotFound{
Bucket: "bucket",
},
},
{
[]string{"bucket", "object"},
1,
"bad_request",
"",
minio.ObjectNameInvalid{
Bucket: "bucket",
Object: "object",
},
},
{
[]string{"bucket"},
1,
"bad_bucket_id",
"",
minio.BucketNotFound{Bucket: "bucket"},
},
{
[]string{"bucket", "object"},
1,
"file_not_present",
"",
minio.ObjectNotFound{
Bucket: "bucket",
Object: "object",
},
},
{
[]string{"bucket", "object"},
1,
"not_found",
"",
minio.ObjectNotFound{
Bucket: "bucket",
Object: "object",
},
},
{
[]string{"bucket"},
1,
"cannot_delete_non_empty_bucket",
"",
minio.BucketNotEmpty{
Bucket: "bucket",
},
},
{
[]string{"bucket", "object", "uploadID"},
1,
"",
"No active upload for",
minio.InvalidUploadID{
UploadID: "uploadID",
},
},
}
for i, testCase := range testCases {
actualErr := b2MsgCodeToObjectError(testCase.code, testCase.msgCode, testCase.msg, testCase.params...)
if actualErr != nil {
if actualErr.Error() != testCase.expectedErr.Error() {
t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.expectedErr, actualErr)
}
}
}
}

2
go.mod
View file

@ -47,11 +47,11 @@ require (
github.com/klauspost/pgzip v1.2.1
github.com/klauspost/readahead v1.3.0
github.com/klauspost/reedsolomon v1.9.1
github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3
github.com/lib/pq v1.0.0
github.com/marstr/guid v1.1.0 // indirect
github.com/mattn/go-isatty v0.0.7
github.com/miekg/dns v1.1.8
github.com/minio/blazer v0.0.0-20171126203752-2081f5bf0465
github.com/minio/cli v1.20.0
github.com/minio/dsync/v2 v2.0.0
github.com/minio/hdfs/v3 v3.0.0

4
go.sum
View file

@ -376,6 +376,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kurin/blazer v0.5.3 h1:SAgYv0TKU0kN/ETfO5ExjNAPyMt2FocO2s/UlCHfjAk=
github.com/kurin/blazer v0.5.3/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3 h1:1sl2HmNtqGnDuydLgCJwZIpDLGqZOdwOkcY8WtUl8Cw=
github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
github.com/lib/pq v0.0.0-20181016162627-9eb73efc1fcc/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=