Merge pull request #1072 from harshavardhana/query

vendor: Update minio-go library with fixes for object listing.
This commit is contained in:
Harshavardhana 2016-02-02 14:56:49 -08:00
commit b01594ac33
14 changed files with 297 additions and 62 deletions

View file

@ -33,10 +33,10 @@ type ObjectInfo struct {
// each parts concatenated into one string.
ETag string `json:"etag"`
Key string `json:"name"` // Name of the object
Key string `json:"name"` // Name of the object
LastModified time.Time `json:"lastModified"` // Date and time the object was last modified.
Size int64 `json:"size"` // Size in bytes of the object.
ContentType string `json:"contentType"` // A standard MIME type describing the format of the object data.
Size int64 `json:"size"` // Size in bytes of the object.
ContentType string `json:"contentType"` // A standard MIME type describing the format of the object data.
// Owner name.
Owner struct {

View file

@ -188,11 +188,11 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit
urlValues := make(url.Values)
// Set object prefix.
if objectPrefix != "" {
urlValues.Set("prefix", urlEncodePath(objectPrefix))
urlValues.Set("prefix", objectPrefix)
}
// Set object marker.
if objectMarker != "" {
urlValues.Set("marker", urlEncodePath(objectMarker))
urlValues.Set("marker", objectMarker)
}
// Set delimiter.
if delimiter != "" {
@ -366,7 +366,7 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker,
urlValues.Set("uploads", "")
// Set object key marker.
if keyMarker != "" {
urlValues.Set("key-marker", urlEncodePath(keyMarker))
urlValues.Set("key-marker", keyMarker)
}
// Set upload id marker.
if uploadIDMarker != "" {
@ -374,7 +374,7 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker,
}
// Set prefix marker.
if prefix != "" {
urlValues.Set("prefix", urlEncodePath(prefix))
urlValues.Set("prefix", prefix)
}
// Set delimiter.
if delimiter != "" {

View file

@ -91,19 +91,14 @@ func (c Client) makeBucketRequest(bucketName string, acl BucketACL, location str
return nil, ErrInvalidArgument("Unrecognized ACL " + acl.String())
}
// Set get bucket location always as path style.
// In case of Amazon S3. The make bucket issued on already
// existing bucket would fail with 'AuthorizationMalformed' error
// if virtual style is used. So we default to 'path style' as that
// is the preferred method here. The final location of the
// 'bucket' is provided through XML LocationConstraint data with
// the request.
targetURL := *c.endpointURL
if bucketName != "" {
// If endpoint supports virtual host style use that always.
// Currently only S3 and Google Cloud Storage would support this.
if isVirtualHostSupported(c.endpointURL, bucketName) {
targetURL.Host = bucketName + "." + c.endpointURL.Host
targetURL.Path = "/"
} else {
// If not fall back to using path style.
targetURL.Path = "/" + bucketName
}
}
targetURL.Path = "/" + bucketName + "/"
// get a new HTTP request for the method.
req, err := http.NewRequest("PUT", targetURL.String(), nil)

View file

@ -94,6 +94,58 @@ func optimalPartInfo(objectSize int64) (totalPartsCount int, partSize int64, las
return totalPartsCount, partSize, lastPartSize, nil
}
// Compatibility code for Golang < 1.5.x.
// copyBuffer is identical to io.CopyBuffer, since such a function is
// not available/implemented in Golang version < 1.5.x, we use a
// custom call exactly implementng io.CopyBuffer from Golang > 1.5.x
// version does.
//
// copyBuffer stages through the provided buffer (if one is required)
// rather than allocating a temporary one. If buf is nil, one is
// allocated; otherwise if it has zero length, copyBuffer panics.
//
// FIXME: Remove this code when distributions move to newer Golang versions.
func copyBuffer(writer io.Writer, reader io.Reader, buf []byte) (written int64, err error) {
// If the reader has a WriteTo method, use it to do the copy.
// Avoids an allocation and a copy.
if wt, ok := reader.(io.WriterTo); ok {
return wt.WriteTo(writer)
}
// Similarly, if the writer has a ReadFrom method, use it to do
// the copy.
if rt, ok := writer.(io.ReaderFrom); ok {
return rt.ReadFrom(reader)
}
if buf == nil {
buf = make([]byte, 32*1024)
}
for {
nr, er := reader.Read(buf)
if nr > 0 {
nw, ew := writer.Write(buf[0:nr])
if nw > 0 {
written += int64(nw)
}
if ew != nil {
err = ew
break
}
if nr != nw {
err = io.ErrShortWrite
break
}
}
if er == io.EOF {
break
}
if er != nil {
err = er
break
}
}
return written, err
}
// hashCopyBuffer is identical to hashCopyN except that it stages
// through the provided buffer (if one is required) rather than
// allocating a temporary one. If buf is nil, one is allocated for 5MiB.
@ -113,9 +165,9 @@ func (c Client) hashCopyBuffer(writer io.Writer, reader io.Reader, buf []byte) (
buf = make([]byte, optimalReadBufferSize)
}
// Using io.CopyBuffer to copy in large buffers, default buffer
// Using copyBuffer to copy in large buffers, default buffer
// for io.Copy of 32KiB is too small.
size, err = io.CopyBuffer(hashWriter, reader, buf)
size, err = copyBuffer(hashWriter, reader, buf)
if err != nil {
return nil, nil, 0, err
}
@ -215,7 +267,7 @@ func (c Client) computeHashBuffer(reader io.ReadSeeker, buf []byte) (md5Sum, sha
return nil, nil, 0, err
}
} else {
size, err = io.CopyBuffer(hashWriter, reader, buf)
size, err = copyBuffer(hashWriter, reader, buf)
if err != nil {
return nil, nil, 0, err
}

View file

@ -88,7 +88,7 @@ func (c Client) FPutObject(bucketName, objectName, filePath, contentType string)
}
// Small object upload is initiated for uploads for input data size smaller than 5MiB.
if fileSize < minPartSize {
if fileSize < minPartSize && fileSize >= 0 {
return c.putObjectSingle(bucketName, objectName, fileReader, fileSize, contentType, nil)
}
// Upload all large objects as multipart.

View file

@ -82,7 +82,7 @@ func (c Client) PutObjectWithProgress(bucketName, objectName string, reader io.R
}
// putSmall object.
if size < minPartSize && size > 0 {
if size < minPartSize && size >= 0 {
return c.putObjectSingle(bucketName, objectName, reader, size, contentType, progress)
}
// For all sizes greater than 5MiB do multipart.

View file

@ -181,7 +181,7 @@ func (c *Client) SetCustomTransport(customHTTPTransport http.RoundTripper) {
}
// TraceOn - enable HTTP tracing.
func (c *Client) TraceOn(outputStream io.Writer) error {
func (c *Client) TraceOn(outputStream io.Writer) {
// if outputStream is nil then default to os.Stdout.
if outputStream == nil {
outputStream = os.Stdout
@ -191,7 +191,6 @@ func (c *Client) TraceOn(outputStream io.Writer) error {
// Enable tracing.
c.isTraceEnabled = true
return nil
}
// TraceOff - disable HTTP tracing.
@ -471,15 +470,15 @@ func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, que
}
} else {
// If not fall back to using path style.
urlStr = urlStr + bucketName
urlStr = urlStr + bucketName + "/"
if objectName != "" {
urlStr = urlStr + "/" + urlEncodePath(objectName)
urlStr = urlStr + urlEncodePath(objectName)
}
}
}
// If there are any query values, add them to the end.
if len(queryValues) > 0 {
urlStr = urlStr + "?" + queryValues.Encode()
urlStr = urlStr + "?" + queryEncode(queryValues)
}
u, err := url.Parse(urlStr)
if err != nil {
@ -528,6 +527,6 @@ type CloudStorageClient interface {
SetCustomTransport(customTransport http.RoundTripper)
// HTTP tracing methods.
TraceOn(traceOutput io.Writer) error
TraceOn(traceOutput io.Writer)
TraceOff()
}

View file

@ -31,6 +31,53 @@ import (
"github.com/minio/minio-go"
)
// Tests bucket re-create errors.
func TestMakeBucketErrorV2(t *testing.T) {
if testing.Short() {
t.Skip("skipping functional tests for short runs")
}
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Instantiate new minio client object.
c, err := minio.NewV2(
"s3.amazonaws.com",
os.Getenv("ACCESS_KEY"),
os.Getenv("SECRET_KEY"),
false,
)
if err != nil {
t.Fatal("Error:", err)
}
// Enable tracing, write to stderr.
// c.TraceOn(os.Stderr)
// Set user agent.
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
// Generate a new random bucket name.
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
// Make a new bucket in 'eu-west-1'.
if err = c.MakeBucket(bucketName, "private", "eu-west-1"); err != nil {
t.Fatal("Error:", err, bucketName)
}
if err = c.MakeBucket(bucketName, "private", "eu-west-1"); err == nil {
t.Fatal("Error: make bucket should should fail for", bucketName)
}
// Verify valid error response from server.
if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" &&
minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" {
t.Fatal("Error: Invalid error returned by server", err)
}
if err = c.RemoveBucket(bucketName); err != nil {
t.Fatal("Error:", err, bucketName)
}
}
// Test get object reader to not throw error on being closed twice.
func TestGetObjectClosedTwiceV2(t *testing.T) {
if testing.Short() {
t.Skip("skipping functional tests for short runs")
@ -39,8 +86,8 @@ func TestGetObjectClosedTwiceV2(t *testing.T) {
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Connect and make sure bucket exists.
c, err := minio.New(
// Instantiate new minio client object.
c, err := minio.NewV2(
"s3.amazonaws.com",
os.Getenv("ACCESS_KEY"),
os.Getenv("SECRET_KEY"),
@ -124,7 +171,7 @@ func TestRemovePartiallyUploadedV2(t *testing.T) {
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Connect and make sure bucket exists.
// Instantiate new minio client object.
c, err := minio.NewV2(
"s3.amazonaws.com",
os.Getenv("ACCESS_KEY"),
@ -191,8 +238,8 @@ func TestResumbalePutObjectV2(t *testing.T) {
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Connect and make sure bucket exists.
c, err := minio.New(
// Instantiate new minio client object.
c, err := minio.NewV2(
"s3.amazonaws.com",
os.Getenv("ACCESS_KEY"),
os.Getenv("SECRET_KEY"),
@ -302,8 +349,8 @@ func TestResumableFPutObjectV2(t *testing.T) {
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Connect and make sure bucket exists.
c, err := minio.New(
// Instantiate new minio client object.
c, err := minio.NewV2(
"s3.amazonaws.com",
os.Getenv("ACCESS_KEY"),
os.Getenv("SECRET_KEY"),
@ -370,6 +417,57 @@ func TestResumableFPutObjectV2(t *testing.T) {
}
}
// Tests various bucket supported formats.
func TestMakeBucketRegionsV2(t *testing.T) {
if testing.Short() {
t.Skip("skipping functional tests for short runs")
}
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Instantiate new minio client object.
c, err := minio.NewV2(
"s3.amazonaws.com",
os.Getenv("ACCESS_KEY"),
os.Getenv("SECRET_KEY"),
false,
)
if err != nil {
t.Fatal("Error:", err)
}
// Enable tracing, write to stderr.
// c.TraceOn(os.Stderr)
// Set user agent.
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
// Generate a new random bucket name.
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
// Make a new bucket in 'eu-central-1'.
if err = c.MakeBucket(bucketName, "private", "eu-west-1"); err != nil {
t.Fatal("Error:", err, bucketName)
}
if err = c.RemoveBucket(bucketName); err != nil {
t.Fatal("Error:", err, bucketName)
}
// Make a new bucket with '.' in its name, in 'us-west-2'. This
// request is internally staged into a path style instead of
// virtual host style.
if err = c.MakeBucket(bucketName+".withperiod", "private", "us-west-2"); err != nil {
t.Fatal("Error:", err, bucketName+".withperiod")
}
// Remove the newly created bucket.
if err = c.RemoveBucket(bucketName + ".withperiod"); err != nil {
t.Fatal("Error:", err, bucketName+".withperiod")
}
}
// Tests resumable put object multipart upload.
func TestResumablePutObjectV2(t *testing.T) {
if testing.Short() {
@ -379,8 +477,8 @@ func TestResumablePutObjectV2(t *testing.T) {
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Connect and make sure bucket exists.
c, err := minio.New(
// Instantiate new minio client object.
c, err := minio.NewV2(
"s3.amazonaws.com",
os.Getenv("ACCESS_KEY"),
os.Getenv("SECRET_KEY"),
@ -443,8 +541,8 @@ func TestGetObjectReadSeekFunctionalV2(t *testing.T) {
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Connect and make sure bucket exists.
c, err := minio.New(
// Instantiate new minio client object.
c, err := minio.NewV2(
"s3.amazonaws.com",
os.Getenv("ACCESS_KEY"),
os.Getenv("SECRET_KEY"),
@ -557,8 +655,8 @@ func TestGetObjectReadAtFunctionalV2(t *testing.T) {
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Connect and make sure bucket exists.
c, err := minio.New(
// Instantiate new minio client object.
c, err := minio.NewV2(
"s3.amazonaws.com",
os.Getenv("ACCESS_KEY"),
os.Getenv("SECRET_KEY"),
@ -699,7 +797,7 @@ func TestFunctionalV2(t *testing.T) {
// Seed random based on current time.
rand.Seed(time.Now().Unix())
c, err := minio.New(
c, err := minio.NewV2(
"s3.amazonaws.com",
os.Getenv("ACCESS_KEY"),
os.Getenv("SECRET_KEY"),

View file

@ -55,6 +55,52 @@ func randString(n int, src rand.Source) string {
return string(b[0:30])
}
// Tests bucket re-create errors.
func TestMakeBucketError(t *testing.T) {
if testing.Short() {
t.Skip("skipping functional tests for short runs")
}
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Instantiate new minio client object.
c, err := minio.New(
"s3.amazonaws.com",
os.Getenv("ACCESS_KEY"),
os.Getenv("SECRET_KEY"),
false,
)
if err != nil {
t.Fatal("Error:", err)
}
// Enable tracing, write to stderr.
// c.TraceOn(os.Stderr)
// Set user agent.
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
// Generate a new random bucket name.
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
// Make a new bucket in 'eu-central-1'.
if err = c.MakeBucket(bucketName, "private", "eu-central-1"); err != nil {
t.Fatal("Error:", err, bucketName)
}
if err = c.MakeBucket(bucketName, "private", "eu-central-1"); err == nil {
t.Fatal("Error: make bucket should should fail for", bucketName)
}
// Verify valid error response from server.
if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" &&
minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" {
t.Fatal("Error: Invalid error returned by server", err)
}
if err = c.RemoveBucket(bucketName); err != nil {
t.Fatal("Error:", err, bucketName)
}
}
// Tests various bucket supported formats.
func TestMakeBucketRegions(t *testing.T) {
if testing.Short() {
@ -64,7 +110,7 @@ func TestMakeBucketRegions(t *testing.T) {
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Connect and make sure bucket exists.
// Instantiate new minio client object.
c, err := minio.New(
"s3.amazonaws.com",
os.Getenv("ACCESS_KEY"),
@ -115,7 +161,7 @@ func TestGetObjectClosedTwice(t *testing.T) {
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Connect and make sure bucket exists.
// Instantiate new minio client object.
c, err := minio.New(
"s3.amazonaws.com",
os.Getenv("ACCESS_KEY"),
@ -200,7 +246,7 @@ func TestRemovePartiallyUploaded(t *testing.T) {
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Connect and make sure bucket exists.
// Instantiate new minio client object.
c, err := minio.New(
"s3.amazonaws.com",
os.Getenv("ACCESS_KEY"),
@ -236,7 +282,7 @@ func TestRemovePartiallyUploaded(t *testing.T) {
}
i++
}
err = writer.CloseWithError(errors.New("Proactively closed to be verified later."))
err := writer.CloseWithError(errors.New("Proactively closed to be verified later."))
if err != nil {
t.Fatal("Error:", err)
}
@ -270,7 +316,7 @@ func TestResumbalePutObject(t *testing.T) {
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Connect and make sure bucket exists.
// Instantiate new minio client object.
c, err := minio.New(
"s3.amazonaws.com",
os.Getenv("ACCESS_KEY"),
@ -380,7 +426,7 @@ func TestResumableFPutObject(t *testing.T) {
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Connect and make sure bucket exists.
// Instantiate new minio client object.
c, err := minio.New(
"s3.amazonaws.com",
os.Getenv("ACCESS_KEY"),
@ -460,7 +506,7 @@ func TestResumablePutObject(t *testing.T) {
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Connect and make sure bucket exists.
// Instantiate new minio client object.
c, err := minio.New(
"s3.amazonaws.com",
os.Getenv("ACCESS_KEY"),
@ -524,7 +570,7 @@ func TestGetObjectReadSeekFunctional(t *testing.T) {
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Connect and make sure bucket exists.
// Instantiate new minio client object.
c, err := minio.New(
"s3.amazonaws.com",
os.Getenv("ACCESS_KEY"),
@ -638,7 +684,7 @@ func TestGetObjectReadAtFunctional(t *testing.T) {
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Connect and make sure bucket exists.
// Instantiate new minio client object.
c, err := minio.New(
"s3.amazonaws.com",
os.Getenv("ACCESS_KEY"),

View file

@ -397,6 +397,22 @@ func TestPartSize(t *testing.T) {
}
}
// Tests query values to URL encoding.
func TestQueryURLEncoding(t *testing.T) {
urlValues := make(url.Values)
urlValues.Set("prefix", "test@1123")
urlValues.Set("delimiter", "/")
urlValues.Set("marker", "%%%@$$$")
queryStr := queryEncode(urlValues)
if !strings.Contains(queryStr, "test%401123") {
t.Fatalf("Error: @ should be encoded as %s, invalid query string %s", "test%401123", queryStr)
}
if !strings.Contains(queryStr, "%25%25%25%40%24%24%24") {
t.Fatalf("Error: %s should be encoded as %s, invalid query string %s", "%%%@$$$", "%25%25%25%40%24%24%24", queryStr)
}
}
// Tests url encoding.
func TestURLEncoding(t *testing.T) {
type urlStrings struct {

View file

@ -127,7 +127,7 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro
// Set get bucket location always as path style.
targetURL := c.endpointURL
targetURL.Path = filepath.Join(bucketName, "")
targetURL.Path = filepath.Join(bucketName, "") + "/"
targetURL.RawQuery = urlValues.Encode()
// Get a new HTTP request for the method.

View file

@ -24,6 +24,7 @@ import (
"fmt"
"net/http"
"net/url"
"path/filepath"
"sort"
"strconv"
"strings"
@ -38,8 +39,11 @@ const (
// Encode input URL path to URL encoded path.
func encodeURL2Path(u *url.URL) (path string) {
// Encode URL path.
if strings.HasSuffix(u.Host, ".s3.amazonaws.com") {
path = "/" + strings.TrimSuffix(u.Host, ".s3.amazonaws.com")
if isS3, _ := filepath.Match("*.s3*.amazonaws.com", u.Host); isS3 {
hostSplits := strings.SplitN(u.Host, ".", 4)
// First element is the bucket name.
bucketName := hostSplits[0]
path = "/" + bucketName
path += u.Path
path = urlEncodePath(path)
return
@ -251,10 +255,9 @@ var resourceList = []string{
// CanonicalizedResource = [ "/" + Bucket ] +
// <HTTP-Request-URI, from the protocol name up to the query string> +
// [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request) error {
func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request) {
// Save request URL.
requestURL := req.URL
// Get encoded URL path.
path := encodeURL2Path(requestURL)
buf.WriteString(path)
@ -285,5 +288,4 @@ func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request) error {
}
}
}
return nil
}

View file

@ -17,6 +17,7 @@
package minio
import (
"bytes"
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
@ -27,6 +28,7 @@ import (
"net/http"
"net/url"
"regexp"
"sort"
"strings"
"time"
"unicode/utf8"
@ -183,7 +185,7 @@ func isValidEndpointURL(endpointURL *url.URL) error {
return ErrInvalidArgument("Endpoint url cannot be empty.")
}
if endpointURL.Path != "/" && endpointURL.Path != "" {
return ErrInvalidArgument("Endpoing url cannot have fully qualified paths.")
return ErrInvalidArgument("Endpoint url cannot have fully qualified paths.")
}
if strings.Contains(endpointURL.Host, ".amazonaws.com") {
if !isAmazonEndpoint(endpointURL) {
@ -264,6 +266,31 @@ func isValidObjectPrefix(objectPrefix string) error {
return nil
}
// queryEncode - encodes query values in their URL encoded form.
func queryEncode(v url.Values) string {
if v == nil {
return ""
}
var buf bytes.Buffer
keys := make([]string, 0, len(v))
for k := range v {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
vs := v[k]
prefix := urlEncodePath(k) + "="
for _, v := range vs {
if buf.Len() > 0 {
buf.WriteByte('&')
}
buf.WriteString(prefix)
buf.WriteString(urlEncodePath(v))
}
}
return buf.String()
}
// urlEncodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences
//
// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8

4
vendor/vendor.json vendored
View file

@ -54,8 +54,8 @@
},
{
"path": "github.com/minio/minio-go",
"revision": "412df729f2c19ce60895770403f266cf5eac56f7",
"revisionTime": "2016-01-22T16:23:42-08:00"
"revision": "c5884ce9ce3ac73b025d0bc58c4d3d72870edc0b",
"revisionTime": "2016-02-02T13:13:10+05:30"
},
{
"path": "github.com/minio/minio-xl/pkg/atomic",