List Objects version 2. (#1815)

object: List Objects v2 support

api-resources.go

@@ -22,7 +22,39 @@ import (
 )
 
 // Parse bucket url queries
-func getBucketResources(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string) {
+func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string) {
+	prefix = values.Get("prefix")
+	marker = values.Get("marker")
+	delimiter = values.Get("delimiter")
+	if values.Get("max-keys") != "" {
+		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+	} else {
+		maxkeys = maxObjectList
+	}
+	encodingType = values.Get("encoding-type")
+	return
+}
+
+// Parse bucket url queries for ListObjects V2.
+func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, maxkeys int, encodingType string) {
+	prefix = values.Get("prefix")
+	startAfter = values.Get("start-after")
+	delimiter = values.Get("delimiter")
+	if values.Get("max-keys") != "" {
+		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+	} else {
+		maxkeys = maxObjectList
+	}
+	encodingType = values.Get("encoding-type")
+	token = values.Get("continuation-token")
+	return
+}
+
+// Parse bucket url queries
+func getBucketResources(values url.Values) (listType int, prefix, marker, delimiter string, maxkeys int, encodingType string) {
+	if values.Get("list-type") != "" {
+		listType, _ = strconv.Atoi(values.Get("list-type"))
+	}
 	prefix = values.Get("prefix")
 	marker = values.Get("marker")
 	delimiter = values.Get("delimiter")

api-response.go

@@ -65,6 +65,37 @@ type ListObjectsResponse struct {
 	Prefix     string
 }
 
+// ListObjectsV2Response - format for list objects response.
+type ListObjectsV2Response struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
+
+	CommonPrefixes []CommonPrefix
+	Contents       []Object
+
+	Delimiter string
+
+	// Encoding type used to encode object keys in the response.
+	EncodingType string
+
+	// A flag that indicates whether or not ListObjects returned all of the results
+	// that satisfied the search criteria.
+	IsTruncated bool
+	StartAfter  string
+	MaxKeys     int
+	Name        string
+
+	// When response is truncated (the IsTruncated element value in the response
+	// is true), you can use the key name in this field as marker in the subsequent
+	// request to get next set of objects. Server lists objects in alphabetical
+	// order Note: This element is returned only if you have delimiter request parameter
+	// specified. If response does not include the NextMaker and it is truncated,
+	// you can use the value of the last Key in the response as the marker in the
+	// subsequent request to get the next set of object keys.
+	ContinuationToken     string
+	NextContinuationToken string
+	Prefix                string
+}
+
 // Part container for part metadata.
 type Part struct {
 	PartNumber   int
@@ -304,6 +335,51 @@ func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKe
 	return data
 }
 
+// generates an ListObjects response for the said bucket with other enumerated options.
+func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter string, maxKeys int, resp ListObjectsInfo) ListObjectsV2Response {
+	var contents []Object
+	var prefixes []CommonPrefix
+	var owner = Owner{}
+	var data = ListObjectsV2Response{}
+
+	owner.ID = "minio"
+	owner.DisplayName = "minio"
+
+	for _, object := range resp.Objects {
+		var content = Object{}
+		if object.Name == "" {
+			continue
+		}
+		content.Key = object.Name
+		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZ)
+		if object.MD5Sum != "" {
+			content.ETag = "\"" + object.MD5Sum + "\""
+		}
+		content.Size = object.Size
+		content.StorageClass = "STANDARD"
+		content.Owner = owner
+		contents = append(contents, content)
+	}
+	// TODO - support EncodingType in xml decoding
+	data.Name = bucket
+	data.Contents = contents
+
+	data.StartAfter = startAfter
+	data.Delimiter = delimiter
+	data.Prefix = prefix
+	data.MaxKeys = maxKeys
+	data.ContinuationToken = token
+	data.NextContinuationToken = resp.NextMarker
+	data.IsTruncated = resp.IsTruncated
+	for _, prefix := range resp.Prefixes {
+		var prefixItem = CommonPrefix{}
+		prefixItem.Prefix = prefix
+		prefixes = append(prefixes, prefixItem)
+	}
+	data.CommonPrefixes = prefixes
+	return data
+}
+
 // generateCopyObjectResponse
 func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
 	return CopyObjectResponse{

bucket-handlers.go

@@ -220,9 +220,22 @@ func (api objectAPIHandlers) ListObjectsHandler(w http.ResponseWriter, r *http.R
 			return
 		}
 	}
-
+	var prefix, marker, token, delimiter, startAfter string
+	var maxkeys int
+	var listV2 bool
 	// TODO handle encoding type.
-	prefix, marker, delimiter, maxkeys, _ := getBucketResources(r.URL.Query())
+	if r.URL.Query().Get("list-type") == "2" {
+		listV2 = true
+		prefix, token, startAfter, delimiter, maxkeys, _ = getListObjectsV2Args(r.URL.Query())
+		// For ListV2 "start-after" is considered only if "continuation-token" is empty.
+		if token == "" {
+			marker = startAfter
+		} else {
+			marker = token
+		}
+	} else {
+		prefix, marker, delimiter, maxkeys, _ = getListObjectsV1Args(r.URL.Query())
+	}
 	if maxkeys < 0 {
 		writeErrorResponse(w, r, ErrInvalidMaxKeys, r.URL.Path)
 		return
@@ -242,10 +255,17 @@ func (api objectAPIHandlers) ListObjectsHandler(w http.ResponseWriter, r *http.R
 	}
 
 	listObjectsInfo, err := api.ObjectAPI.ListObjects(bucket, prefix, marker, delimiter, maxkeys)
+
 	if err == nil {
+		var encodedSuccessResponse []byte
 		// generate response
-		response := generateListObjectsResponse(bucket, prefix, marker, delimiter, maxkeys, listObjectsInfo)
-		encodedSuccessResponse := encodeResponse(response)
+		if listV2 {
+			response := generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter, maxkeys, listObjectsInfo)
+			encodedSuccessResponse = encodeResponse(response)
+		} else {
+			response := generateListObjectsResponse(bucket, prefix, marker, delimiter, maxkeys, listObjectsInfo)
+			encodedSuccessResponse = encodeResponse(response)
+		}
 		// Write headers
 		setCommonHeaders(w)
 		// Write success response.

xl-v1-list-objects.go

@@ -79,13 +79,10 @@ func (xl xlObjects) listObjects(bucket, prefix, marker, delimiter string, maxKey
 
 	result := ListObjectsInfo{IsTruncated: !eof}
 	for _, objInfo := range objInfos {
-		// With delimiter set we fill in NextMarker and Prefixes.
-		if delimiter == slashSeparator {
-			result.NextMarker = objInfo.Name
-			if objInfo.IsDir {
-				result.Prefixes = append(result.Prefixes, objInfo.Name)
-				continue
-			}
+		result.NextMarker = objInfo.Name
+		if objInfo.IsDir {
+			result.Prefixes = append(result.Prefixes, objInfo.Name)
+			continue
 		}
 		result.Objects = append(result.Objects, ObjectInfo{
 			Name:    objInfo.Name,