maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

api: Implement multiple objects Delete api - fixes #956

Browse source 
This API takes input XML input in following form.

```
<?xml version="1.0" encoding="UTF-8"?>
<Delete>
    <Quiet>true</Quiet>
    <Object>
         <Key>Key</Key>
    </Object>
    <Object>
         <Key>Key</Key>
    </Object>
    ...
</Delete>
```

and responds the list of successful deletes, list of errors
for all the deleted objects.

```
<?xml version="1.0" encoding="UTF-8"?>
<DeleteResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
  <Deleted>
    <Key>sample1.txt</Key>
  </Deleted>
  <Error>
    <Key>sample2.txt</Key>
    <Code>AccessDenied</Code>
    <Message>Access Denied</Message>
  </Error>
</DeleteResult>
```
This commit is contained in:
Harshavardhana 2016-03-05 16:43:48 -08:00
parent 6f842124ad
commit aed62788d9
9 changed files with 214 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
api-datatypes.go Normal file
View file

@ -0,0 +1,30 @@
/*
* Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
// ObjectIdentifier carries key name for the object to delete.
type ObjectIdentifier struct {
ObjectName string `xml:"Key"`
}
// DeleteObjectsRequest - xml carrying the object key names which needs to be deleted.
type DeleteObjectsRequest struct {
// Element to enable quiet mode for the request
Quiet bool
// List of objects to be deleted
Objects []ObjectIdentifier `xml:"Object"`
}

10
api-errors.go
View file

@ -62,6 +62,7 @@ const (
InvalidCopyDest
MalformedXML
MissingContentLength
MissingContentMD5
MissingRequestBodyError
NoSuchBucket
NoSuchKey
@ -125,7 +126,7 @@ var errorCodeResponse = map[int]APIError{
},
BadDigest: {
Code: "BadDigest",
Description: "The Content-MD5 you specified did not match what we received.",
Description: "The Content-Md5 you specified did not match what we received.",
HTTPStatusCode: http.StatusBadRequest,
},
BucketAlreadyExists: {
@ -165,7 +166,7 @@ var errorCodeResponse = map[int]APIError{
},
InvalidDigest: {
Code: "InvalidDigest",
Description: "The Content-MD5 you specified is not valid.",
Description: "The Content-Md5 you specified is not valid.",
HTTPStatusCode: http.StatusBadRequest,
},
InvalidRange: {
@ -183,6 +184,11 @@ var errorCodeResponse = map[int]APIError{
Description: "You must provide the Content-Length HTTP header.",
HTTPStatusCode: http.StatusLengthRequired,
},
MissingContentMD5: {
Code: "MissingContentMD5",
Description: "Missing required header for this request: Content-Md5.",
HTTPStatusCode: http.StatusBadRequest,
},
MissingRequestBodyError: {
Code: "MissingRequestBodyError",
Description: "Request body is empty.",

28
api-response.go
View file

@ -218,6 +218,24 @@ type CompleteMultipartUploadResponse struct {
ETag string
}
// DeleteError structure.
type DeleteError struct {
Code string
Message string
Key string
}
// DeleteObjectsResponse container for multiple object deletes.
type DeleteObjectsResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult" json:"-"`
// Collection of all deleted objects
DeletedObjects []ObjectIdentifier `xml:"Deleted,omitempty"`
// Collection of errors deleting certain objects.
Errors []DeleteError `xml:"Error,omitempty"`
}
// getLocation get URL location.
func getLocation(r *http.Request) string {
return r.URL.Path
@ -414,6 +432,16 @@ func generateListMultipartUploadsResponse(bucket string, metadata fs.BucketMulti
return listMultipartUploadsResponse
}
// generate multi objects delete response.
func generateMultiDeleteResponse(quiet bool, deletedObjects []ObjectIdentifier, errs []DeleteError) DeleteObjectsResponse {
deleteResp := DeleteObjectsResponse{}
if !quiet {
deleteResp.DeletedObjects = deletedObjects
}
deleteResp.Errors = errs
return deleteResp
}
// writeSuccessResponse write success headers and response if any.
func writeSuccessResponse(w http.ResponseWriter, response []byte) {
setCommonHeaders(w)

139
bucket-handlers.go
View file

@ -1,5 +1,5 @@
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
* Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -18,7 +18,10 @@ package main
import (
"bytes"
"crypto/md5"
"encoding/base64"
"encoding/hex"
"encoding/xml"
"io"
"io/ioutil"
"mime/multipart"
@ -212,6 +215,140 @@ func (api storageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request)
writeErrorResponse(w, r, InternalError, r.URL.Path)
}
// DeleteMultipleObjectsHandler - deletes multiple objects.
func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
if isRequestRequiresACLCheck(r) {
writeErrorResponse(w, r, AccessDenied, r.URL.Path)
return
}
// Content-Length is required and should be non-zero
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
if r.ContentLength <= 0 {
writeErrorResponse(w, r, MissingContentLength, r.URL.Path)
return
}
// Content-Md5 is requied should be set
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
if _, ok := r.Header["Content-Md5"]; !ok {
writeErrorResponse(w, r, MissingContentMD5, r.URL.Path)
return
}
// Set http request for signature.
auth := api.Signature.SetHTTPRequestToVerify(r)
// Allocate incoming content length bytes.
deleteXMLBytes := make([]byte, r.ContentLength)
// Read incoming body XML bytes.
_, e := io.ReadFull(r.Body, deleteXMLBytes)
if e != nil {
errorIf(probe.NewError(e), "DeleteMultipleObjects failed.", nil)
writeErrorResponse(w, r, InternalError, r.URL.Path)
return
}
// Check if request is presigned.
if isRequestPresignedSignatureV4(r) {
ok, err := auth.DoesPresignedSignatureMatch()
if err != nil {
errorIf(err.Trace(r.URL.String()), "Presigned signature verification failed.", nil)
writeErrorResponse(w, r, SignatureDoesNotMatch, r.URL.Path)
return
}
if !ok {
writeErrorResponse(w, r, SignatureDoesNotMatch, r.URL.Path)
return
}
} else if isRequestSignatureV4(r) {
// Check if request is signed.
sha := sha256.New()
mdSh := md5.New()
sha.Write(deleteXMLBytes)
mdSh.Write(deleteXMLBytes)
ok, err := auth.DoesSignatureMatch(hex.EncodeToString(sha.Sum(nil)))
if err != nil {
errorIf(err.Trace(), "DeleteMultipleObjects failed.", nil)
writeErrorResponse(w, r, InternalError, r.URL.Path)
return
}
if !ok {
writeErrorResponse(w, r, SignatureDoesNotMatch, r.URL.Path)
return
}
// Verify content md5.
if r.Header.Get("Content-Md5") != base64.StdEncoding.EncodeToString(mdSh.Sum(nil)) {
writeErrorResponse(w, r, BadDigest, r.URL.Path)
return
}
}
// Unmarshal list of keys to be deleted.
deleteObjects := &DeleteObjectsRequest{}
if e := xml.Unmarshal(deleteXMLBytes, deleteObjects); e != nil {
writeErrorResponse(w, r, MalformedXML, r.URL.Path)
return
}
var deleteErrors []DeleteError
var deletedObjects []ObjectIdentifier
// Loop through all the objects and delete them sequentially.
for _, object := range deleteObjects.Objects {
err := api.Filesystem.DeleteObject(bucket, object.ObjectName)
if err == nil {
deletedObjects = append(deletedObjects, ObjectIdentifier{
ObjectName: object.ObjectName,
})
} else {
errorIf(err.Trace(object.ObjectName), "DeleteObject failed.", nil)
switch err.ToGoError().(type) {
case fs.BucketNameInvalid:
deleteErrors = append(deleteErrors, DeleteError{
Code: errorCodeResponse[InvalidBucketName].Code,
Message: errorCodeResponse[InvalidBucketName].Description,
Key: object.ObjectName,
})
case fs.BucketNotFound:
deleteErrors = append(deleteErrors, DeleteError{
Code: errorCodeResponse[NoSuchBucket].Code,
Message: errorCodeResponse[NoSuchBucket].Description,
Key: object.ObjectName,
})
case fs.ObjectNotFound:
deleteErrors = append(deleteErrors, DeleteError{
Code: errorCodeResponse[NoSuchKey].Code,
Message: errorCodeResponse[NoSuchKey].Description,
Key: object.ObjectName,
})
case fs.ObjectNameInvalid:
deleteErrors = append(deleteErrors, DeleteError{
Code: errorCodeResponse[NoSuchKey].Code,
Message: errorCodeResponse[NoSuchKey].Description,
Key: object.ObjectName,
})
default:
deleteErrors = append(deleteErrors, DeleteError{
Code: errorCodeResponse[InternalError].Code,
Message: errorCodeResponse[InternalError].Description,
Key: object.ObjectName,
})
}
}
}
// Generate response
response := generateMultiDeleteResponse(deleteObjects.Quiet, deletedObjects, deleteErrors)
encodedSuccessResponse := encodeResponse(response)
// Write headers
setCommonHeaders(w)
// Write success response.
writeSuccessResponse(w, encodedSuccessResponse)
}
// PutBucketHandler - PUT Bucket
// ----------
// This implementation of the PUT operation creates a new bucket for authenticated request

1
generic-handlers.go
View file

@ -289,7 +289,6 @@ var notimplementedBucketResourceNames = map[string]bool{
"requestPayment": true,
"versioning": true,
"website": true,
"delete": true,
}
// List of not implemented object queries

10
object-handlers.go
View file

@ -420,8 +420,8 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
}
}
// get Content-MD5 sent by client and verify if valid
md5 := r.Header.Get("Content-MD5")
// get Content-Md5 sent by client and verify if valid
md5 := r.Header.Get("Content-Md5")
if !isValidMD5(md5) {
writeErrorResponse(w, r, InvalidDigest, r.URL.Path)
return
@ -554,8 +554,8 @@ func (api storageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Reques
}
}
// get Content-MD5 sent by client and verify if valid
md5 := r.Header.Get("Content-MD5")
// get Content-Md5 sent by client and verify if valid
md5 := r.Header.Get("Content-Md5")
if !isValidMD5(md5) {
writeErrorResponse(w, r, InvalidDigest, r.URL.Path)
return
@ -811,7 +811,7 @@ func (api storageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter, r *h
/// Delete storageAPI
// DeleteObjectHandler - Delete object
// DeleteObjectHandler - delete an object
func (api storageAPI) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]

2
pkg/fs/fs-multipart.go
View file

@ -202,7 +202,7 @@ func saveParts(partPathPrefix string, mw io.Writer, parts []CompletePart) *probe
if !os.IsNotExist(e) {
return probe.NewError(e)
}
// Some clients do not set Content-MD5, so we would have
// Some clients do not set Content-Md5, so we would have
// created part files without 'ETag' in them.
partFile, e = os.OpenFile(partPathPrefix+fmt.Sprintf("$%d-$multiparts", part.PartNumber), os.O_RDONLY, 0600)
if e != nil {

2
routers.go
View file

@ -149,6 +149,8 @@ func registerAPIHandlers(mux *router.Router, a storageAPI, w *webAPI) {
bucket.Methods("PUT").HandlerFunc(a.PutBucketHandler)
// HeadBucket
bucket.Methods("HEAD").HandlerFunc(a.HeadBucketHandler)
// DeleteMultipleObjects
bucket.Methods("POST").HandlerFunc(a.DeleteMultipleObjectsHandler)
// PostPolicy
bucket.Methods("POST").HandlerFunc(a.PostPolicyBucketHandler)
// DeleteBucket

4
server_fs_test.go
View file

@ -1158,7 +1158,7 @@ func (s *MyAPIFSCacheSuite) TestObjectMultipart(c *C) {
buffer1 := bytes.NewReader([]byte("hello world"))
request, err = s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultiparts/object?uploadId="+uploadID+"&partNumber=1", int64(buffer1.Len()), buffer1)
request.Header.Set("Content-MD5", base64.StdEncoding.EncodeToString(md5Sum))
request.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(md5Sum))
c.Assert(err, IsNil)
client = http.Client{}
@ -1168,7 +1168,7 @@ func (s *MyAPIFSCacheSuite) TestObjectMultipart(c *C) {
buffer2 := bytes.NewReader([]byte("hello world"))
request, err = s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultiparts/object?uploadId="+uploadID+"&partNumber=2", int64(buffer2.Len()), buffer2)
request.Header.Set("Content-MD5", base64.StdEncoding.EncodeToString(md5Sum))
request.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(md5Sum))
c.Assert(err, IsNil)
client = http.Client{}