From e4c0d574dc5883c3706a7078f089bfe7819c04e2 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 18 May 2015 11:53:28 -0700 Subject: [PATCH] Update totalnumber of multiparts inside CreateObjectPart(), also introduce support for UploadIDMarker and Prefix for listMultipartUploads --- pkg/api/api_bucket_handlers.go | 13 ++-- pkg/api/api_object_handlers.go | 35 +++++------ .../drivers/memory/memory_multipart.go | 59 +++++++++++++------ 3 files changed, 64 insertions(+), 43 deletions(-) diff --git a/pkg/api/api_bucket_handlers.go b/pkg/api/api_bucket_handlers.go index ac6788fe0..38934c7bc 100644 --- a/pkg/api/api_bucket_handlers.go +++ b/pkg/api/api_bucket_handlers.go @@ -56,6 +56,11 @@ func (server *minioAPI) isValidOp(w http.ResponseWriter, req *http.Request, acce //return false } } + default: + { + log.Error.Println(iodine.New(err, nil)) + writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) + } } return true } @@ -79,7 +84,7 @@ func (server *minioAPI) listMultipartUploadsHandler(w http.ResponseWriter, req * bucket := vars["bucket"] resources, err := server.driver.ListMultipartUploads(bucket, resources) - switch err := iodine.ToError(err).(type) { + switch iodine.ToError(err).(type) { case nil: // success { // generate response @@ -130,7 +135,7 @@ func (server *minioAPI) listObjectsHandler(w http.ResponseWriter, req *http.Requ bucket := vars["bucket"] objects, resources, err := server.driver.ListObjects(bucket, resources) - switch err := iodine.ToError(err).(type) { + switch iodine.ToError(err).(type) { case nil: // success { // generate response @@ -170,7 +175,7 @@ func (server *minioAPI) listBucketsHandler(w http.ResponseWriter, req *http.Requ // return // } buckets, err := server.driver.ListBuckets() - switch err := iodine.ToError(err).(type) { + switch iodine.ToError(err).(type) { case nil: { // generate response @@ -307,7 +312,7 @@ func (server *minioAPI) headBucketHandler(w http.ResponseWriter, req *http.Reque } default: { - log.Println(err) + log.Error.Println(iodine.New(err, nil)) error := getErrorCode(InternalError) w.WriteHeader(error.HTTPStatusCode) } diff --git a/pkg/api/api_object_handlers.go b/pkg/api/api_object_handlers.go index 2b38da17c..0c7a59281 100644 --- a/pkg/api/api_object_handlers.go +++ b/pkg/api/api_object_handlers.go @@ -50,7 +50,7 @@ func (server *minioAPI) getObjectHandler(w http.ResponseWriter, req *http.Reques object = vars["object"] metadata, err := server.driver.GetObjectMetadata(bucket, object) - switch err := iodine.ToError(err).(type) { + switch iodine.ToError(err).(type) { case nil: // success { httpRange, err := getRequestedRange(req, metadata.Size) @@ -63,7 +63,7 @@ func (server *minioAPI) getObjectHandler(w http.ResponseWriter, req *http.Reques setObjectHeaders(w, metadata) if _, err := server.driver.GetObject(w, bucket, object); err != nil { // unable to write headers, we've already printed data. Just close the connection. - log.Error.Println(err) + log.Error.Println(iodine.New(err, nil)) } case false: metadata.Size = httpRange.length @@ -107,7 +107,7 @@ func (server *minioAPI) headObjectHandler(w http.ResponseWriter, req *http.Reque object = vars["object"] metadata, err := server.driver.GetObjectMetadata(bucket, object) - switch err := iodine.ToError(err).(type) { + switch iodine.ToError(err).(type) { case nil: { setObjectHeaders(w, metadata) @@ -178,7 +178,7 @@ func (server *minioAPI) putObjectHandler(w http.ResponseWriter, req *http.Reques return } calculatedMD5, err := server.driver.CreateObject(bucket, object, "", md5, sizeInt64, req.Body) - switch err := iodine.ToError(err).(type) { + switch iodine.ToError(err).(type) { case nil: { w.Header().Set("ETag", calculatedMD5) @@ -201,14 +201,9 @@ func (server *minioAPI) putObjectHandler(w http.ResponseWriter, req *http.Reques { writeErrorResponse(w, req, InvalidDigest, acceptsContentType, req.URL.Path) } - case drivers.ImplementationError: - { - log.Error.Println(err) - writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) - } default: { - log.Error.Println(err) + log.Error.Println(iodine.New(err, nil)) writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) } } @@ -231,7 +226,7 @@ func (server *minioAPI) newMultipartUploadHandler(w http.ResponseWriter, req *ht bucket = vars["bucket"] object = vars["object"] uploadID, err := server.driver.NewMultipartUpload(bucket, object, "") - switch err := iodine.ToError(err).(type) { + switch iodine.ToError(err).(type) { case nil: { response := generateInitiateMultipartUploadResult(bucket, object, uploadID) @@ -247,7 +242,7 @@ func (server *minioAPI) newMultipartUploadHandler(w http.ResponseWriter, req *ht } default: { - log.Println(iodine.New(err, nil)) + log.Error.Println(iodine.New(err, nil)) writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) } } @@ -307,7 +302,7 @@ func (server *minioAPI) putObjectPartHandler(w http.ResponseWriter, req *http.Re writeErrorResponse(w, req, InvalidPart, acceptsContentType, req.URL.Path) } calculatedMD5, err := server.driver.CreateObjectPart(bucket, object, uploadID, partID, "", md5, sizeInt64, req.Body) - switch err := iodine.ToError(err).(type) { + switch iodine.ToError(err).(type) { case nil: { w.Header().Set("ETag", calculatedMD5) @@ -336,7 +331,7 @@ func (server *minioAPI) putObjectPartHandler(w http.ResponseWriter, req *http.Re } default: { - log.Error.Println(err) + log.Error.Println(iodine.New(err, nil)) writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) } } @@ -356,7 +351,7 @@ func (server *minioAPI) abortMultipartUploadHandler(w http.ResponseWriter, req * objectResourcesMetadata := getObjectResources(req.URL.Query()) err := server.driver.AbortMultipartUpload(bucket, object, objectResourcesMetadata.UploadID) - switch err := iodine.ToError(err).(type) { + switch iodine.ToError(err).(type) { case nil: { setCommonHeaders(w, getContentTypeString(acceptsContentType), 0) @@ -368,7 +363,7 @@ func (server *minioAPI) abortMultipartUploadHandler(w http.ResponseWriter, req * } default: { - log.Println(err) + log.Error.Println(iodine.New(err, nil)) writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) } } @@ -391,7 +386,7 @@ func (server *minioAPI) listObjectPartsHandler(w http.ResponseWriter, req *http. object := vars["object"] objectResourcesMetadata, err := server.driver.ListObjectParts(bucket, object, objectResourcesMetadata) - switch err := iodine.ToError(err).(type) { + switch iodine.ToError(err).(type) { case nil: { response := generateListPartsResult(objectResourcesMetadata) @@ -407,7 +402,7 @@ func (server *minioAPI) listObjectPartsHandler(w http.ResponseWriter, req *http. } default: { - log.Println(err) + log.Error.Println(iodine.New(err, nil)) writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) } } @@ -444,7 +439,7 @@ func (server *minioAPI) completeMultipartUploadHandler(w http.ResponseWriter, re } etag, err := server.driver.CompleteMultipartUpload(bucket, object, objectResourcesMetadata.UploadID, partMap) - switch err := iodine.ToError(err).(type) { + switch iodine.ToError(err).(type) { case nil: { response := generateCompleteMultpartUploadResult(bucket, object, "", etag) @@ -460,7 +455,7 @@ func (server *minioAPI) completeMultipartUploadHandler(w http.ResponseWriter, re } default: { - log.Println(iodine.New(err, nil)) + log.Error.Println(iodine.New(err, nil)) writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) } } diff --git a/pkg/storage/drivers/memory/memory_multipart.go b/pkg/storage/drivers/memory/memory_multipart.go index cb24aea49..2da8ea7ce 100644 --- a/pkg/storage/drivers/memory/memory_multipart.go +++ b/pkg/storage/drivers/memory/memory_multipart.go @@ -105,10 +105,6 @@ func (memory *memoryDriver) CreateObjectPart(bucket, key, uploadID string, partI if err != nil { return "", iodine.New(err, nil) } - // once successful, update totalParts - multiPartSession := storedBucket.multiPartSession[key] - multiPartSession.totalParts++ - storedBucket.multiPartSession[key] = multiPartSession // free debug.FreeOSMemory() return etag, nil @@ -204,6 +200,9 @@ func (memory *memoryDriver) createObjectPart(bucket, key, uploadID string, partI default: storedBucket.partMetadata[partKey] = newPart } + multiPartSession := storedBucket.multiPartSession[key] + multiPartSession.totalParts++ + storedBucket.multiPartSession[key] = multiPartSession memory.storedBuckets[bucket] = storedBucket memory.lock.Unlock() @@ -294,7 +293,7 @@ func (a byKey) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a byKey) Less(i, j int) bool { return a[i].Key < a[j].Key } func (memory *memoryDriver) ListMultipartUploads(bucket string, resources drivers.BucketMultipartResourcesMetadata) (drivers.BucketMultipartResourcesMetadata, error) { - // TODO handle delimiter, prefix, uploadIDMarker + // TODO handle delimiter memory.lock.RLock() defer memory.lock.RUnlock() if _, ok := memory.storedBuckets[bucket]; ok == false { @@ -304,20 +303,42 @@ func (memory *memoryDriver) ListMultipartUploads(bucket string, resources driver var uploads []*drivers.UploadMetadata for key, session := range storedBucket.multiPartSession { - if len(uploads) > resources.MaxUploads { - sort.Sort(byKey(uploads)) - resources.Upload = uploads - resources.NextKeyMarker = key - resources.NextUploadIDMarker = session.uploadID - resources.IsTruncated = true - return resources, nil - } - if key > resources.KeyMarker { - upload := new(drivers.UploadMetadata) - upload.Key = key - upload.UploadID = session.uploadID - upload.Initiated = session.initiated - uploads = append(uploads, upload) + if strings.HasPrefix(key, resources.Prefix) { + if len(uploads) > resources.MaxUploads { + sort.Sort(byKey(uploads)) + resources.Upload = uploads + resources.NextKeyMarker = key + resources.NextUploadIDMarker = session.uploadID + resources.IsTruncated = true + return resources, nil + } + // uploadIDMarker is ignored if KeyMarker is empty + switch { + case resources.KeyMarker != "" && resources.UploadIDMarker == "": + if key > resources.KeyMarker { + upload := new(drivers.UploadMetadata) + upload.Key = key + upload.UploadID = session.uploadID + upload.Initiated = session.initiated + uploads = append(uploads, upload) + } + case resources.KeyMarker != "" && resources.UploadIDMarker != "": + if session.uploadID > resources.UploadIDMarker { + if key >= resources.KeyMarker { + upload := new(drivers.UploadMetadata) + upload.Key = key + upload.UploadID = session.uploadID + upload.Initiated = session.initiated + uploads = append(uploads, upload) + } + } + default: + upload := new(drivers.UploadMetadata) + upload.Key = key + upload.UploadID = session.uploadID + upload.Initiated = session.initiated + uploads = append(uploads, upload) + } } } sort.Sort(byKey(uploads))