2016-05-03 01:57:31 +02:00
|
|
|
/*
|
2019-04-09 20:39:42 +02:00
|
|
|
* MinIO Cloud Storage, (C) 2016, 2017, 2018 MinIO, Inc.
|
2016-05-03 01:57:31 +02:00
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2016-08-19 01:23:42 +02:00
|
|
|
package cmd
|
2016-05-03 01:57:31 +02:00
|
|
|
|
|
|
|
import (
|
2018-03-14 20:01:47 +01:00
|
|
|
"context"
|
2018-01-31 22:17:24 +01:00
|
|
|
"encoding/json"
|
2016-05-03 01:57:31 +02:00
|
|
|
"fmt"
|
2018-01-31 22:17:24 +01:00
|
|
|
"io/ioutil"
|
2017-01-17 02:05:00 +01:00
|
|
|
"os"
|
|
|
|
pathutil "path"
|
2018-01-31 22:17:24 +01:00
|
|
|
"sort"
|
|
|
|
"strconv"
|
2016-05-04 01:10:24 +02:00
|
|
|
"strings"
|
2016-05-21 05:48:47 +02:00
|
|
|
"time"
|
2016-12-20 04:32:55 +01:00
|
|
|
|
2019-09-06 00:51:27 +02:00
|
|
|
jsoniter "github.com/json-iterator/go"
|
2018-04-06 00:04:40 +02:00
|
|
|
"github.com/minio/minio/cmd/logger"
|
2018-01-31 22:17:24 +01:00
|
|
|
mioutil "github.com/minio/minio/pkg/ioutil"
|
2016-05-03 01:57:31 +02:00
|
|
|
)
|
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
// Returns EXPORT/.minio.sys/multipart/SHA256/UPLOADID
|
2018-02-20 21:21:12 +01:00
|
|
|
func (fs *FSObjects) getUploadIDDir(bucket, object, uploadID string) string {
|
2018-01-31 22:17:24 +01:00
|
|
|
return pathJoin(fs.fsPath, minioMetaMultipartBucket, getSHA256Hash([]byte(pathJoin(bucket, object))), uploadID)
|
2017-01-26 21:51:12 +01:00
|
|
|
}
|
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
// Returns EXPORT/.minio.sys/multipart/SHA256
|
2018-02-20 21:21:12 +01:00
|
|
|
func (fs *FSObjects) getMultipartSHADir(bucket, object string) string {
|
2018-01-31 22:17:24 +01:00
|
|
|
return pathJoin(fs.fsPath, minioMetaMultipartBucket, getSHA256Hash([]byte(pathJoin(bucket, object))))
|
2017-01-26 21:51:12 +01:00
|
|
|
}
|
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
// Returns partNumber.etag
|
2018-09-28 05:36:17 +02:00
|
|
|
func (fs *FSObjects) encodePartFile(partNumber int, etag string, actualSize int64) string {
|
|
|
|
return fmt.Sprintf("%.5d.%s.%d", partNumber, etag, actualSize)
|
2017-01-26 21:51:12 +01:00
|
|
|
}
|
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
// Returns partNumber and etag
|
2018-09-28 05:36:17 +02:00
|
|
|
func (fs *FSObjects) decodePartFile(name string) (partNumber int, etag string, actualSize int64, err error) {
|
2018-01-31 22:17:24 +01:00
|
|
|
result := strings.Split(name, ".")
|
2018-09-28 05:36:17 +02:00
|
|
|
if len(result) != 3 {
|
|
|
|
return 0, "", 0, errUnexpected
|
2017-08-31 20:29:22 +02:00
|
|
|
}
|
2018-01-31 22:17:24 +01:00
|
|
|
partNumber, err = strconv.Atoi(result[0])
|
2017-01-17 02:05:00 +01:00
|
|
|
if err != nil {
|
2018-09-28 05:36:17 +02:00
|
|
|
return 0, "", 0, errUnexpected
|
2017-01-17 02:05:00 +01:00
|
|
|
}
|
2018-09-28 05:36:17 +02:00
|
|
|
actualSize, err = strconv.ParseInt(result[2], 10, 64)
|
|
|
|
if err != nil {
|
|
|
|
return 0, "", 0, errUnexpected
|
|
|
|
}
|
|
|
|
return partNumber, result[1], actualSize, nil
|
2018-01-31 22:17:24 +01:00
|
|
|
}
|
2017-01-17 02:05:00 +01:00
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
// Appends parts to an appendFile sequentially.
|
2018-04-06 00:04:40 +02:00
|
|
|
func (fs *FSObjects) backgroundAppend(ctx context.Context, bucket, object, uploadID string) {
|
2018-01-31 22:17:24 +01:00
|
|
|
fs.appendFileMapMu.Lock()
|
2018-04-06 00:04:40 +02:00
|
|
|
logger.GetReqInfo(ctx).AppendTags("uploadID", uploadID)
|
2018-01-31 22:17:24 +01:00
|
|
|
file := fs.appendFileMap[uploadID]
|
|
|
|
if file == nil {
|
|
|
|
file = &fsAppendFile{
|
|
|
|
filePath: pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, fmt.Sprintf("%s.%s", uploadID, mustGetUUID())),
|
2017-01-17 02:05:00 +01:00
|
|
|
}
|
2018-01-31 22:17:24 +01:00
|
|
|
fs.appendFileMap[uploadID] = file
|
2017-01-17 02:05:00 +01:00
|
|
|
}
|
2018-01-31 22:17:24 +01:00
|
|
|
fs.appendFileMapMu.Unlock()
|
2017-01-17 02:05:00 +01:00
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
file.Lock()
|
|
|
|
defer file.Unlock()
|
2017-01-17 02:05:00 +01:00
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
// Since we append sequentially nextPartNumber will always be len(file.parts)+1
|
|
|
|
nextPartNumber := len(file.parts) + 1
|
|
|
|
uploadIDDir := fs.getUploadIDDir(bucket, object, uploadID)
|
2016-05-04 01:10:24 +02:00
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
entries, err := readDir(uploadIDDir)
|
|
|
|
if err != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
logger.GetReqInfo(ctx).AppendTags("uploadIDDir", uploadIDDir)
|
|
|
|
logger.LogIf(ctx, err)
|
2018-01-31 22:17:24 +01:00
|
|
|
return
|
2016-05-04 01:10:24 +02:00
|
|
|
}
|
2018-01-31 22:17:24 +01:00
|
|
|
sort.Strings(entries)
|
2017-01-17 02:05:00 +01:00
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
for _, entry := range entries {
|
2018-03-28 23:14:06 +02:00
|
|
|
if entry == fs.metaJSONFile {
|
2018-01-31 22:17:24 +01:00
|
|
|
continue
|
|
|
|
}
|
2018-09-28 05:36:17 +02:00
|
|
|
partNumber, etag, actualSize, err := fs.decodePartFile(entry)
|
2016-05-26 23:43:17 +02:00
|
|
|
if err != nil {
|
2019-04-13 00:51:32 +02:00
|
|
|
// Skip part files whose name don't match expected format. These could be backend filesystem specific files.
|
|
|
|
continue
|
2016-05-26 23:43:17 +02:00
|
|
|
}
|
2018-01-31 22:17:24 +01:00
|
|
|
if partNumber < nextPartNumber {
|
|
|
|
// Part already appended.
|
|
|
|
continue
|
2016-05-26 23:43:17 +02:00
|
|
|
}
|
2018-01-31 22:17:24 +01:00
|
|
|
if partNumber > nextPartNumber {
|
|
|
|
// Required part number is not yet uploaded.
|
|
|
|
return
|
2016-05-26 23:43:17 +02:00
|
|
|
}
|
2017-01-17 02:05:00 +01:00
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
partPath := pathJoin(uploadIDDir, entry)
|
|
|
|
err = mioutil.AppendFile(file.filePath, partPath)
|
|
|
|
if err != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
reqInfo := logger.GetReqInfo(ctx).AppendTags("partPath", partPath)
|
|
|
|
reqInfo.AppendTags("filepath", file.filePath)
|
|
|
|
logger.LogIf(ctx, err)
|
2018-01-31 22:17:24 +01:00
|
|
|
return
|
2016-05-04 01:10:24 +02:00
|
|
|
}
|
2016-07-07 18:06:35 +02:00
|
|
|
|
2018-09-28 05:36:17 +02:00
|
|
|
file.parts = append(file.parts, PartInfo{PartNumber: partNumber, ETag: etag, ActualSize: actualSize})
|
2018-01-31 22:17:24 +01:00
|
|
|
nextPartNumber++
|
2016-05-04 01:10:24 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-10-06 01:08:25 +02:00
|
|
|
// ListMultipartUploads - lists all the uploadIDs for the specified object.
|
|
|
|
// We do not support prefix based listing.
|
2018-03-14 20:01:47 +01:00
|
|
|
func (fs *FSObjects) ListMultipartUploads(ctx context.Context, bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, e error) {
|
2018-04-06 00:04:40 +02:00
|
|
|
if err := checkListMultipartArgs(ctx, bucket, object, keyMarker, uploadIDMarker, delimiter, fs); err != nil {
|
|
|
|
return result, toObjectErr(err)
|
2017-08-04 19:45:57 +02:00
|
|
|
}
|
|
|
|
|
2018-04-06 00:04:40 +02:00
|
|
|
if _, err := fs.statBucketDir(ctx, bucket); err != nil {
|
|
|
|
return result, toObjectErr(err, bucket)
|
2017-08-04 19:45:57 +02:00
|
|
|
}
|
|
|
|
|
2017-10-06 01:08:25 +02:00
|
|
|
result.MaxUploads = maxUploads
|
|
|
|
result.KeyMarker = keyMarker
|
|
|
|
result.Prefix = object
|
|
|
|
result.Delimiter = delimiter
|
2018-01-31 22:17:24 +01:00
|
|
|
result.NextKeyMarker = object
|
|
|
|
result.UploadIDMarker = uploadIDMarker
|
2017-10-06 01:08:25 +02:00
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
uploadIDs, err := readDir(fs.getMultipartSHADir(bucket, object))
|
2017-10-06 01:08:25 +02:00
|
|
|
if err != nil {
|
2018-01-31 22:17:24 +01:00
|
|
|
if err == errFileNotFound {
|
|
|
|
result.IsTruncated = false
|
|
|
|
return result, nil
|
|
|
|
}
|
2018-04-06 00:04:40 +02:00
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
return result, toObjectErr(err)
|
2016-06-02 21:18:56 +02:00
|
|
|
}
|
2017-01-17 02:05:00 +01:00
|
|
|
|
2019-07-23 07:36:15 +02:00
|
|
|
// S3 spec says uploadIDs should be sorted based on initiated time. ModTime of fs.json
|
2018-01-31 22:17:24 +01:00
|
|
|
// is the creation time of the uploadID, hence we will use that.
|
|
|
|
var uploads []MultipartInfo
|
|
|
|
for _, uploadID := range uploadIDs {
|
2018-03-28 23:14:06 +02:00
|
|
|
metaFilePath := pathJoin(fs.getMultipartSHADir(bucket, object), uploadID, fs.metaJSONFile)
|
2018-04-06 00:04:40 +02:00
|
|
|
fi, err := fsStatFile(ctx, metaFilePath)
|
2018-01-31 22:17:24 +01:00
|
|
|
if err != nil {
|
|
|
|
return result, toObjectErr(err, bucket, object)
|
|
|
|
}
|
|
|
|
uploads = append(uploads, MultipartInfo{
|
|
|
|
Object: object,
|
2019-08-06 21:08:58 +02:00
|
|
|
UploadID: strings.TrimSuffix(uploadID, SlashSeparator),
|
2018-01-31 22:17:24 +01:00
|
|
|
Initiated: fi.ModTime(),
|
|
|
|
})
|
2017-01-17 02:05:00 +01:00
|
|
|
}
|
2018-01-31 22:17:24 +01:00
|
|
|
sort.Slice(uploads, func(i int, j int) bool {
|
|
|
|
return uploads[i].Initiated.Before(uploads[j].Initiated)
|
|
|
|
})
|
2017-01-17 02:05:00 +01:00
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
uploadIndex := 0
|
|
|
|
if uploadIDMarker != "" {
|
|
|
|
for uploadIndex < len(uploads) {
|
|
|
|
if uploads[uploadIndex].UploadID != uploadIDMarker {
|
|
|
|
uploadIndex++
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if uploads[uploadIndex].UploadID == uploadIDMarker {
|
|
|
|
uploadIndex++
|
|
|
|
break
|
|
|
|
}
|
|
|
|
uploadIndex++
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for uploadIndex < len(uploads) {
|
|
|
|
result.Uploads = append(result.Uploads, uploads[uploadIndex])
|
|
|
|
result.NextUploadIDMarker = uploads[uploadIndex].UploadID
|
|
|
|
uploadIndex++
|
|
|
|
if len(result.Uploads) == maxUploads {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
result.IsTruncated = uploadIndex < len(uploads)
|
2017-10-06 01:08:25 +02:00
|
|
|
|
|
|
|
if !result.IsTruncated {
|
|
|
|
result.NextKeyMarker = ""
|
|
|
|
result.NextUploadIDMarker = ""
|
|
|
|
}
|
|
|
|
|
|
|
|
return result, nil
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
|
|
|
|
2016-06-02 21:18:56 +02:00
|
|
|
// NewMultipartUpload - initialize a new multipart upload, returns a
|
|
|
|
// unique id. The unique id returned here is of UUID form, for each
|
|
|
|
// subsequent request each UUID is unique.
|
|
|
|
//
|
|
|
|
// Implements S3 compatible initiate multipart API.
|
2019-02-09 06:31:06 +01:00
|
|
|
func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (string, error) {
|
2018-04-06 00:04:40 +02:00
|
|
|
if err := checkNewMultipartArgs(ctx, bucket, object, fs); err != nil {
|
2018-01-31 22:17:24 +01:00
|
|
|
return "", toObjectErr(err, bucket)
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
2017-01-17 02:05:00 +01:00
|
|
|
|
2018-04-06 00:04:40 +02:00
|
|
|
if _, err := fs.statBucketDir(ctx, bucket); err != nil {
|
2017-01-17 02:05:00 +01:00
|
|
|
return "", toObjectErr(err, bucket)
|
|
|
|
}
|
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
uploadID := mustGetUUID()
|
|
|
|
uploadIDDir := fs.getUploadIDDir(bucket, object, uploadID)
|
|
|
|
|
|
|
|
err := mkdirAll(uploadIDDir, 0755)
|
|
|
|
if err != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
return "", err
|
2017-08-31 20:29:22 +02:00
|
|
|
}
|
2017-01-17 02:05:00 +01:00
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
// Initialize fs.json values.
|
|
|
|
fsMeta := newFSMetaV1()
|
2019-02-09 06:31:06 +01:00
|
|
|
fsMeta.Meta = opts.UserDefined
|
2016-05-21 05:48:47 +02:00
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
fsMetaBytes, err := json.Marshal(fsMeta)
|
|
|
|
if err != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
return "", err
|
2016-08-31 22:42:57 +02:00
|
|
|
}
|
2017-01-17 02:05:00 +01:00
|
|
|
|
2018-03-28 23:14:06 +02:00
|
|
|
if err = ioutil.WriteFile(pathJoin(uploadIDDir, fs.metaJSONFile), fsMetaBytes, 0644); err != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
return "", err
|
2016-08-31 22:42:57 +02:00
|
|
|
}
|
2018-03-01 20:37:57 +01:00
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
return uploadID, nil
|
2016-08-31 22:42:57 +02:00
|
|
|
}
|
|
|
|
|
2017-01-31 18:38:34 +01:00
|
|
|
// CopyObjectPart - similar to PutObjectPart but reads data from an existing
|
|
|
|
// object. Internally incoming data is written to '.minio.sys/tmp' location
|
|
|
|
// and safely renamed to '.minio.sys/multipart' for reach parts.
|
2018-03-14 20:01:47 +01:00
|
|
|
func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int,
|
2018-09-10 18:42:43 +02:00
|
|
|
startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, e error) {
|
2018-01-13 05:34:52 +01:00
|
|
|
|
2018-04-06 00:04:40 +02:00
|
|
|
if err := checkNewMultipartArgs(ctx, srcBucket, srcObject, fs); err != nil {
|
|
|
|
return pi, toObjectErr(err)
|
2017-01-31 18:38:34 +01:00
|
|
|
}
|
|
|
|
|
2018-11-15 02:36:41 +01:00
|
|
|
partInfo, err := fs.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, srcInfo.PutObjReader, dstOpts)
|
2017-01-31 18:38:34 +01:00
|
|
|
if err != nil {
|
2017-06-22 04:53:09 +02:00
|
|
|
return pi, toObjectErr(err, dstBucket, dstObject)
|
2017-01-31 18:38:34 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
return partInfo, nil
|
|
|
|
}
|
|
|
|
|
2016-06-03 00:19:13 +02:00
|
|
|
// PutObjectPart - reads incoming data until EOF for the part file on
|
2016-06-02 21:18:56 +02:00
|
|
|
// an ongoing multipart transaction. Internally incoming data is
|
2016-09-07 05:31:50 +02:00
|
|
|
// written to '.minio.sys/tmp' location and safely renamed to
|
|
|
|
// '.minio.sys/multipart' for reach parts.
|
2018-11-15 02:36:41 +01:00
|
|
|
func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (pi PartInfo, e error) {
|
|
|
|
data := r.Reader
|
2018-04-06 00:04:40 +02:00
|
|
|
if err := checkPutObjectPartArgs(ctx, bucket, object, fs); err != nil {
|
|
|
|
return pi, toObjectErr(err, bucket)
|
2016-06-03 00:19:13 +02:00
|
|
|
}
|
|
|
|
|
2018-04-06 00:04:40 +02:00
|
|
|
if _, err := fs.statBucketDir(ctx, bucket); err != nil {
|
|
|
|
return pi, toObjectErr(err, bucket)
|
2017-01-17 02:05:00 +01:00
|
|
|
}
|
|
|
|
|
2018-09-28 05:36:17 +02:00
|
|
|
// Validate input data size and it can never be less than -1.
|
|
|
|
if data.Size() < -1 {
|
2019-10-12 03:50:54 +02:00
|
|
|
logger.LogIf(ctx, errInvalidArgument, logger.Application)
|
2018-04-06 00:04:40 +02:00
|
|
|
return pi, toObjectErr(errInvalidArgument)
|
2017-10-06 18:38:01 +02:00
|
|
|
}
|
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
uploadIDDir := fs.getUploadIDDir(bucket, object, uploadID)
|
2016-06-03 00:19:13 +02:00
|
|
|
|
|
|
|
// Just check if the uploadID exists to avoid copy if it doesn't.
|
2018-04-06 00:04:40 +02:00
|
|
|
_, err := fsStatFile(ctx, pathJoin(uploadIDDir, fs.metaJSONFile))
|
2017-01-17 02:05:00 +01:00
|
|
|
if err != nil {
|
2018-04-10 18:36:37 +02:00
|
|
|
if err == errFileNotFound || err == errFileAccessDenied {
|
2018-04-06 00:04:40 +02:00
|
|
|
return pi, InvalidUploadID{UploadID: uploadID}
|
2017-01-17 02:05:00 +01:00
|
|
|
}
|
2018-01-31 22:17:24 +01:00
|
|
|
return pi, toObjectErr(err, bucket, object)
|
2017-01-17 02:05:00 +01:00
|
|
|
}
|
2016-05-21 05:48:47 +02:00
|
|
|
|
2016-07-06 05:59:54 +02:00
|
|
|
bufSize := int64(readSizeV1)
|
2017-09-19 21:40:27 +02:00
|
|
|
if size := data.Size(); size > 0 && bufSize > size {
|
2016-07-06 05:59:54 +02:00
|
|
|
bufSize = size
|
|
|
|
}
|
2017-09-19 21:40:27 +02:00
|
|
|
buf := make([]byte, bufSize)
|
2016-10-29 21:44:44 +02:00
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
tmpPartPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, uploadID+"."+mustGetUUID()+"."+strconv.Itoa(partID))
|
2018-04-06 00:04:40 +02:00
|
|
|
bytesWritten, err := fsCreateFile(ctx, tmpPartPath, data, buf, data.Size())
|
2018-01-31 22:17:24 +01:00
|
|
|
if err != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
fsRemoveFile(ctx, tmpPartPath)
|
2018-01-31 22:17:24 +01:00
|
|
|
return pi, toObjectErr(err, minioMetaTmpBucket, tmpPartPath)
|
2016-07-19 04:06:48 +02:00
|
|
|
}
|
2016-12-03 20:53:12 +01:00
|
|
|
|
2016-07-19 04:06:48 +02:00
|
|
|
// Should return IncompleteBody{} error when reader has fewer
|
|
|
|
// bytes than specified in request header.
|
2017-09-19 21:40:27 +02:00
|
|
|
if bytesWritten < data.Size() {
|
2018-04-06 00:04:40 +02:00
|
|
|
fsRemoveFile(ctx, tmpPartPath)
|
|
|
|
return pi, IncompleteBody{}
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
|
|
|
|
2016-10-20 07:52:03 +02:00
|
|
|
// Delete temporary part in case of failure. If
|
|
|
|
// PutObjectPart succeeds then there would be nothing to
|
2018-01-31 22:17:24 +01:00
|
|
|
// delete in which case we just ignore the error.
|
2018-04-06 00:04:40 +02:00
|
|
|
defer fsRemoveFile(ctx, tmpPartPath)
|
2018-11-15 02:36:41 +01:00
|
|
|
etag := r.MD5CurrentHexString()
|
2018-01-31 22:17:24 +01:00
|
|
|
|
|
|
|
if etag == "" {
|
|
|
|
etag = GenETag()
|
2017-08-31 20:29:22 +02:00
|
|
|
}
|
2018-11-15 02:36:41 +01:00
|
|
|
|
2018-09-28 05:36:17 +02:00
|
|
|
partPath := pathJoin(uploadIDDir, fs.encodePartFile(partID, etag, data.ActualSize()))
|
2017-01-17 02:05:00 +01:00
|
|
|
|
2019-07-23 07:36:15 +02:00
|
|
|
// Make sure not to create parent directories if they don't exist - the upload might have been aborted.
|
|
|
|
if err = fsSimpleRenameFile(ctx, tmpPartPath, partPath); err != nil {
|
|
|
|
if err == errFileNotFound || err == errFileAccessDenied {
|
|
|
|
return pi, InvalidUploadID{UploadID: uploadID}
|
|
|
|
}
|
2017-06-22 04:53:09 +02:00
|
|
|
return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
2017-01-17 02:05:00 +01:00
|
|
|
|
2018-04-06 00:04:40 +02:00
|
|
|
go fs.backgroundAppend(ctx, bucket, object, uploadID)
|
2017-10-22 07:30:34 +02:00
|
|
|
|
2018-04-06 00:04:40 +02:00
|
|
|
fi, err := fsStatFile(ctx, partPath)
|
2017-01-31 18:38:34 +01:00
|
|
|
if err != nil {
|
2018-01-31 22:17:24 +01:00
|
|
|
return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
2017-01-31 18:38:34 +01:00
|
|
|
return PartInfo{
|
|
|
|
PartNumber: partID,
|
|
|
|
LastModified: fi.ModTime(),
|
2018-01-31 22:17:24 +01:00
|
|
|
ETag: etag,
|
2017-01-31 18:38:34 +01:00
|
|
|
Size: fi.Size(),
|
2018-09-28 05:36:17 +02:00
|
|
|
ActualSize: data.ActualSize(),
|
2017-01-31 18:38:34 +01:00
|
|
|
}, nil
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
|
|
|
|
2017-10-06 01:08:25 +02:00
|
|
|
// ListObjectParts - lists all previously uploaded parts for a given
|
|
|
|
// object and uploadID. Takes additional input of part-number-marker
|
|
|
|
// to indicate where the listing should begin from.
|
|
|
|
//
|
|
|
|
// Implements S3 compatible ListObjectParts API. The resulting
|
|
|
|
// ListPartsInfo structure is unmarshalled directly into XML and
|
|
|
|
// replied back to the client.
|
2019-01-05 23:16:43 +01:00
|
|
|
func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int, opts ObjectOptions) (result ListPartsInfo, e error) {
|
2018-04-06 00:04:40 +02:00
|
|
|
if err := checkListPartsArgs(ctx, bucket, object, fs); err != nil {
|
|
|
|
return result, toObjectErr(err)
|
2016-06-02 21:18:56 +02:00
|
|
|
}
|
2018-01-31 22:17:24 +01:00
|
|
|
result.Bucket = bucket
|
|
|
|
result.Object = object
|
|
|
|
result.UploadID = uploadID
|
|
|
|
result.MaxParts = maxParts
|
|
|
|
result.PartNumberMarker = partNumberMarker
|
2016-08-31 20:39:08 +02:00
|
|
|
|
2017-08-04 19:45:57 +02:00
|
|
|
// Check if bucket exists
|
2018-04-06 00:04:40 +02:00
|
|
|
if _, err := fs.statBucketDir(ctx, bucket); err != nil {
|
|
|
|
return result, toObjectErr(err, bucket)
|
2016-05-04 01:10:24 +02:00
|
|
|
}
|
2016-05-21 05:48:47 +02:00
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
uploadIDDir := fs.getUploadIDDir(bucket, object, uploadID)
|
2018-09-28 05:36:17 +02:00
|
|
|
if _, err := fsStatFile(ctx, pathJoin(uploadIDDir, fs.metaJSONFile)); err != nil {
|
2018-04-10 18:36:37 +02:00
|
|
|
if err == errFileNotFound || err == errFileAccessDenied {
|
2018-04-06 00:04:40 +02:00
|
|
|
return result, InvalidUploadID{UploadID: uploadID}
|
2018-01-31 22:17:24 +01:00
|
|
|
}
|
2018-04-06 00:04:40 +02:00
|
|
|
return result, toObjectErr(err, bucket, object)
|
2017-10-06 01:08:25 +02:00
|
|
|
}
|
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
entries, err := readDir(uploadIDDir)
|
2017-01-17 02:05:00 +01:00
|
|
|
if err != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
return result, toObjectErr(err, bucket)
|
2016-10-29 21:44:44 +02:00
|
|
|
}
|
2017-01-17 02:05:00 +01:00
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
partsMap := make(map[int]string)
|
|
|
|
for _, entry := range entries {
|
2018-03-28 23:14:06 +02:00
|
|
|
if entry == fs.metaJSONFile {
|
2018-01-31 22:17:24 +01:00
|
|
|
continue
|
|
|
|
}
|
2018-09-28 05:36:17 +02:00
|
|
|
partNumber, etag1, _, derr := fs.decodePartFile(entry)
|
2018-03-01 20:37:57 +01:00
|
|
|
if derr != nil {
|
2019-04-13 00:51:32 +02:00
|
|
|
// Skip part files whose name don't match expected format. These could be backend filesystem specific files.
|
|
|
|
continue
|
2018-01-31 22:17:24 +01:00
|
|
|
}
|
|
|
|
etag2, ok := partsMap[partNumber]
|
|
|
|
if !ok {
|
|
|
|
partsMap[partNumber] = etag1
|
|
|
|
continue
|
|
|
|
}
|
2018-09-28 05:36:17 +02:00
|
|
|
stat1, serr := fsStatFile(ctx, pathJoin(uploadIDDir, getPartFile(entries, partNumber, etag1)))
|
2018-03-01 20:37:57 +01:00
|
|
|
if serr != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
return result, toObjectErr(serr)
|
2018-01-31 22:17:24 +01:00
|
|
|
}
|
2018-09-28 05:36:17 +02:00
|
|
|
stat2, serr := fsStatFile(ctx, pathJoin(uploadIDDir, getPartFile(entries, partNumber, etag2)))
|
2018-03-01 20:37:57 +01:00
|
|
|
if serr != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
return result, toObjectErr(serr)
|
2018-01-31 22:17:24 +01:00
|
|
|
}
|
|
|
|
if stat1.ModTime().After(stat2.ModTime()) {
|
|
|
|
partsMap[partNumber] = etag1
|
|
|
|
}
|
|
|
|
}
|
2018-03-01 20:37:57 +01:00
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
var parts []PartInfo
|
2018-09-28 05:36:17 +02:00
|
|
|
var actualSize int64
|
2018-01-31 22:17:24 +01:00
|
|
|
for partNumber, etag := range partsMap {
|
2018-09-28 05:36:17 +02:00
|
|
|
partFile := getPartFile(entries, partNumber, etag)
|
|
|
|
if partFile == "" {
|
|
|
|
return result, InvalidPart{}
|
|
|
|
}
|
|
|
|
// Read the actualSize from the pathFileName.
|
|
|
|
subParts := strings.Split(partFile, ".")
|
|
|
|
actualSize, err = strconv.ParseInt(subParts[len(subParts)-1], 10, 64)
|
|
|
|
if err != nil {
|
|
|
|
return result, InvalidPart{}
|
|
|
|
}
|
|
|
|
parts = append(parts, PartInfo{PartNumber: partNumber, ETag: etag, ActualSize: actualSize})
|
2018-01-31 22:17:24 +01:00
|
|
|
}
|
2018-02-28 04:03:00 +01:00
|
|
|
sort.Slice(parts, func(i int, j int) bool {
|
2018-01-31 22:17:24 +01:00
|
|
|
return parts[i].PartNumber < parts[j].PartNumber
|
|
|
|
})
|
|
|
|
i := 0
|
|
|
|
if partNumberMarker != 0 {
|
|
|
|
// If the marker was set, skip the entries till the marker.
|
|
|
|
for _, part := range parts {
|
|
|
|
i++
|
|
|
|
if part.PartNumber == partNumberMarker {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
partsCount := 0
|
|
|
|
for partsCount < maxParts && i < len(parts) {
|
|
|
|
result.Parts = append(result.Parts, parts[i])
|
|
|
|
i++
|
|
|
|
partsCount++
|
|
|
|
}
|
|
|
|
if i < len(parts) {
|
|
|
|
result.IsTruncated = true
|
|
|
|
if partsCount != 0 {
|
|
|
|
result.NextPartNumberMarker = result.Parts[partsCount-1].PartNumber
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for i, part := range result.Parts {
|
2018-03-01 20:37:57 +01:00
|
|
|
var stat os.FileInfo
|
2019-08-19 20:35:52 +02:00
|
|
|
stat, err = fsStatFile(ctx, pathJoin(uploadIDDir,
|
|
|
|
fs.encodePartFile(part.PartNumber, part.ETag, part.ActualSize)))
|
2018-01-31 22:17:24 +01:00
|
|
|
if err != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
return result, toObjectErr(err)
|
2018-01-31 22:17:24 +01:00
|
|
|
}
|
|
|
|
result.Parts[i].LastModified = stat.ModTime()
|
2018-09-28 05:36:17 +02:00
|
|
|
result.Parts[i].Size = part.ActualSize
|
2018-01-31 22:17:24 +01:00
|
|
|
}
|
2018-03-01 20:37:57 +01:00
|
|
|
|
2018-03-28 23:14:06 +02:00
|
|
|
fsMetaBytes, err := ioutil.ReadFile(pathJoin(uploadIDDir, fs.metaJSONFile))
|
2018-03-01 20:37:57 +01:00
|
|
|
if err != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
return result, err
|
2018-03-01 20:37:57 +01:00
|
|
|
}
|
|
|
|
|
2019-09-06 00:51:27 +02:00
|
|
|
var fsMeta fsMetaV1
|
|
|
|
var json = jsoniter.ConfigCompatibleWithStandardLibrary
|
|
|
|
if err = json.Unmarshal(fsMetaBytes, &fsMeta); err != nil {
|
2019-08-19 20:35:52 +02:00
|
|
|
return result, err
|
|
|
|
}
|
|
|
|
|
2019-09-06 00:51:27 +02:00
|
|
|
result.UserDefined = fsMeta.Meta
|
2018-01-31 22:17:24 +01:00
|
|
|
return result, nil
|
2016-10-29 21:44:44 +02:00
|
|
|
}
|
|
|
|
|
2016-06-02 21:18:56 +02:00
|
|
|
// CompleteMultipartUpload - completes an ongoing multipart
|
|
|
|
// transaction after receiving all the parts indicated by the client.
|
|
|
|
// Returns an md5sum calculated by concatenating all the individual
|
|
|
|
// md5sums of all the parts.
|
|
|
|
//
|
|
|
|
// Implements S3 compatible Complete multipart API.
|
2018-11-15 02:36:41 +01:00
|
|
|
func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart, opts ObjectOptions) (oi ObjectInfo, e error) {
|
2018-09-28 05:36:17 +02:00
|
|
|
|
|
|
|
var actualSize int64
|
|
|
|
|
2018-04-06 00:04:40 +02:00
|
|
|
if err := checkCompleteMultipartArgs(ctx, bucket, object, fs); err != nil {
|
2018-01-31 22:17:24 +01:00
|
|
|
return oi, toObjectErr(err)
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
2016-06-02 21:18:56 +02:00
|
|
|
|
2017-05-10 02:46:46 +02:00
|
|
|
// Check if an object is present as one of the parent dir.
|
2018-04-06 00:04:40 +02:00
|
|
|
if fs.parentDirIsObject(ctx, bucket, pathutil.Dir(object)) {
|
2019-03-20 21:06:53 +01:00
|
|
|
return oi, toObjectErr(errFileParentIsFile, bucket, object)
|
2017-05-10 02:46:46 +02:00
|
|
|
}
|
|
|
|
|
2018-04-06 00:04:40 +02:00
|
|
|
if _, err := fs.statBucketDir(ctx, bucket); err != nil {
|
2017-06-22 04:53:09 +02:00
|
|
|
return oi, toObjectErr(err, bucket)
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
uploadIDDir := fs.getUploadIDDir(bucket, object, uploadID)
|
|
|
|
// Just check if the uploadID exists to avoid copy if it doesn't.
|
2018-04-06 00:04:40 +02:00
|
|
|
_, err := fsStatFile(ctx, pathJoin(uploadIDDir, fs.metaJSONFile))
|
2017-01-17 02:05:00 +01:00
|
|
|
if err != nil {
|
2018-04-10 18:36:37 +02:00
|
|
|
if err == errFileNotFound || err == errFileAccessDenied {
|
2018-04-06 00:04:40 +02:00
|
|
|
return oi, InvalidUploadID{UploadID: uploadID}
|
2017-01-17 02:05:00 +01:00
|
|
|
}
|
2018-01-31 22:17:24 +01:00
|
|
|
return oi, toObjectErr(err, bucket, object)
|
2017-01-17 02:05:00 +01:00
|
|
|
}
|
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
// Calculate s3 compatible md5sum for complete multipart.
|
2019-05-09 03:35:40 +02:00
|
|
|
s3MD5 := getCompleteMultipartMD5(parts)
|
2016-05-21 05:48:47 +02:00
|
|
|
|
2017-10-17 21:01:28 +02:00
|
|
|
partSize := int64(-1) // Used later to ensure that all parts sizes are same.
|
2018-01-31 22:17:24 +01:00
|
|
|
|
2018-03-01 20:37:57 +01:00
|
|
|
fsMeta := fsMetaV1{}
|
|
|
|
|
|
|
|
// Allocate parts similar to incoming slice.
|
2019-01-05 23:16:43 +01:00
|
|
|
fsMeta.Parts = make([]ObjectPartInfo, len(parts))
|
2018-03-01 20:37:57 +01:00
|
|
|
|
2018-09-28 05:36:17 +02:00
|
|
|
entries, err := readDir(uploadIDDir)
|
|
|
|
if err != nil {
|
|
|
|
logger.GetReqInfo(ctx).AppendTags("uploadIDDir", uploadIDDir)
|
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
return oi, err
|
|
|
|
}
|
|
|
|
|
2019-04-01 21:19:52 +02:00
|
|
|
// ensure that part ETag is canonicalized to strip off extraneous quotes
|
|
|
|
for i := range parts {
|
|
|
|
parts[i].ETag = canonicalizeETag(parts[i].ETag)
|
|
|
|
}
|
|
|
|
|
2018-09-28 05:36:17 +02:00
|
|
|
// Save consolidated actual size.
|
|
|
|
var objectActualSize int64
|
2017-07-07 17:41:29 +02:00
|
|
|
// Validate all parts and then commit to disk.
|
|
|
|
for i, part := range parts {
|
2018-09-28 05:36:17 +02:00
|
|
|
partFile := getPartFile(entries, part.PartNumber, part.ETag)
|
|
|
|
if partFile == "" {
|
|
|
|
return oi, InvalidPart{
|
|
|
|
PartNumber: part.PartNumber,
|
|
|
|
GotETag: part.ETag,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Read the actualSize from the pathFileName.
|
|
|
|
subParts := strings.Split(partFile, ".")
|
|
|
|
actualSize, err = strconv.ParseInt(subParts[len(subParts)-1], 10, 64)
|
|
|
|
if err != nil {
|
|
|
|
return oi, InvalidPart{
|
|
|
|
PartNumber: part.PartNumber,
|
|
|
|
GotETag: part.ETag,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
partPath := pathJoin(uploadIDDir, partFile)
|
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
var fi os.FileInfo
|
2018-04-06 00:04:40 +02:00
|
|
|
fi, err = fsStatFile(ctx, partPath)
|
2018-01-31 22:17:24 +01:00
|
|
|
if err != nil {
|
2018-04-10 18:36:37 +02:00
|
|
|
if err == errFileNotFound || err == errFileAccessDenied {
|
2018-04-06 00:04:40 +02:00
|
|
|
return oi, InvalidPart{}
|
2018-01-31 22:17:24 +01:00
|
|
|
}
|
2018-04-06 00:04:40 +02:00
|
|
|
return oi, err
|
2017-07-07 17:41:29 +02:00
|
|
|
}
|
2018-01-31 22:17:24 +01:00
|
|
|
if partSize == -1 {
|
2018-09-28 05:36:17 +02:00
|
|
|
partSize = actualSize
|
2018-01-31 22:17:24 +01:00
|
|
|
}
|
2018-03-01 20:37:57 +01:00
|
|
|
|
2019-01-05 23:16:43 +01:00
|
|
|
fsMeta.Parts[i] = ObjectPartInfo{
|
2018-09-28 05:36:17 +02:00
|
|
|
Number: part.PartNumber,
|
|
|
|
ETag: part.ETag,
|
|
|
|
Size: fi.Size(),
|
|
|
|
ActualSize: actualSize,
|
2018-03-01 20:37:57 +01:00
|
|
|
}
|
|
|
|
|
2018-09-28 05:36:17 +02:00
|
|
|
// Consolidate the actual size.
|
|
|
|
objectActualSize += actualSize
|
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
if i == len(parts)-1 {
|
|
|
|
break
|
2017-07-07 17:41:29 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// All parts except the last part has to be atleast 5MB.
|
2018-09-28 05:36:17 +02:00
|
|
|
if !isMinAllowedPartSize(actualSize) {
|
2018-07-25 06:31:03 +02:00
|
|
|
return oi, PartTooSmall{
|
2017-07-07 17:41:29 +02:00
|
|
|
PartNumber: part.PartNumber,
|
2018-09-28 05:36:17 +02:00
|
|
|
PartSize: actualSize,
|
2017-07-07 17:41:29 +02:00
|
|
|
PartETag: part.ETag,
|
2018-04-06 00:04:40 +02:00
|
|
|
}
|
2017-10-17 21:01:28 +02:00
|
|
|
}
|
2017-07-07 17:41:29 +02:00
|
|
|
}
|
|
|
|
|
2016-11-23 12:04:04 +01:00
|
|
|
appendFallback := true // In case background-append did not append the required parts.
|
2018-01-31 22:17:24 +01:00
|
|
|
appendFilePath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, fmt.Sprintf("%s.%s", uploadID, mustGetUUID()))
|
|
|
|
|
|
|
|
// Most of the times appendFile would already be fully appended by now. We call fs.backgroundAppend()
|
|
|
|
// to take care of the following corner case:
|
|
|
|
// 1. The last PutObjectPart triggers go-routine fs.backgroundAppend, this go-routine has not started yet.
|
|
|
|
// 2. Now CompleteMultipartUpload gets called which sees that lastPart is not appended and starts appending
|
|
|
|
// from the beginning
|
2018-04-06 00:04:40 +02:00
|
|
|
fs.backgroundAppend(ctx, bucket, object, uploadID)
|
2018-01-31 22:17:24 +01:00
|
|
|
|
|
|
|
fs.appendFileMapMu.Lock()
|
|
|
|
file := fs.appendFileMap[uploadID]
|
|
|
|
delete(fs.appendFileMap, uploadID)
|
|
|
|
fs.appendFileMapMu.Unlock()
|
|
|
|
|
|
|
|
if file != nil {
|
|
|
|
file.Lock()
|
|
|
|
defer file.Unlock()
|
|
|
|
// Verify that appendFile has all the parts.
|
|
|
|
if len(file.parts) == len(parts) {
|
|
|
|
for i := range parts {
|
|
|
|
if parts[i].ETag != file.parts[i].ETag {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
if parts[i].PartNumber != file.parts[i].PartNumber {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
if i == len(parts)-1 {
|
|
|
|
appendFilePath = file.filePath
|
|
|
|
appendFallback = false
|
|
|
|
}
|
2016-11-21 08:42:53 +01:00
|
|
|
}
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
2016-11-21 08:42:53 +01:00
|
|
|
}
|
2016-11-01 08:56:03 +01:00
|
|
|
|
2016-11-21 08:42:53 +01:00
|
|
|
if appendFallback {
|
2018-10-01 18:50:09 +02:00
|
|
|
if file != nil {
|
|
|
|
fsRemoveFile(ctx, file.filePath)
|
|
|
|
}
|
2017-07-07 17:41:29 +02:00
|
|
|
for _, part := range parts {
|
2018-09-28 05:36:17 +02:00
|
|
|
partPath := getPartFile(entries, part.PartNumber, part.ETag)
|
2018-10-01 18:50:09 +02:00
|
|
|
if err = mioutil.AppendFile(appendFilePath, pathJoin(uploadIDDir, partPath)); err != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
return oi, toObjectErr(err)
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
// Hold write lock on the object.
|
2019-11-13 21:17:45 +01:00
|
|
|
destLock := fs.NewNSLock(ctx, bucket, object)
|
2018-01-31 22:17:24 +01:00
|
|
|
if err = destLock.GetLock(globalObjectTimeout); err != nil {
|
|
|
|
return oi, err
|
|
|
|
}
|
|
|
|
defer destLock.Unlock()
|
2018-03-28 23:14:06 +02:00
|
|
|
fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile)
|
2018-01-31 22:17:24 +01:00
|
|
|
metaFile, err := fs.rwPool.Create(fsMetaPath)
|
|
|
|
if err != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
return oi, toObjectErr(err, bucket, object)
|
2018-01-31 22:17:24 +01:00
|
|
|
}
|
2018-02-01 10:57:12 +01:00
|
|
|
defer metaFile.Close()
|
2016-07-22 02:31:14 +02:00
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
// Read saved fs metadata for ongoing multipart.
|
2018-03-28 23:14:06 +02:00
|
|
|
fsMetaBuf, err := ioutil.ReadFile(pathJoin(uploadIDDir, fs.metaJSONFile))
|
2018-01-31 22:17:24 +01:00
|
|
|
if err != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
return oi, toObjectErr(err, bucket, object)
|
2018-01-31 22:17:24 +01:00
|
|
|
}
|
|
|
|
err = json.Unmarshal(fsMetaBuf, &fsMeta)
|
|
|
|
if err != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
return oi, toObjectErr(err, bucket, object)
|
2018-01-31 22:17:24 +01:00
|
|
|
}
|
2016-11-12 01:36:07 +01:00
|
|
|
// Save additional metadata.
|
|
|
|
if len(fsMeta.Meta) == 0 {
|
|
|
|
fsMeta.Meta = make(map[string]string)
|
|
|
|
}
|
2017-05-14 21:05:51 +02:00
|
|
|
fsMeta.Meta["etag"] = s3MD5
|
2018-09-28 05:36:17 +02:00
|
|
|
// Save consolidated actual size.
|
|
|
|
fsMeta.Meta[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10)
|
2017-01-17 02:05:00 +01:00
|
|
|
if _, err = fsMeta.WriteTo(metaFile); err != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
return oi, toObjectErr(err, bucket, object)
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
|
|
|
|
2018-03-28 01:44:45 +02:00
|
|
|
// Deny if WORM is enabled
|
2019-12-17 19:13:12 +01:00
|
|
|
if _, ok := isWORMEnabled(bucket); ok {
|
2019-11-20 22:18:09 +01:00
|
|
|
if _, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object)); err == nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
return ObjectInfo{}, ObjectAlreadyExists{Bucket: bucket, Object: object}
|
2018-03-28 01:44:45 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-06 00:04:40 +02:00
|
|
|
err = fsRenameFile(ctx, appendFilePath, pathJoin(fs.fsPath, bucket, object))
|
2017-09-28 17:09:28 +02:00
|
|
|
if err != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
return oi, toObjectErr(err, bucket, object)
|
2017-01-17 04:23:43 +01:00
|
|
|
}
|
2018-04-06 00:04:40 +02:00
|
|
|
fsRemoveAll(ctx, uploadIDDir)
|
2018-12-19 20:27:10 +01:00
|
|
|
// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
|
|
|
|
fsRemoveDir(ctx, fs.getMultipartSHADir(bucket, object))
|
2018-04-06 00:04:40 +02:00
|
|
|
fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object))
|
2017-01-17 04:23:43 +01:00
|
|
|
if err != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
return oi, toObjectErr(err, bucket, object)
|
2016-06-03 00:54:00 +02:00
|
|
|
}
|
|
|
|
|
2017-01-17 04:23:43 +01:00
|
|
|
return fsMeta.ToObjectInfo(bucket, object, fi), nil
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
|
|
|
|
2016-06-02 21:18:56 +02:00
|
|
|
// AbortMultipartUpload - aborts an ongoing multipart operation
|
|
|
|
// signified by the input uploadID. This is an atomic operation
|
|
|
|
// doesn't require clients to initiate multiple such requests.
|
|
|
|
//
|
|
|
|
// All parts are purged from all disks and reference to the uploadID
|
|
|
|
// would be removed from the system, rollback is not possible on this
|
|
|
|
// operation.
|
|
|
|
//
|
|
|
|
// Implements S3 compatible Abort multipart API, slight difference is
|
|
|
|
// that this is an atomic idempotent operation. Subsequent calls have
|
|
|
|
// no affect and further requests to the same uploadID would not be
|
|
|
|
// honored.
|
2018-03-14 20:01:47 +01:00
|
|
|
func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error {
|
2018-04-06 00:04:40 +02:00
|
|
|
if err := checkAbortMultipartArgs(ctx, bucket, object, fs); err != nil {
|
2016-12-02 08:15:17 +01:00
|
|
|
return err
|
2016-06-02 21:18:56 +02:00
|
|
|
}
|
|
|
|
|
2018-04-06 00:04:40 +02:00
|
|
|
if _, err := fs.statBucketDir(ctx, bucket); err != nil {
|
|
|
|
return toObjectErr(err, bucket)
|
2017-01-17 02:05:00 +01:00
|
|
|
}
|
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
fs.appendFileMapMu.Lock()
|
2018-10-22 16:36:30 +02:00
|
|
|
// Remove file in tmp folder
|
|
|
|
file := fs.appendFileMap[uploadID]
|
|
|
|
if file != nil {
|
|
|
|
fsRemoveFile(ctx, file.filePath)
|
|
|
|
}
|
2018-01-31 22:17:24 +01:00
|
|
|
delete(fs.appendFileMap, uploadID)
|
|
|
|
fs.appendFileMapMu.Unlock()
|
2017-09-28 17:09:28 +02:00
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
uploadIDDir := fs.getUploadIDDir(bucket, object, uploadID)
|
|
|
|
// Just check if the uploadID exists to avoid copy if it doesn't.
|
2018-04-06 00:04:40 +02:00
|
|
|
_, err := fsStatFile(ctx, pathJoin(uploadIDDir, fs.metaJSONFile))
|
2017-01-17 02:05:00 +01:00
|
|
|
if err != nil {
|
2018-04-10 18:36:37 +02:00
|
|
|
if err == errFileNotFound || err == errFileAccessDenied {
|
2018-04-06 00:04:40 +02:00
|
|
|
return InvalidUploadID{UploadID: uploadID}
|
2017-01-17 02:05:00 +01:00
|
|
|
}
|
2018-04-06 00:04:40 +02:00
|
|
|
return toObjectErr(err, bucket, object)
|
2017-01-17 02:05:00 +01:00
|
|
|
}
|
2018-10-22 16:36:30 +02:00
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
// Ignore the error returned as Windows fails to remove directory if a file in it
|
|
|
|
// is Open()ed by the backgroundAppend()
|
2018-04-06 00:04:40 +02:00
|
|
|
fsRemoveAll(ctx, uploadIDDir)
|
2017-01-17 02:05:00 +01:00
|
|
|
|
2018-10-22 16:36:30 +02:00
|
|
|
// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
|
|
|
|
fsRemoveDir(ctx, fs.getMultipartSHADir(bucket, object))
|
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
return nil
|
|
|
|
}
|
2016-06-02 21:18:56 +02:00
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
// Removes multipart uploads if any older than `expiry` duration
|
|
|
|
// on all buckets for every `cleanupInterval`, this function is
|
|
|
|
// blocking and should be run in a go-routine.
|
2018-04-06 00:04:40 +02:00
|
|
|
func (fs *FSObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh chan struct{}) {
|
2018-01-31 22:17:24 +01:00
|
|
|
ticker := time.NewTicker(cleanupInterval)
|
2018-05-04 19:43:20 +02:00
|
|
|
defer ticker.Stop()
|
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-doneCh:
|
|
|
|
return
|
|
|
|
case <-ticker.C:
|
|
|
|
now := time.Now()
|
|
|
|
entries, err := readDir(pathJoin(fs.fsPath, minioMetaMultipartBucket))
|
|
|
|
if err != nil {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
for _, entry := range entries {
|
|
|
|
uploadIDs, err := readDir(pathJoin(fs.fsPath, minioMetaMultipartBucket, entry))
|
|
|
|
if err != nil {
|
|
|
|
continue
|
|
|
|
}
|
2019-10-16 20:57:52 +02:00
|
|
|
|
|
|
|
// Remove the trailing slash separator
|
|
|
|
for i := range uploadIDs {
|
|
|
|
uploadIDs[i] = strings.TrimSuffix(uploadIDs[i], SlashSeparator)
|
|
|
|
}
|
|
|
|
|
2018-01-31 22:17:24 +01:00
|
|
|
for _, uploadID := range uploadIDs {
|
2018-04-06 00:04:40 +02:00
|
|
|
fi, err := fsStatDir(ctx, pathJoin(fs.fsPath, minioMetaMultipartBucket, entry, uploadID))
|
2018-01-31 22:17:24 +01:00
|
|
|
if err != nil {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if now.Sub(fi.ModTime()) > expiry {
|
2018-04-06 00:04:40 +02:00
|
|
|
fsRemoveAll(ctx, pathJoin(fs.fsPath, minioMetaMultipartBucket, entry, uploadID))
|
2018-12-19 20:27:10 +01:00
|
|
|
// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
|
|
|
|
fsRemoveDir(ctx, pathJoin(fs.fsPath, minioMetaMultipartBucket, entry))
|
|
|
|
|
2019-10-16 20:57:52 +02:00
|
|
|
// Remove uploadID from the append file map and its corresponding temporary file
|
|
|
|
fs.appendFileMapMu.Lock()
|
|
|
|
bgAppend, ok := fs.appendFileMap[uploadID]
|
|
|
|
if ok {
|
|
|
|
err := fsRemoveFile(ctx, bgAppend.filePath)
|
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
delete(fs.appendFileMap, uploadID)
|
|
|
|
}
|
|
|
|
fs.appendFileMapMu.Unlock()
|
2018-01-31 22:17:24 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2016-06-02 21:18:56 +02:00
|
|
|
}
|
2016-05-03 01:57:31 +02:00
|
|
|
}
|