2016-06-02 01:43:31 +02:00
/ *
2019-04-09 20:39:42 +02:00
* MinIO Cloud Storage , ( C ) 2016 , 2017 , 2018 MinIO , Inc .
2016-06-02 01:43:31 +02:00
*
* Licensed under the Apache License , Version 2.0 ( the "License" ) ;
* you may not use this file except in compliance with the License .
* You may obtain a copy of the License at
*
* http : //www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing , software
* distributed under the License is distributed on an "AS IS" BASIS ,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
* See the License for the specific language governing permissions and
* limitations under the License .
* /
2016-08-19 01:23:42 +02:00
package cmd
2016-05-21 05:48:47 +02:00
import (
2018-09-21 04:22:09 +02:00
"bytes"
2018-03-14 20:01:47 +01:00
"context"
2020-03-03 01:29:30 +01:00
"fmt"
2016-05-21 05:48:47 +02:00
"io"
2018-09-21 04:22:09 +02:00
"net/http"
2016-05-21 05:48:47 +02:00
"path"
2020-03-11 16:56:36 +01:00
"sync"
2016-05-21 05:48:47 +02:00
2020-05-05 23:18:13 +02:00
"github.com/minio/minio-go/v6/pkg/tags"
2019-10-07 07:50:24 +02:00
xhttp "github.com/minio/minio/cmd/http"
2018-04-06 00:04:40 +02:00
"github.com/minio/minio/cmd/logger"
2016-05-21 05:48:47 +02:00
"github.com/minio/minio/pkg/mimedb"
2019-10-14 18:44:51 +02:00
"github.com/minio/minio/pkg/sync/errgroup"
2016-05-21 05:48:47 +02:00
)
2016-11-21 01:57:12 +01:00
// list all errors which can be ignored in object operations.
2017-02-01 20:16:17 +01:00
var objectOpIgnoredErrs = append ( baseIgnoredErrs , errDiskAccessDenied )
2016-11-21 01:57:12 +01:00
2018-01-30 03:43:13 +01:00
// putObjectDir hints the bottom layer to create a new directory.
2018-04-06 00:04:40 +02:00
func ( xl xlObjects ) putObjectDir ( ctx context . Context , bucket , object string , writeQuorum int ) error {
2019-10-14 18:44:51 +02:00
storageDisks := xl . getDisks ( )
g := errgroup . WithNErrs ( len ( storageDisks ) )
2018-01-30 03:43:13 +01:00
// Prepare object creation in all disks
2019-10-14 18:44:51 +02:00
for index := range storageDisks {
if storageDisks [ index ] == nil {
2018-01-30 03:43:13 +01:00
continue
}
2019-10-14 18:44:51 +02:00
index := index
g . Go ( func ( ) error {
err := storageDisks [ index ] . MakeVol ( pathJoin ( bucket , object ) )
if err != nil && err != errVolumeExists {
return err
2018-01-30 03:43:13 +01:00
}
2019-10-14 18:44:51 +02:00
return nil
} , index )
2018-01-30 03:43:13 +01:00
}
2019-10-14 18:44:51 +02:00
return reduceWriteQuorumErrs ( ctx , g . Wait ( ) , objectOpIgnoredErrs , writeQuorum )
2018-01-30 03:43:13 +01:00
}
2016-05-21 05:48:47 +02:00
/// Object Operations
2016-12-27 01:29:26 +01:00
// CopyObject - copy object source object to destination object.
// if source object and destination object are same we only
// update metadata.
2018-09-10 18:42:43 +02:00
func ( xl xlObjects ) CopyObject ( ctx context . Context , srcBucket , srcObject , dstBucket , dstObject string , srcInfo ObjectInfo , srcOpts , dstOpts ObjectOptions ) ( oi ObjectInfo , e error ) {
2018-02-24 00:07:21 +01:00
cpSrcDstSame := isStringEqual ( pathJoin ( srcBucket , srcObject ) , pathJoin ( dstBucket , dstObject ) )
2018-01-13 05:34:52 +01:00
2019-05-09 03:35:40 +02:00
// Check if this request is only metadata update.
if cpSrcDstSame {
2020-04-27 19:06:21 +02:00
defer ObjectPathUpdated ( path . Join ( dstBucket , dstObject ) )
2019-05-09 03:35:40 +02:00
// Read metadata associated with the object from all disks.
storageDisks := xl . getDisks ( )
2017-12-22 12:28:13 +01:00
2019-05-09 03:35:40 +02:00
metaArr , errs := readAllXLMetadata ( ctx , storageDisks , srcBucket , srcObject )
2017-12-22 12:28:13 +01:00
2019-05-09 03:35:40 +02:00
// get Quorum for this object
readQuorum , writeQuorum , err := objectQuorumFromMeta ( ctx , xl , metaArr , errs )
if err != nil {
return oi , toObjectErr ( err , srcBucket , srcObject )
}
2016-12-27 01:29:26 +01:00
2019-05-09 03:35:40 +02:00
if reducedErr := reduceReadQuorumErrs ( ctx , errs , objectOpIgnoredErrs , readQuorum ) ; reducedErr != nil {
return oi , toObjectErr ( reducedErr , srcBucket , srcObject )
}
2016-12-27 01:29:26 +01:00
2019-05-09 03:35:40 +02:00
// List all online disks.
_ , modTime := listOnlineDisks ( storageDisks , metaArr , errs )
2016-12-27 01:29:26 +01:00
2019-05-09 03:35:40 +02:00
// Pick latest valid metadata.
xlMeta , err := pickValidXLMeta ( ctx , metaArr , modTime , readQuorum )
if err != nil {
return oi , toObjectErr ( err , srcBucket , srcObject )
}
2016-12-27 01:29:26 +01:00
// Update `xl.json` content on each disks.
2018-03-03 02:24:02 +01:00
for index := range metaArr {
metaArr [ index ] . Meta = srcInfo . UserDefined
2018-03-09 19:50:39 +01:00
metaArr [ index ] . Meta [ "etag" ] = srcInfo . ETag
2016-12-27 01:29:26 +01:00
}
2018-03-03 02:24:02 +01:00
var onlineDisks [ ] StorageAPI
2016-12-27 01:29:26 +01:00
tempObj := mustGetUUID ( )
2019-04-25 16:33:26 +02:00
// Cleanup in case of xl.json writing failure
defer xl . deleteObject ( ctx , minioMetaTmpBucket , tempObj , writeQuorum , false )
2016-12-27 01:29:26 +01:00
// Write unique `xl.json` for each disk.
2018-04-06 00:04:40 +02:00
if onlineDisks , err = writeUniqueXLMetadata ( ctx , storageDisks , minioMetaTmpBucket , tempObj , metaArr , writeQuorum ) ; err != nil {
2017-06-22 04:53:09 +02:00
return oi , toObjectErr ( err , srcBucket , srcObject )
2016-12-27 01:29:26 +01:00
}
2019-05-09 03:35:40 +02:00
2016-12-27 01:29:26 +01:00
// Rename atomically `xl.json` from tmp location to destination for each disk.
2018-04-06 00:04:40 +02:00
if _ , err = renameXLMetadata ( ctx , onlineDisks , minioMetaTmpBucket , tempObj , srcBucket , srcObject , writeQuorum ) ; err != nil {
2017-06-22 04:53:09 +02:00
return oi , toObjectErr ( err , srcBucket , srcObject )
2016-12-27 01:29:26 +01:00
}
2019-05-09 03:35:40 +02:00
return xlMeta . ToObjectInfo ( srcBucket , srcObject ) , nil
2016-12-27 01:29:26 +01:00
}
2019-05-09 03:35:40 +02:00
putOpts := ObjectOptions { ServerSideEncryption : dstOpts . ServerSideEncryption , UserDefined : srcInfo . UserDefined }
2019-05-13 19:12:06 +02:00
return xl . PutObject ( ctx , dstBucket , dstObject , srcInfo . PutObjReader , putOpts )
2016-12-27 01:29:26 +01:00
}
2018-09-21 04:22:09 +02:00
// GetObjectNInfo - returns object info and an object
// Read(Closer). When err != nil, the returned reader is always nil.
2018-09-27 12:06:45 +02:00
func ( xl xlObjects ) GetObjectNInfo ( ctx context . Context , bucket , object string , rs * HTTPRangeSpec , h http . Header , lockType LockType , opts ObjectOptions ) ( gr * GetObjectReader , err error ) {
2018-09-21 04:22:09 +02:00
if err = checkGetObjArgs ( ctx , bucket , object ) ; err != nil {
return nil , err
}
// Handler directory request by returning a reader that
// returns no bytes.
2019-12-06 08:16:06 +01:00
if HasSuffix ( object , SlashSeparator ) {
2018-09-21 04:22:09 +02:00
var objInfo ObjectInfo
if objInfo , err = xl . getObjectInfoDir ( ctx , bucket , object ) ; err != nil {
return nil , toObjectErr ( err , bucket , object )
}
2020-04-21 07:01:59 +02:00
return NewGetObjectReaderFromReader ( bytes . NewBuffer ( nil ) , objInfo , opts )
2018-09-21 04:22:09 +02:00
}
var objInfo ObjectInfo
2020-04-21 07:01:59 +02:00
objInfo , err = xl . getObjectInfo ( ctx , bucket , object , opts )
2018-09-21 04:22:09 +02:00
if err != nil {
return nil , toObjectErr ( err , bucket , object )
}
2020-04-21 07:01:59 +02:00
fn , off , length , nErr := NewGetObjectReader ( rs , objInfo , opts )
2018-09-21 04:22:09 +02:00
if nErr != nil {
return nil , nErr
}
pr , pw := io . Pipe ( )
go func ( ) {
2018-09-27 12:06:45 +02:00
err := xl . getObject ( ctx , bucket , object , off , length , pw , "" , opts )
2018-09-21 04:22:09 +02:00
pw . CloseWithError ( err )
} ( )
2020-04-21 07:01:59 +02:00
2018-09-21 20:42:06 +02:00
// Cleanup function to cause the go routine above to exit, in
// case of incomplete read.
pipeCloser := func ( ) { pr . Close ( ) }
2018-09-21 04:22:09 +02:00
2019-03-06 21:38:41 +01:00
return fn ( pr , h , opts . CheckCopyPrecondFn , pipeCloser )
2018-09-21 04:22:09 +02:00
}
2016-06-02 01:43:31 +02:00
// GetObject - reads an object erasured coded across multiple
// disks. Supports additional parameters like offset and length
2016-12-27 01:29:26 +01:00
// which are synonymous with HTTP Range requests.
2016-06-02 01:43:31 +02:00
//
2016-12-27 01:29:26 +01:00
// startOffset indicates the starting read location of the object.
// length indicates the total length of the object.
2018-09-10 18:42:43 +02:00
func ( xl xlObjects ) GetObject ( ctx context . Context , bucket , object string , startOffset int64 , length int64 , writer io . Writer , etag string , opts ObjectOptions ) error {
return xl . getObject ( ctx , bucket , object , startOffset , length , writer , etag , opts )
2018-01-13 05:34:52 +01:00
}
// getObject wrapper for xl GetObject
2018-09-10 18:42:43 +02:00
func ( xl xlObjects ) getObject ( ctx context . Context , bucket , object string , startOffset int64 , length int64 , writer io . Writer , etag string , opts ObjectOptions ) error {
2018-01-13 05:34:52 +01:00
2018-04-06 00:04:40 +02:00
if err := checkGetObjArgs ( ctx , bucket , object ) ; err != nil {
2016-12-02 08:15:17 +01:00
return err
2016-05-21 05:48:47 +02:00
}
2016-12-21 20:29:32 +01:00
// Start offset cannot be negative.
if startOffset < 0 {
2019-10-12 03:50:54 +02:00
logger . LogIf ( ctx , errUnexpected , logger . Application )
2018-04-06 00:04:40 +02:00
return errUnexpected
2016-07-07 10:30:34 +02:00
}
2016-12-21 20:29:32 +01:00
2016-07-08 16:46:49 +02:00
// Writer cannot be nil.
if writer == nil {
2018-04-06 00:04:40 +02:00
logger . LogIf ( ctx , errUnexpected )
return errUnexpected
2016-07-08 16:46:49 +02:00
}
2016-08-31 20:39:08 +02:00
2018-01-30 03:43:13 +01:00
// If its a directory request, we return an empty body.
2019-12-06 08:16:06 +01:00
if HasSuffix ( object , SlashSeparator ) {
2018-01-30 03:43:13 +01:00
_ , err := writer . Write ( [ ] byte ( "" ) )
2018-04-06 00:04:40 +02:00
logger . LogIf ( ctx , err )
return toObjectErr ( err , bucket , object )
2018-01-30 03:43:13 +01:00
}
2016-06-01 05:23:31 +02:00
// Read metadata associated with the object from all disks.
2018-04-06 00:04:40 +02:00
metaArr , errs := readAllXLMetadata ( ctx , xl . getDisks ( ) , bucket , object )
2017-12-22 12:28:13 +01:00
// get Quorum for this object
2018-07-31 09:23:29 +02:00
readQuorum , _ , err := objectQuorumFromMeta ( ctx , xl , metaArr , errs )
2017-12-22 12:28:13 +01:00
if err != nil {
return toObjectErr ( err , bucket , object )
}
2018-04-06 00:04:40 +02:00
if reducedErr := reduceReadQuorumErrs ( ctx , errs , objectOpIgnoredErrs , readQuorum ) ; reducedErr != nil {
2016-07-20 04:24:32 +02:00
return toObjectErr ( reducedErr , bucket , object )
2016-07-09 22:01:32 +02:00
}
2016-05-26 01:42:31 +02:00
// List all online disks.
2018-02-16 02:45:57 +01:00
onlineDisks , modTime := listOnlineDisks ( xl . getDisks ( ) , metaArr , errs )
2016-05-27 04:55:48 +02:00
2016-06-17 07:26:18 +02:00
// Pick latest valid metadata.
2018-08-17 23:42:04 +02:00
xlMeta , err := pickValidXLMeta ( ctx , metaArr , modTime , readQuorum )
2016-11-21 05:56:44 +01:00
if err != nil {
return err
}
2016-07-25 07:49:27 +02:00
// Reorder online disks based on erasure distribution order.
2017-02-24 18:20:40 +01:00
onlineDisks = shuffleDisks ( onlineDisks , xlMeta . Erasure . Distribution )
2016-07-25 07:49:27 +02:00
// Reorder parts metadata based on erasure distribution order.
2017-02-24 18:20:40 +01:00
metaArr = shufflePartsMetadata ( metaArr , xlMeta . Erasure . Distribution )
2016-05-26 01:42:31 +02:00
2016-12-21 20:29:32 +01:00
// For negative length read everything.
if length < 0 {
length = xlMeta . Stat . Size - startOffset
2016-07-08 16:46:49 +02:00
}
2016-12-21 20:29:32 +01:00
// Reply back invalid range if the input offset and length fall out of range.
if startOffset > xlMeta . Stat . Size || startOffset + length > xlMeta . Stat . Size {
2019-10-12 03:50:54 +02:00
logger . LogIf ( ctx , InvalidRange { startOffset , length , xlMeta . Stat . Size } , logger . Application )
2018-04-06 00:04:40 +02:00
return InvalidRange { startOffset , length , xlMeta . Stat . Size }
2016-07-08 16:46:49 +02:00
}
2016-06-19 22:35:26 +02:00
// Get start part index and offset.
2018-04-06 00:04:40 +02:00
partIndex , partOffset , err := xlMeta . ObjectToPartOffset ( ctx , startOffset )
2016-05-21 05:48:47 +02:00
if err != nil {
2018-04-06 00:04:40 +02:00
return InvalidRange { startOffset , length , xlMeta . Stat . Size }
2016-05-21 05:48:47 +02:00
}
2016-06-01 05:23:31 +02:00
2017-01-27 19:51:02 +01:00
// Calculate endOffset according to length
endOffset := startOffset
if length > 0 {
endOffset += length - 1
}
2016-06-19 22:35:26 +02:00
// Get last part index to read given length.
2018-04-06 00:04:40 +02:00
lastPartIndex , _ , err := xlMeta . ObjectToPartOffset ( ctx , endOffset )
2016-06-19 22:35:26 +02:00
if err != nil {
2018-04-06 00:04:40 +02:00
return InvalidRange { startOffset , length , xlMeta . Stat . Size }
2016-06-19 22:35:26 +02:00
}
2017-02-24 18:20:40 +01:00
var totalBytesRead int64
2018-08-24 08:35:37 +02:00
erasure , err := NewErasure ( ctx , xlMeta . Erasure . DataBlocks , xlMeta . Erasure . ParityBlocks , xlMeta . Erasure . BlockSize )
2017-08-15 03:08:42 +02:00
if err != nil {
return toObjectErr ( err , bucket , object )
}
2018-08-07 00:14:08 +02:00
2016-06-19 22:35:26 +02:00
for ; partIndex <= lastPartIndex ; partIndex ++ {
2016-06-21 23:34:11 +02:00
if length == totalBytesRead {
break
}
2020-03-03 01:29:30 +01:00
partNumber := xlMeta . Parts [ partIndex ] . Number
2016-06-01 05:23:31 +02:00
// Save the current part name and size.
partSize := xlMeta . Parts [ partIndex ] . Size
2016-06-22 18:05:03 +02:00
2018-08-07 00:14:08 +02:00
partLength := partSize - partOffset
// partLength should be adjusted so that we don't write more data than what was requested.
if partLength > ( length - totalBytesRead ) {
partLength = length - totalBytesRead
2016-06-19 22:35:26 +02:00
}
2016-06-01 05:23:31 +02:00
2019-01-17 13:58:18 +01:00
tillOffset := erasure . ShardFileTillOffset ( partOffset , partLength , partSize )
2016-07-16 17:35:30 +02:00
// Get the checksums of the current part.
2019-01-17 13:58:18 +01:00
readers := make ( [ ] io . ReaderAt , len ( onlineDisks ) )
2018-08-07 00:14:08 +02:00
for index , disk := range onlineDisks {
2017-08-15 03:08:42 +02:00
if disk == OfflineDisk {
2016-07-25 07:49:27 +02:00
continue
}
2020-03-03 01:29:30 +01:00
checksumInfo := metaArr [ index ] . Erasure . GetChecksumInfo ( partNumber )
partPath := pathJoin ( object , fmt . Sprintf ( "part.%d" , partNumber ) )
readers [ index ] = newBitrotReader ( disk , bucket , partPath , tillOffset ,
checksumInfo . Algorithm , checksumInfo . Hash , erasure . ShardSize ( ) )
2016-07-16 17:35:30 +02:00
}
2019-01-17 13:58:18 +01:00
err := erasure . Decode ( ctx , writer , readers , partOffset , partLength , partSize )
// Note: we should not be defer'ing the following closeBitrotReaders() call as we are inside a for loop i.e if we use defer, we would accumulate a lot of open files by the time
// we return from this function.
closeBitrotReaders ( readers )
2016-06-01 05:23:31 +02:00
if err != nil {
2020-04-01 21:14:00 +02:00
if decodeHealErr , ok := err . ( * errDecodeHealRequired ) ; ok {
go deepHealObject ( pathJoin ( bucket , object ) )
err = decodeHealErr . err
}
if err != nil {
return toObjectErr ( err , bucket , object )
}
2016-06-01 05:23:31 +02:00
}
2019-01-17 13:58:18 +01:00
for i , r := range readers {
2018-08-07 00:14:08 +02:00
if r == nil {
onlineDisks [ i ] = OfflineDisk
}
}
2016-07-28 11:20:34 +02:00
// Track total bytes read from disk and written to the client.
2018-08-07 00:14:08 +02:00
totalBytesRead += partLength
2016-06-01 05:23:31 +02:00
2016-06-22 18:05:03 +02:00
// partOffset will be valid only for the first part, hence reset it to 0 for
// the remaining parts.
2016-06-01 05:23:31 +02:00
partOffset = 0
2016-06-02 01:43:31 +02:00
} // End of read all parts loop.
2016-06-01 05:23:31 +02:00
// Return success.
2016-05-29 00:13:15 +02:00
return nil
2016-05-21 05:48:47 +02:00
}
2018-01-30 03:43:13 +01:00
// getObjectInfoDir - This getObjectInfo is specific to object directory lookup.
2019-10-14 18:44:51 +02:00
func ( xl xlObjects ) getObjectInfoDir ( ctx context . Context , bucket , object string ) ( ObjectInfo , error ) {
storageDisks := xl . getDisks ( )
g := errgroup . WithNErrs ( len ( storageDisks ) )
2018-01-30 03:43:13 +01:00
// Prepare object creation in a all disks
2019-10-14 18:44:51 +02:00
for index , disk := range storageDisks {
2018-01-30 03:43:13 +01:00
if disk == nil {
continue
}
2019-10-14 18:44:51 +02:00
index := index
g . Go ( func ( ) error {
2019-04-23 23:54:28 +02:00
// Check if 'prefix' is an object on this 'disk'.
2019-10-14 18:44:51 +02:00
entries , err := storageDisks [ index ] . ListDir ( bucket , object , 1 , "" )
2019-04-23 23:54:28 +02:00
if err != nil {
2019-10-14 18:44:51 +02:00
return err
2019-04-23 23:54:28 +02:00
}
if len ( entries ) > 0 {
// Not a directory if not empty.
2019-10-14 18:44:51 +02:00
return errFileNotFound
2018-01-30 03:43:13 +01:00
}
2019-10-14 18:44:51 +02:00
return nil
} , index )
2018-01-30 03:43:13 +01:00
}
2020-03-31 18:32:16 +02:00
readQuorum := getReadQuorum ( len ( storageDisks ) )
2019-10-14 18:44:51 +02:00
err := reduceReadQuorumErrs ( ctx , g . Wait ( ) , objectOpIgnoredErrs , readQuorum )
return dirObjectInfo ( bucket , object , 0 , map [ string ] string { } ) , err
2018-01-30 03:43:13 +01:00
}
2016-06-02 01:43:31 +02:00
// GetObjectInfo - reads object metadata and replies back ObjectInfo.
2018-09-10 18:42:43 +02:00
func ( xl xlObjects ) GetObjectInfo ( ctx context . Context , bucket , object string , opts ObjectOptions ) ( oi ObjectInfo , e error ) {
2018-04-06 00:04:40 +02:00
if err := checkGetObjArgs ( ctx , bucket , object ) ; err != nil {
2017-06-22 04:53:09 +02:00
return oi , err
2016-05-21 05:48:47 +02:00
}
2016-08-31 20:39:08 +02:00
2019-12-06 08:16:06 +01:00
if HasSuffix ( object , SlashSeparator ) {
2019-04-23 23:54:28 +02:00
info , err := xl . getObjectInfoDir ( ctx , bucket , object )
if err != nil {
return oi , toObjectErr ( err , bucket , object )
2018-02-02 09:34:15 +01:00
}
2019-04-23 23:54:28 +02:00
return info , nil
2018-01-30 03:43:13 +01:00
}
2020-04-21 07:01:59 +02:00
info , err := xl . getObjectInfo ( ctx , bucket , object , opts )
2016-05-21 05:48:47 +02:00
if err != nil {
2017-06-22 04:53:09 +02:00
return oi , toObjectErr ( err , bucket , object )
2016-05-21 05:48:47 +02:00
}
2018-01-30 03:43:13 +01:00
2016-05-21 05:48:47 +02:00
return info , nil
}
2018-10-19 20:00:09 +02:00
// getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
2020-04-21 07:01:59 +02:00
func ( xl xlObjects ) getObjectInfo ( ctx context . Context , bucket , object string , opt ObjectOptions ) ( objInfo ObjectInfo , err error ) {
2018-10-19 20:00:09 +02:00
disks := xl . getDisks ( )
// Read metadata associated with the object from all disks.
metaArr , errs := readAllXLMetadata ( ctx , disks , bucket , object )
2019-02-06 02:58:48 +01:00
readQuorum , _ , err := objectQuorumFromMeta ( ctx , xl , metaArr , errs )
if err != nil {
return objInfo , err
2018-02-01 19:47:49 +01:00
}
// List all the file commit ids from parts metadata.
modTimes := listObjectModtimes ( metaArr , errs )
// Reduce list of UUIDs to a single common value.
modTime , _ := commonTime ( modTimes )
// Pick latest valid metadata.
2018-08-17 23:42:04 +02:00
xlMeta , err := pickValidXLMeta ( ctx , metaArr , modTime , readQuorum )
2016-05-25 10:33:39 +02:00
if err != nil {
2018-02-01 19:47:49 +01:00
return objInfo , err
2016-05-21 05:48:47 +02:00
}
2016-09-09 07:38:18 +02:00
2018-03-01 20:37:57 +01:00
return xlMeta . ToObjectInfo ( bucket , object ) , nil
2016-05-21 05:48:47 +02:00
}
2016-12-27 01:29:26 +01:00
func undoRename ( disks [ ] StorageAPI , srcBucket , srcEntry , dstBucket , dstEntry string , isDir bool , errs [ ] error ) {
2016-06-17 20:57:51 +02:00
// Undo rename object on disks where RenameFile succeeded.
2016-06-21 04:11:55 +02:00
// If srcEntry/dstEntry are objects then add a trailing slash to copy
// over all the parts inside the object directory
2016-12-27 01:29:26 +01:00
if isDir {
2016-06-21 04:11:55 +02:00
srcEntry = retainSlash ( srcEntry )
dstEntry = retainSlash ( dstEntry )
}
2019-10-14 18:44:51 +02:00
g := errgroup . WithNErrs ( len ( disks ) )
2016-07-12 07:53:54 +02:00
for index , disk := range disks {
2016-06-17 20:57:51 +02:00
if disk == nil {
continue
}
2019-10-14 18:44:51 +02:00
index := index
g . Go ( func ( ) error {
if errs [ index ] == nil {
_ = disks [ index ] . RenameFile ( dstBucket , dstEntry , srcBucket , srcEntry )
2016-06-17 20:57:51 +02:00
}
2019-10-14 18:44:51 +02:00
return nil
} , index )
2016-06-17 20:57:51 +02:00
}
2019-10-14 18:44:51 +02:00
g . Wait ( )
2016-06-17 20:57:51 +02:00
}
2016-06-21 04:11:55 +02:00
// rename - common function that renamePart and renameObject use to rename
// the respective underlying storage layer representations.
2018-04-12 02:15:42 +02:00
func rename ( ctx context . Context , disks [ ] StorageAPI , srcBucket , srcEntry , dstBucket , dstEntry string , isDir bool , writeQuorum int , ignoredErr [ ] error ) ( [ ] StorageAPI , error ) {
2016-05-21 05:48:47 +02:00
2016-12-27 01:29:26 +01:00
if isDir {
2016-06-21 04:11:55 +02:00
dstEntry = retainSlash ( dstEntry )
srcEntry = retainSlash ( srcEntry )
}
2019-10-14 18:44:51 +02:00
g := errgroup . WithNErrs ( len ( disks ) )
2016-05-21 05:48:47 +02:00
// Rename file on all underlying storage disks.
2019-10-14 18:44:51 +02:00
for index := range disks {
index := index
g . Go ( func ( ) error {
if disks [ index ] == nil {
return errDiskNotFound
}
if err := disks [ index ] . RenameFile ( srcBucket , srcEntry , dstBucket , dstEntry ) ; err != nil {
2018-04-12 02:15:42 +02:00
if ! IsErrIgnored ( err , ignoredErr ... ) {
2019-10-14 18:44:51 +02:00
return err
2018-04-12 02:15:42 +02:00
}
2016-05-21 05:48:47 +02:00
}
2019-10-14 18:44:51 +02:00
return nil
} , index )
2016-05-21 05:48:47 +02:00
}
2016-06-02 01:43:31 +02:00
// Wait for all renames to finish.
2019-10-14 18:44:51 +02:00
errs := g . Wait ( )
2016-05-21 05:48:47 +02:00
2018-02-16 02:45:57 +01:00
// We can safely allow RenameFile errors up to len(xl.getDisks()) - writeQuorum
2016-05-21 05:48:47 +02:00
// otherwise return failure. Cleanup successful renames.
2018-04-06 00:04:40 +02:00
err := reduceWriteQuorumErrs ( ctx , errs , objectOpIgnoredErrs , writeQuorum )
2018-04-10 18:36:37 +02:00
if err == errXLWriteQuorum {
2016-06-17 20:57:51 +02:00
// Undo all the partial rename operations.
2016-12-27 01:29:26 +01:00
undoRename ( disks , srcBucket , srcEntry , dstBucket , dstEntry , isDir , errs )
2016-05-21 05:48:47 +02:00
}
2017-06-15 02:14:27 +02:00
return evalDisks ( disks , errs ) , err
2016-05-21 05:48:47 +02:00
}
2016-06-02 01:43:31 +02:00
// PutObject - creates an object upon reading from the input stream
// until EOF, erasure codes the data across all disk and additionally
// writes `xl.json` which carries the necessary metadata for future
// object operations.
2019-02-09 06:31:06 +01:00
func ( xl xlObjects ) PutObject ( ctx context . Context , bucket string , object string , data * PutObjReader , opts ObjectOptions ) ( objInfo ObjectInfo , err error ) {
2018-02-10 00:19:30 +01:00
// Validate put object input args.
2018-04-06 00:04:40 +02:00
if err = checkPutObjectArgs ( ctx , bucket , object , xl , data . Size ( ) ) ; err != nil {
2018-02-10 00:19:30 +01:00
return ObjectInfo { } , err
}
2018-09-21 04:22:09 +02:00
2019-02-09 06:31:06 +01:00
return xl . putObject ( ctx , bucket , object , data , opts )
2018-01-13 05:34:52 +01:00
}
// putObject wrapper for xl PutObject
2019-02-09 06:31:06 +01:00
func ( xl xlObjects ) putObject ( ctx context . Context , bucket string , object string , r * PutObjReader , opts ObjectOptions ) ( objInfo ObjectInfo , err error ) {
2020-04-27 19:06:21 +02:00
defer ObjectPathUpdated ( path . Join ( bucket , object ) )
2018-11-15 02:36:41 +01:00
data := r . Reader
2018-01-30 03:43:13 +01:00
uniqueID := mustGetUUID ( )
tempObj := uniqueID
// No metadata is set, allocate a new one.
2019-02-09 06:31:06 +01:00
if opts . UserDefined == nil {
opts . UserDefined = make ( map [ string ] string )
2018-01-30 03:43:13 +01:00
}
2019-10-07 07:50:24 +02:00
storageDisks := xl . getDisks ( )
2018-01-30 03:43:13 +01:00
// Get parity and data drive count based on storage class metadata
2019-10-23 07:59:13 +02:00
parityDrives := globalStorageClass . GetParityForSC ( opts . UserDefined [ xhttp . AmzStorageClass ] )
2019-10-07 07:50:24 +02:00
if parityDrives == 0 {
2020-03-31 18:32:16 +02:00
parityDrives = getDefaultParityBlocks ( len ( storageDisks ) )
2019-10-07 07:50:24 +02:00
}
dataDrives := len ( storageDisks ) - parityDrives
2018-01-30 03:43:13 +01:00
// we now know the number of blocks this object needs for data and parity.
// writeQuorum is dataBlocks + 1
writeQuorum := dataDrives + 1
// Delete temporary object in the event of failure.
// If PutObject succeeded there would be no temporary
// object to delete.
2018-08-29 22:36:19 +02:00
defer xl . deleteObject ( ctx , minioMetaTmpBucket , tempObj , writeQuorum , false )
2018-01-30 03:43:13 +01:00
2017-01-21 01:33:01 +01:00
// This is a special case with size as '0' and object ends with
// a slash separator, we treat it like a valid operation and
// return success.
2017-09-19 21:40:27 +02:00
if isObjectDir ( object , data . Size ( ) ) {
2017-02-22 04:43:44 +01:00
// Check if an object is present as one of the parent dir.
// -- FIXME. (needs a new kind of lock).
2017-05-09 23:32:24 +02:00
// -- FIXME (this also causes performance issue when disks are down).
2018-04-06 00:04:40 +02:00
if xl . parentDirIsObject ( ctx , bucket , path . Dir ( object ) ) {
2019-03-20 21:06:53 +01:00
return ObjectInfo { } , toObjectErr ( errFileParentIsFile , bucket , object )
2017-02-22 04:43:44 +01:00
}
2018-01-30 03:43:13 +01:00
2018-04-06 00:04:40 +02:00
if err = xl . putObjectDir ( ctx , minioMetaTmpBucket , tempObj , writeQuorum ) ; err != nil {
return ObjectInfo { } , toObjectErr ( err , bucket , object )
2018-01-30 03:43:13 +01:00
}
2018-10-18 01:37:02 +02:00
// Rename the successfully written temporary object to final location. Ignore errFileAccessDenied
// error because it means that the target object dir exists and we want to be close to S3 specification.
2019-10-07 07:50:24 +02:00
if _ , err = rename ( ctx , storageDisks , minioMetaTmpBucket , tempObj , bucket , object , true , writeQuorum , [ ] error { errFileAccessDenied } ) ; err != nil {
2018-01-30 03:43:13 +01:00
return ObjectInfo { } , toObjectErr ( err , bucket , object )
}
2019-02-09 06:31:06 +01:00
return dirObjectInfo ( bucket , object , data . Size ( ) , opts . UserDefined ) , nil
2017-01-21 01:33:01 +01:00
}
2017-02-22 04:43:44 +01:00
2017-10-06 18:38:01 +02:00
// Validate input data size and it can never be less than zero.
2018-09-28 05:36:17 +02:00
if data . Size ( ) < - 1 {
2019-10-12 03:50:54 +02:00
logger . LogIf ( ctx , errInvalidArgument , logger . Application )
2018-04-06 00:04:40 +02:00
return ObjectInfo { } , toObjectErr ( errInvalidArgument )
2017-10-06 18:38:01 +02:00
}
2017-02-22 04:43:44 +01:00
// Check if an object is present as one of the parent dir.
// -- FIXME. (needs a new kind of lock).
2017-05-09 23:32:24 +02:00
// -- FIXME (this also causes performance issue when disks are down).
2018-04-06 00:04:40 +02:00
if xl . parentDirIsObject ( ctx , bucket , path . Dir ( object ) ) {
2019-03-20 21:06:53 +01:00
return ObjectInfo { } , toObjectErr ( errFileParentIsFile , bucket , object )
2017-02-22 04:43:44 +01:00
}
2017-01-31 00:44:42 +01:00
// Initialize parts metadata
2018-02-16 02:45:57 +01:00
partsMetadata := make ( [ ] xlMetaV1 , len ( xl . getDisks ( ) ) )
2017-01-31 00:44:42 +01:00
2017-12-22 12:28:13 +01:00
xlMeta := newXLMetaV1 ( object , dataDrives , parityDrives )
2016-07-13 20:56:25 +02:00
2017-01-31 00:44:42 +01:00
// Initialize xl meta.
for index := range partsMetadata {
partsMetadata [ index ] = xlMeta
}
// Order disks according to erasure distribution
2019-10-07 07:50:24 +02:00
onlineDisks := shuffleDisks ( storageDisks , xlMeta . Erasure . Distribution )
2017-01-31 00:44:42 +01:00
2018-08-24 08:35:37 +02:00
erasure , err := NewErasure ( ctx , xlMeta . Erasure . DataBlocks , xlMeta . Erasure . ParityBlocks , xlMeta . Erasure . BlockSize )
2017-08-15 03:08:42 +02:00
if err != nil {
return ObjectInfo { } , toObjectErr ( err , bucket , object )
}
2017-10-06 18:38:01 +02:00
2018-02-16 02:45:57 +01:00
// Fetch buffer for I/O, returns from the pool if not allocates a new one and returns.
2018-06-13 20:55:12 +02:00
var buffer [ ] byte
switch size := data . Size ( ) ; {
case size == 0 :
buffer = make ( [ ] byte , 1 ) // Allocate atleast a byte to reach EOF
2018-10-05 00:37:15 +02:00
case size == - 1 || size >= blockSizeV1 :
2018-09-28 05:36:17 +02:00
buffer = xl . bp . Get ( )
defer xl . bp . Put ( buffer )
2018-06-13 20:55:12 +02:00
case size < blockSizeV1 :
// No need to allocate fully blockSizeV1 buffer if the incoming data is smaller.
2020-01-18 23:21:58 +01:00
buffer = make ( [ ] byte , size , 2 * size + int64 ( erasure . parityBlocks + erasure . dataBlocks - 1 ) )
2018-06-13 20:55:12 +02:00
}
2017-10-06 18:38:01 +02:00
2018-08-07 00:14:08 +02:00
if len ( buffer ) > int ( xlMeta . Erasure . BlockSize ) {
buffer = buffer [ : xlMeta . Erasure . BlockSize ]
}
2019-05-14 21:33:18 +02:00
partName := "part.1"
tempErasureObj := pathJoin ( uniqueID , partName )
2017-01-31 00:44:42 +01:00
2019-05-14 21:33:18 +02:00
writers := make ( [ ] io . Writer , len ( onlineDisks ) )
for i , disk := range onlineDisks {
if disk == nil {
continue
2018-09-28 05:36:17 +02:00
}
2019-05-14 21:33:18 +02:00
writers [ i ] = newBitrotWriter ( disk , minioMetaTmpBucket , tempErasureObj , erasure . ShardFileSize ( data . Size ( ) ) , DefaultBitrotAlgorithm , erasure . ShardSize ( ) )
}
2018-09-28 05:36:17 +02:00
2019-05-14 21:33:18 +02:00
n , erasureErr := erasure . Encode ( ctx , data , writers , buffer , erasure . dataBlocks + 1 )
closeBitrotWriters ( writers )
if erasureErr != nil {
return ObjectInfo { } , toObjectErr ( erasureErr , minioMetaTmpBucket , tempErasureObj )
}
2017-08-15 03:08:42 +02:00
2019-05-14 21:33:18 +02:00
// Should return IncompleteBody{} error when reader has fewer bytes
// than specified in request header.
if n < data . Size ( ) {
2019-10-12 03:50:54 +02:00
logger . LogIf ( ctx , IncompleteBody { } , logger . Application )
2019-05-14 21:33:18 +02:00
return ObjectInfo { } , IncompleteBody { }
}
2017-02-01 00:34:49 +01:00
2019-05-14 21:33:18 +02:00
for i , w := range writers {
if w == nil {
onlineDisks [ i ] = nil
continue
2017-01-31 00:44:42 +01:00
}
2020-03-03 01:29:30 +01:00
partsMetadata [ i ] . AddObjectPart ( 1 , "" , n , data . ActualSize ( ) )
partsMetadata [ i ] . Erasure . AddChecksumInfo ( ChecksumInfo {
PartNumber : 1 ,
Algorithm : DefaultBitrotAlgorithm ,
Hash : bitrotWriterSum ( w ) ,
} )
2016-07-19 04:06:48 +02:00
}
2016-07-01 23:33:28 +02:00
2016-05-21 05:48:47 +02:00
// Save additional erasureMetadata.
2017-03-18 19:28:41 +01:00
modTime := UTCNow ( )
2018-11-15 02:36:41 +01:00
2019-02-09 06:31:06 +01:00
opts . UserDefined [ "etag" ] = r . MD5CurrentHexString ( )
2016-06-02 01:43:31 +02:00
2016-06-17 06:42:02 +02:00
// Guess content-type from the extension if possible.
2019-02-09 06:31:06 +01:00
if opts . UserDefined [ "content-type" ] == "" {
opts . UserDefined [ "content-type" ] = mimedb . TypeByExtension ( path . Ext ( object ) )
2016-05-21 05:48:47 +02:00
}
2016-06-20 15:18:47 +02:00
if xl . isObject ( bucket , object ) {
2017-02-22 04:43:44 +01:00
// Rename if an object already exists to temporary location.
newUniqueID := mustGetUUID ( )
// Delete successfully renamed object.
2018-08-29 22:36:19 +02:00
defer xl . deleteObject ( ctx , minioMetaTmpBucket , newUniqueID , writeQuorum , false )
2016-10-20 07:52:03 +02:00
2018-10-05 02:22:49 +02:00
// NOTE: Do not use online disks slice here: the reason is that existing object should be purged
// regardless of `xl.json` status and rolled back in case of errors. Also allow renaming the
// existing object if it is not present in quorum disks so users can overwrite stale objects.
2020-01-16 03:30:32 +01:00
_ , err = rename ( ctx , storageDisks , bucket , object , minioMetaTmpBucket , newUniqueID , true , writeQuorum , [ ] error { errFileNotFound } )
2016-06-20 15:18:47 +02:00
if err != nil {
2016-09-02 21:18:35 +02:00
return ObjectInfo { } , toObjectErr ( err , bucket , object )
2016-06-20 15:18:47 +02:00
}
2016-05-21 05:48:47 +02:00
}
2016-05-27 04:55:48 +02:00
// Fill all the necessary metadata.
2016-06-02 01:43:31 +02:00
// Update `xl.json` content on each disks.
2016-06-01 05:23:31 +02:00
for index := range partsMetadata {
2019-02-09 06:31:06 +01:00
partsMetadata [ index ] . Meta = opts . UserDefined
2019-05-14 21:33:18 +02:00
partsMetadata [ index ] . Stat . Size = n
2017-01-31 00:44:42 +01:00
partsMetadata [ index ] . Stat . ModTime = modTime
2016-06-01 05:23:31 +02:00
}
// Write unique `xl.json` for each disk.
2018-04-06 00:04:40 +02:00
if onlineDisks , err = writeUniqueXLMetadata ( ctx , onlineDisks , minioMetaTmpBucket , tempObj , partsMetadata , writeQuorum ) ; err != nil {
2016-09-02 21:18:35 +02:00
return ObjectInfo { } , toObjectErr ( err , bucket , object )
2016-05-21 05:48:47 +02:00
}
2016-05-29 09:42:09 +02:00
2016-06-02 01:43:31 +02:00
// Rename the successfully written temporary object to final location.
2020-01-16 03:30:32 +01:00
if onlineDisks , err = rename ( ctx , onlineDisks , minioMetaTmpBucket , tempObj , bucket , object , true , writeQuorum , nil ) ; err != nil {
2016-09-02 21:18:35 +02:00
return ObjectInfo { } , toObjectErr ( err , bucket , object )
2016-05-29 00:13:15 +02:00
}
2016-05-29 09:42:09 +02:00
2020-01-16 03:30:32 +01:00
// Whether a disk was initially or becomes offline
// during this upload, send it to the MRF list.
for i := 0 ; i < len ( onlineDisks ) ; i ++ {
if onlineDisks [ i ] == nil || storageDisks [ i ] == nil {
xl . addPartialUpload ( bucket , object )
break
}
}
2017-01-31 00:44:42 +01:00
// Object info is the same in all disks, so we can pick the first meta
// of the first disk
xlMeta = partsMetadata [ 0 ]
2016-09-02 21:18:35 +02:00
objInfo = ObjectInfo {
IsDir : false ,
Bucket : bucket ,
Name : object ,
Size : xlMeta . Stat . Size ,
ModTime : xlMeta . Stat . ModTime ,
2017-05-14 21:05:51 +02:00
ETag : xlMeta . Meta [ "etag" ] ,
2016-09-02 21:18:35 +02:00
ContentType : xlMeta . Meta [ "content-type" ] ,
ContentEncoding : xlMeta . Meta [ "content-encoding" ] ,
UserDefined : xlMeta . Meta ,
}
2017-01-17 04:23:43 +01:00
2016-09-02 21:18:35 +02:00
return objInfo , nil
2016-05-21 05:48:47 +02:00
}
2016-06-02 01:43:31 +02:00
// deleteObject - wrapper for delete object, deletes an object from
// all the disks in parallel, including `xl.json` associated with the
// object.
2018-08-29 22:36:19 +02:00
func ( xl xlObjects ) deleteObject ( ctx context . Context , bucket , object string , writeQuorum int , isDir bool ) error {
var disks [ ] StorageAPI
2018-01-30 03:43:13 +01:00
var err error
2020-04-27 19:06:21 +02:00
defer ObjectPathUpdated ( path . Join ( bucket , object ) )
2018-02-21 00:33:26 +01:00
2018-08-29 22:36:19 +02:00
tmpObj := mustGetUUID ( )
if bucket == minioMetaTmpBucket {
tmpObj = object
disks = xl . getDisks ( )
} else {
2018-10-05 02:22:49 +02:00
// Rename the current object while requiring write quorum, but also consider
// that a non found object in a given disk as a success since it already
// confirms that the object doesn't have a part in that disk (already removed)
2018-08-29 22:36:19 +02:00
if isDir {
2018-10-05 02:22:49 +02:00
disks , err = rename ( ctx , xl . getDisks ( ) , bucket , object , minioMetaTmpBucket , tmpObj , true , writeQuorum ,
[ ] error { errFileNotFound , errFileAccessDenied } )
2018-08-29 22:36:19 +02:00
} else {
2018-10-05 02:22:49 +02:00
disks , err = rename ( ctx , xl . getDisks ( ) , bucket , object , minioMetaTmpBucket , tmpObj , true , writeQuorum ,
[ ] error { errFileNotFound } )
2018-01-30 03:43:13 +01:00
}
2018-03-27 01:39:28 +02:00
if err != nil {
2018-08-29 22:36:19 +02:00
return toObjectErr ( err , bucket , object )
2018-03-27 01:39:28 +02:00
}
2017-12-22 12:28:13 +01:00
}
2019-10-14 18:44:51 +02:00
g := errgroup . WithNErrs ( len ( disks ) )
2018-08-29 22:36:19 +02:00
2019-10-14 18:44:51 +02:00
for index := range disks {
index := index
g . Go ( func ( ) error {
if disks [ index ] == nil {
return errDiskNotFound
}
var err error
2018-02-21 00:33:26 +01:00
if isDir {
// DeleteFile() simply tries to remove a directory
// and will succeed only if that directory is empty.
2019-10-14 18:44:51 +02:00
err = disks [ index ] . DeleteFile ( minioMetaTmpBucket , tmpObj )
2018-02-21 00:33:26 +01:00
} else {
2019-10-14 18:44:51 +02:00
err = cleanupDir ( ctx , disks [ index ] , minioMetaTmpBucket , tmpObj )
2018-02-21 00:33:26 +01:00
}
2019-10-14 18:44:51 +02:00
if err != nil && err != errVolumeNotFound {
return err
2016-05-25 23:32:49 +02:00
}
2019-10-14 18:44:51 +02:00
return nil
} , index )
2016-05-21 05:48:47 +02:00
}
2018-03-27 01:39:28 +02:00
// return errors if any during deletion
2019-10-14 18:44:51 +02:00
return reduceWriteQuorumErrs ( ctx , g . Wait ( ) , objectOpIgnoredErrs , writeQuorum )
2016-05-21 05:48:47 +02:00
}
2019-05-13 21:25:49 +02:00
// deleteObject - wrapper for delete object, deletes an object from
// all the disks in parallel, including `xl.json` associated with the
// object.
func ( xl xlObjects ) doDeleteObjects ( ctx context . Context , bucket string , objects [ ] string , errs [ ] error , writeQuorums [ ] int , isDirs [ ] bool ) ( [ ] error , error ) {
var tmpObjs = make ( [ ] string , len ( objects ) )
if bucket == minioMetaTmpBucket {
copy ( tmpObjs , objects )
} else {
2020-03-06 22:44:24 +01:00
for idx := range objects {
if errs [ idx ] != nil {
2019-05-13 21:25:49 +02:00
continue
}
2020-04-27 19:06:21 +02:00
2020-03-06 22:44:24 +01:00
tmpObjs [ idx ] = mustGetUUID ( )
2019-05-13 21:25:49 +02:00
var err error
2020-03-06 22:44:24 +01:00
// Rename the current object while requiring
// write quorum, but also consider that a non
// found object in a given disk as a success
// since it already confirms that the object
// doesn't have a part in that disk (already removed)
if isDirs [ idx ] {
_ , err = rename ( ctx , xl . getDisks ( ) , bucket , objects [ idx ] ,
minioMetaTmpBucket , tmpObjs [ idx ] , true , writeQuorums [ idx ] ,
2019-05-13 21:25:49 +02:00
[ ] error { errFileNotFound , errFileAccessDenied } )
} else {
2020-03-06 22:44:24 +01:00
_ , err = rename ( ctx , xl . getDisks ( ) , bucket , objects [ idx ] ,
minioMetaTmpBucket , tmpObjs [ idx ] , true , writeQuorums [ idx ] ,
2019-05-13 21:25:49 +02:00
[ ] error { errFileNotFound } )
}
if err != nil {
2020-03-06 22:44:24 +01:00
errs [ idx ] = err
2019-05-13 21:25:49 +02:00
}
2020-04-27 19:06:21 +02:00
ObjectPathUpdated ( path . Join ( bucket , objects [ idx ] ) )
2019-05-13 21:25:49 +02:00
}
}
2020-03-06 22:44:24 +01:00
disks := xl . getDisks ( )
2019-05-13 21:25:49 +02:00
// Initialize list of errors.
var opErrs = make ( [ ] error , len ( disks ) )
var delObjErrs = make ( [ ] [ ] error , len ( disks ) )
2020-03-11 16:56:36 +01:00
var wg = sync . WaitGroup { }
2019-05-13 21:25:49 +02:00
2019-05-14 23:43:22 +02:00
// Remove objects in bulk for each disk
2020-03-11 16:56:36 +01:00
for i , d := range disks {
if d == nil {
opErrs [ i ] = errDiskNotFound
2019-05-13 21:25:49 +02:00
continue
}
2020-03-11 16:56:36 +01:00
wg . Add ( 1 )
go func ( index int , disk StorageAPI ) {
defer wg . Done ( )
delObjErrs [ index ] , opErrs [ index ] = disk . DeletePrefixes ( minioMetaTmpBucket , tmpObjs )
if opErrs [ index ] == errVolumeNotFound || opErrs [ index ] == errFileNotFound {
opErrs [ index ] = nil
}
} ( i , d )
2019-05-13 21:25:49 +02:00
}
2020-03-11 16:56:36 +01:00
wg . Wait ( )
2019-05-13 21:25:49 +02:00
// Return errors if any during deletion
2020-02-02 03:11:29 +01:00
if err := reduceWriteQuorumErrs ( ctx , opErrs , objectOpIgnoredErrs , len ( disks ) / 2 + 1 ) ; err != nil {
2019-05-13 21:25:49 +02:00
return nil , err
}
// Reduce errors for each object
for objIndex := range objects {
if errs [ objIndex ] != nil {
continue
}
2020-02-02 03:11:29 +01:00
listErrs := make ( [ ] error , len ( disks ) )
2020-03-11 16:56:36 +01:00
// Iterate over disks to fetch the error
// of deleting of the current object
2019-05-13 21:25:49 +02:00
for i := range delObjErrs {
2020-03-11 16:56:36 +01:00
// delObjErrs[i] is not nil when disks[i] is also not nil
2019-05-13 21:25:49 +02:00
if delObjErrs [ i ] != nil {
2020-03-11 16:56:36 +01:00
if delObjErrs [ i ] [ objIndex ] != errFileNotFound {
listErrs [ i ] = delObjErrs [ i ] [ objIndex ]
}
2019-05-13 21:25:49 +02:00
}
}
errs [ objIndex ] = reduceWriteQuorumErrs ( ctx , listErrs , objectOpIgnoredErrs , writeQuorums [ objIndex ] )
}
return errs , nil
}
func ( xl xlObjects ) deleteObjects ( ctx context . Context , bucket string , objects [ ] string ) ( [ ] error , error ) {
errs := make ( [ ] error , len ( objects ) )
writeQuorums := make ( [ ] int , len ( objects ) )
isObjectDirs := make ( [ ] bool , len ( objects ) )
for i , object := range objects {
errs [ i ] = checkDelObjArgs ( ctx , bucket , object )
}
for i , object := range objects {
2019-12-06 08:16:06 +01:00
isObjectDirs [ i ] = HasSuffix ( object , SlashSeparator )
2019-05-13 21:25:49 +02:00
}
2020-03-31 18:32:16 +02:00
storageDisks := xl . getDisks ( )
2019-05-13 21:25:49 +02:00
for i , object := range objects {
if isObjectDirs [ i ] {
_ , err := xl . getObjectInfoDir ( ctx , bucket , object )
if err == errXLReadQuorum {
2020-03-31 18:32:16 +02:00
if isObjectDirDangling ( statAllDirs ( ctx , storageDisks , bucket , object ) ) {
2019-05-13 21:25:49 +02:00
// If object is indeed dangling, purge it.
errs [ i ] = nil
}
}
if err != nil {
errs [ i ] = toObjectErr ( err , bucket , object )
continue
}
}
}
2020-03-06 22:44:24 +01:00
for i := range objects {
2019-05-13 21:25:49 +02:00
if errs [ i ] != nil {
continue
}
2020-03-06 22:44:24 +01:00
// Assume (N/2 + 1) quorums for all objects
// this is a theoretical assumption such that
// for delete's we do not need to honor storage
// class for objects which have reduced quorum
// storage class only needs to be honored for
// Read() requests alone which we already do.
2020-03-31 18:32:16 +02:00
writeQuorums [ i ] = getWriteQuorum ( len ( storageDisks ) )
2019-05-13 21:25:49 +02:00
}
return xl . doDeleteObjects ( ctx , bucket , objects , errs , writeQuorums , isObjectDirs )
}
// DeleteObjects deletes objects in bulk, this function will still automatically split objects list
// into smaller bulks if some object names are found to be duplicated in the delete list, splitting
// into smaller bulks will avoid holding twice the write lock of the duplicated object names.
func ( xl xlObjects ) DeleteObjects ( ctx context . Context , bucket string , objects [ ] string ) ( [ ] error , error ) {
var (
i , start , end int
// Deletion result for all objects
deleteErrs [ ] error
// Object names store will be used to check for object name duplication
objectNamesStore = make ( map [ string ] interface { } )
)
for {
if i >= len ( objects ) {
break
}
object := objects [ i ]
_ , duplicationFound := objectNamesStore [ object ]
if duplicationFound {
end = i - 1
} else {
objectNamesStore [ object ] = true
end = i
}
if duplicationFound || i == len ( objects ) - 1 {
errs , err := xl . deleteObjects ( ctx , bucket , objects [ start : end + 1 ] )
if err != nil {
return nil , err
}
deleteErrs = append ( deleteErrs , errs ... )
objectNamesStore = make ( map [ string ] interface { } )
}
if duplicationFound {
// Avoid to increase the index if object
// name is found to be duplicated.
start = i
} else {
i ++
}
}
return deleteErrs , nil
}
2016-06-02 01:43:31 +02:00
// DeleteObject - deletes an object, this call doesn't necessary reply
// any error as it is not necessary for the handler to reply back a
// response to the client request.
2018-03-14 20:01:47 +01:00
func ( xl xlObjects ) DeleteObject ( ctx context . Context , bucket , object string ) ( err error ) {
2018-04-06 00:04:40 +02:00
if err = checkDelObjArgs ( ctx , bucket , object ) ; err != nil {
2016-12-02 08:15:17 +01:00
return err
2016-05-21 05:48:47 +02:00
}
2016-08-31 20:39:08 +02:00
2018-08-31 22:16:35 +02:00
var writeQuorum int
2019-12-06 08:16:06 +01:00
var isObjectDir = HasSuffix ( object , SlashSeparator )
2018-08-29 22:36:19 +02:00
2020-03-31 18:32:16 +02:00
storageDisks := xl . getDisks ( )
2019-04-23 23:54:28 +02:00
if isObjectDir {
_ , err = xl . getObjectInfoDir ( ctx , bucket , object )
if err == errXLReadQuorum {
2020-03-31 18:32:16 +02:00
if isObjectDirDangling ( statAllDirs ( ctx , storageDisks , bucket , object ) ) {
2019-04-23 23:54:28 +02:00
// If object is indeed dangling, purge it.
err = nil
}
}
if err != nil {
return toObjectErr ( err , bucket , object )
}
2018-10-31 00:07:57 +01:00
}
2018-08-31 22:16:35 +02:00
if isObjectDir {
2020-03-31 18:32:16 +02:00
writeQuorum = getWriteQuorum ( len ( storageDisks ) )
2018-08-31 22:16:35 +02:00
} else {
// Read metadata associated with the object from all disks.
2020-03-31 18:32:16 +02:00
partsMetadata , errs := readAllXLMetadata ( ctx , storageDisks , bucket , object )
2018-08-31 22:16:35 +02:00
// get Quorum for this object
_ , writeQuorum , err = objectQuorumFromMeta ( ctx , xl , partsMetadata , errs )
if err != nil {
return toObjectErr ( err , bucket , object )
}
2018-08-29 22:36:19 +02:00
}
2016-06-17 07:18:43 +02:00
2016-06-17 20:57:51 +02:00
// Delete the object on all disks.
2018-08-31 22:16:35 +02:00
if err = xl . deleteObject ( ctx , bucket , object , writeQuorum , isObjectDir ) ; err != nil {
2016-06-17 20:57:51 +02:00
return toObjectErr ( err , bucket , object )
2016-06-07 20:35:03 +02:00
}
2016-06-17 20:57:51 +02:00
// Success.
return nil
2016-05-21 05:48:47 +02:00
}
2018-02-10 00:19:30 +01:00
// ListObjectsV2 lists all blobs in bucket filtered by prefix
2018-03-14 20:01:47 +01:00
func ( xl xlObjects ) ListObjectsV2 ( ctx context . Context , bucket , prefix , continuationToken , delimiter string , maxKeys int , fetchOwner bool , startAfter string ) ( result ListObjectsV2Info , err error ) {
2018-07-01 06:22:45 +02:00
marker := continuationToken
if marker == "" {
marker = startAfter
}
loi , err := xl . ListObjects ( ctx , bucket , prefix , marker , delimiter , maxKeys )
2018-02-10 00:19:30 +01:00
if err != nil {
return result , err
}
listObjectsV2Info := ListObjectsV2Info {
IsTruncated : loi . IsTruncated ,
ContinuationToken : continuationToken ,
NextContinuationToken : loi . NextMarker ,
Objects : loi . Objects ,
Prefixes : loi . Prefixes ,
}
return listObjectsV2Info , err
}
2020-01-16 03:30:32 +01:00
2020-01-20 17:45:59 +01:00
// Send the successful but partial upload, however ignore
2020-01-16 03:30:32 +01:00
// if the channel is blocked by other items.
func ( xl xlObjects ) addPartialUpload ( bucket , key string ) {
select {
case xl . mrfUploadCh <- partialUpload { bucket : bucket , object : key } :
default :
}
}
2020-01-20 17:45:59 +01:00
// PutObjectTag - replace or add tags to an existing object
func ( xl xlObjects ) PutObjectTag ( ctx context . Context , bucket , object string , tags string ) error {
disks := xl . getDisks ( )
// Read metadata associated with the object from all disks.
metaArr , errs := readAllXLMetadata ( ctx , disks , bucket , object )
_ , writeQuorum , err := objectQuorumFromMeta ( ctx , xl , metaArr , errs )
if err != nil {
2020-05-05 23:18:13 +02:00
return toObjectErr ( err , bucket , object )
2020-01-20 17:45:59 +01:00
}
for i , xlMeta := range metaArr {
// clean xlMeta.Meta of tag key, before updating the new tags
delete ( xlMeta . Meta , xhttp . AmzObjectTagging )
// Don't update for empty tags
if tags != "" {
xlMeta . Meta [ xhttp . AmzObjectTagging ] = tags
}
metaArr [ i ] . Meta = xlMeta . Meta
}
tempObj := mustGetUUID ( )
// Write unique `xl.json` for each disk.
if disks , err = writeUniqueXLMetadata ( ctx , disks , minioMetaTmpBucket , tempObj , metaArr , writeQuorum ) ; err != nil {
return toObjectErr ( err , bucket , object )
}
// Atomically rename `xl.json` from tmp location to destination for each disk.
if _ , err = renameXLMetadata ( ctx , disks , minioMetaTmpBucket , tempObj , bucket , object , writeQuorum ) ; err != nil {
return toObjectErr ( err , bucket , object )
}
return nil
}
// DeleteObjectTag - delete object tags from an existing object
func ( xl xlObjects ) DeleteObjectTag ( ctx context . Context , bucket , object string ) error {
return xl . PutObjectTag ( ctx , bucket , object , "" )
}
// GetObjectTag - get object tags from an existing object
2020-05-05 23:18:13 +02:00
func ( xl xlObjects ) GetObjectTag ( ctx context . Context , bucket , object string ) ( * tags . Tags , error ) {
2020-01-20 17:45:59 +01:00
// GetObjectInfo will return tag value as well
oi , err := xl . GetObjectInfo ( ctx , bucket , object , ObjectOptions { } )
if err != nil {
2020-05-05 23:18:13 +02:00
return nil , err
2020-01-20 17:45:59 +01:00
}
2020-05-05 23:18:13 +02:00
return tags . ParseObjectTags ( oi . UserTags )
2020-01-20 17:45:59 +01:00
}