2020-07-22 02:49:56 +02:00
/ *
* MinIO Cloud Storage , ( C ) 2020 MinIO , Inc .
*
* Licensed under the Apache License , Version 2.0 ( the "License" ) ;
* you may not use this file except in compliance with the License .
* You may obtain a copy of the License at
*
* http : //www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing , software
* distributed under the License is distributed on an "AS IS" BASIS ,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
* See the License for the specific language governing permissions and
* limitations under the License .
* /
package cmd
import (
"context"
2020-09-16 05:44:48 +02:00
"fmt"
2020-07-22 02:49:56 +02:00
"net/http"
2021-02-04 05:41:33 +01:00
"reflect"
2020-08-13 02:32:24 +02:00
"strings"
2021-03-09 11:56:42 +01:00
"sync"
2020-07-22 02:49:56 +02:00
"time"
2020-11-19 20:50:22 +01:00
minio "github.com/minio/minio-go/v7"
2020-07-22 02:49:56 +02:00
miniogo "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/encrypt"
"github.com/minio/minio-go/v7/pkg/tags"
"github.com/minio/minio/cmd/crypto"
xhttp "github.com/minio/minio/cmd/http"
"github.com/minio/minio/cmd/logger"
2020-10-10 05:36:00 +02:00
"github.com/minio/minio/pkg/bucket/bandwidth"
2020-07-22 02:49:56 +02:00
"github.com/minio/minio/pkg/bucket/replication"
"github.com/minio/minio/pkg/event"
iampolicy "github.com/minio/minio/pkg/iam/policy"
2020-11-19 19:38:50 +01:00
"github.com/minio/minio/pkg/madmin"
2020-07-22 02:49:56 +02:00
)
2020-07-31 04:55:22 +02:00
// gets replication config associated to a given bucket name.
func getReplicationConfig ( ctx context . Context , bucketName string ) ( rc * replication . Config , err error ) {
2020-07-22 02:49:56 +02:00
if globalIsGateway {
2020-10-09 18:59:52 +02:00
objAPI := newObjectLayerFn ( )
2020-07-22 02:49:56 +02:00
if objAPI == nil {
return nil , errServerNotInitialized
}
return nil , BucketReplicationConfigNotFound { Bucket : bucketName }
}
return globalBucketMetadataSys . GetReplicationConfig ( ctx , bucketName )
}
2020-07-31 04:55:22 +02:00
// validateReplicationDestination returns error if replication destination bucket missing or not configured
2020-07-22 02:49:56 +02:00
// It also returns true if replication destination is same as this server.
2020-07-31 04:55:22 +02:00
func validateReplicationDestination ( ctx context . Context , bucket string , rCfg * replication . Config ) ( bool , error ) {
2020-11-19 19:38:50 +01:00
arn , err := madmin . ParseARN ( rCfg . RoleArn )
if err != nil {
return false , BucketRemoteArnInvalid { }
}
if arn . Type != madmin . ReplicationService {
return false , BucketRemoteArnTypeInvalid { }
}
2020-10-08 19:54:11 +02:00
clnt := globalBucketTargetSys . GetRemoteTargetClient ( ctx , rCfg . RoleArn )
2020-07-22 02:49:56 +02:00
if clnt == nil {
2020-07-31 04:55:22 +02:00
return false , BucketRemoteTargetNotFound { Bucket : bucket }
2020-07-22 02:49:56 +02:00
}
if found , _ := clnt . BucketExists ( ctx , rCfg . GetDestination ( ) . Bucket ) ; ! found {
2020-10-08 19:54:11 +02:00
return false , BucketRemoteDestinationNotFound { Bucket : rCfg . GetDestination ( ) . Bucket }
2020-07-22 02:49:56 +02:00
}
2020-08-05 08:02:27 +02:00
if ret , err := globalBucketObjectLockSys . Get ( bucket ) ; err == nil {
if ret . LockEnabled {
lock , _ , _ , _ , err := clnt . GetObjectLockConfig ( ctx , rCfg . GetDestination ( ) . Bucket )
if err != nil || lock != "Enabled" {
return false , BucketReplicationDestinationMissingLock { Bucket : rCfg . GetDestination ( ) . Bucket }
}
}
}
2020-07-22 02:49:56 +02:00
// validate replication ARN against target endpoint
2020-08-07 02:10:21 +02:00
c , ok := globalBucketTargetSys . arnRemotesMap [ rCfg . RoleArn ]
2020-07-31 04:55:22 +02:00
if ok {
if c . EndpointURL ( ) . String ( ) == clnt . EndpointURL ( ) . String ( ) {
sameTarget , _ := isLocalHost ( clnt . EndpointURL ( ) . Hostname ( ) , clnt . EndpointURL ( ) . Port ( ) , globalMinioPort )
return sameTarget , nil
2020-07-22 02:49:56 +02:00
}
}
2020-07-31 04:55:22 +02:00
return false , BucketRemoteTargetNotFound { Bucket : bucket }
2020-07-22 02:49:56 +02:00
}
2021-01-12 07:36:51 +01:00
func mustReplicateWeb ( ctx context . Context , r * http . Request , bucket , object string , meta map [ string ] string , replStatus string , permErr APIErrorCode ) ( replicate bool , sync bool ) {
2020-08-13 02:32:24 +02:00
if permErr != ErrNone {
2021-01-12 07:36:51 +01:00
return
2020-08-13 02:32:24 +02:00
}
2020-11-20 03:43:58 +01:00
return mustReplicater ( ctx , bucket , object , meta , replStatus )
2020-08-13 02:32:24 +02:00
}
2021-01-12 07:36:51 +01:00
// mustReplicate returns 2 booleans - true if object meets replication criteria and true if replication is to be done in
// a synchronous manner.
func mustReplicate ( ctx context . Context , r * http . Request , bucket , object string , meta map [ string ] string , replStatus string ) ( replicate bool , sync bool ) {
2020-11-04 18:13:34 +01:00
if s3Err := isPutActionAllowed ( ctx , getRequestAuthType ( r ) , bucket , "" , r , iampolicy . GetReplicationConfigurationAction ) ; s3Err != ErrNone {
2021-01-12 07:36:51 +01:00
return
2020-08-13 02:32:24 +02:00
}
2020-11-20 03:43:58 +01:00
return mustReplicater ( ctx , bucket , object , meta , replStatus )
2020-08-13 02:32:24 +02:00
}
2021-01-12 07:36:51 +01:00
// mustReplicater returns 2 booleans - true if object meets replication criteria and true if replication is to be done in
// a synchronous manner.
func mustReplicater ( ctx context . Context , bucket , object string , meta map [ string ] string , replStatus string ) ( replicate bool , sync bool ) {
2020-07-22 02:49:56 +02:00
if globalIsGateway {
2021-01-12 07:36:51 +01:00
return replicate , sync
2020-07-22 02:49:56 +02:00
}
if rs , ok := meta [ xhttp . AmzBucketReplicationStatus ] ; ok {
replStatus = rs
}
if replication . StatusType ( replStatus ) == replication . Replica {
2021-01-12 07:36:51 +01:00
return replicate , sync
2020-07-22 02:49:56 +02:00
}
2020-07-31 04:55:22 +02:00
cfg , err := getReplicationConfig ( ctx , bucket )
2020-07-22 02:49:56 +02:00
if err != nil {
2021-01-12 07:36:51 +01:00
return replicate , sync
2020-07-22 02:49:56 +02:00
}
opts := replication . ObjectOpts {
Name : object ,
SSEC : crypto . SSEC . IsEncrypted ( meta ) ,
}
tagStr , ok := meta [ xhttp . AmzObjectTagging ]
if ok {
opts . UserTags = tagStr
}
2021-01-12 07:36:51 +01:00
tgt := globalBucketTargetSys . GetRemoteTargetClient ( ctx , cfg . RoleArn )
2021-01-27 20:22:34 +01:00
// the target online status should not be used here while deciding
// whether to replicate as the target could be temporarily down
if tgt != nil {
return cfg . Replicate ( opts ) , tgt . replicateSync
2021-01-12 07:36:51 +01:00
}
2021-01-27 20:22:34 +01:00
return cfg . Replicate ( opts ) , false
2021-01-12 07:36:51 +01:00
}
// Standard headers that needs to be extracted from User metadata.
var standardHeaders = [ ] string {
2021-01-27 20:22:34 +01:00
xhttp . ContentType ,
xhttp . CacheControl ,
xhttp . ContentEncoding ,
xhttp . ContentLanguage ,
xhttp . ContentDisposition ,
2021-01-12 07:36:51 +01:00
xhttp . AmzStorageClass ,
xhttp . AmzObjectTagging ,
xhttp . AmzBucketReplicationStatus ,
2021-01-27 20:22:34 +01:00
xhttp . AmzObjectLockMode ,
xhttp . AmzObjectLockRetainUntilDate ,
xhttp . AmzObjectLockLegalHold ,
xhttp . AmzTagCount ,
xhttp . AmzServerSideEncryption ,
2020-07-22 02:49:56 +02:00
}
2020-11-20 03:43:58 +01:00
// returns true if any of the objects being deleted qualifies for replication.
func hasReplicationRules ( ctx context . Context , bucket string , objects [ ] ObjectToDelete ) bool {
c , err := getReplicationConfig ( ctx , bucket )
if err != nil || c == nil {
return false
}
for _ , obj := range objects {
if c . HasActiveRules ( obj . ObjectName , true ) {
return true
}
}
return false
}
2021-01-12 07:36:51 +01:00
// isStandardHeader returns true if header is a supported header and not a custom header
2021-02-04 05:41:33 +01:00
func isStandardHeader ( matchHeaderKey string ) bool {
return equals ( matchHeaderKey , standardHeaders ... )
2021-01-12 07:36:51 +01:00
}
2020-11-20 03:43:58 +01:00
// returns whether object version is a deletemarker and if object qualifies for replication
2021-03-31 02:15:36 +02:00
func checkReplicateDelete ( ctx context . Context , bucket string , dobj ObjectToDelete , oi ObjectInfo , gerr error ) ( replicate , sync bool ) {
2020-11-20 03:43:58 +01:00
rcfg , err := getReplicationConfig ( ctx , bucket )
if err != nil || rcfg == nil {
2021-03-31 02:15:36 +02:00
return false , sync
2020-11-20 03:43:58 +01:00
}
2021-02-19 01:35:37 +01:00
opts := replication . ObjectOpts {
Name : dobj . ObjectName ,
SSEC : crypto . SSEC . IsEncrypted ( oi . UserDefined ) ,
UserTags : oi . UserTags ,
DeleteMarker : oi . DeleteMarker ,
VersionID : dobj . VersionID ,
2021-03-13 19:28:35 +01:00
OpType : replication . DeleteReplicationType ,
2021-02-19 01:35:37 +01:00
}
replicate = rcfg . Replicate ( opts )
2020-11-20 03:43:58 +01:00
// when incoming delete is removal of a delete marker( a.k.a versioned delete),
// GetObjectInfo returns extra information even though it returns errFileNotFound
2020-11-25 20:24:50 +01:00
if gerr != nil {
2020-11-20 03:43:58 +01:00
validReplStatus := false
switch oi . ReplicationStatus {
2021-01-13 20:52:28 +01:00
case replication . Pending , replication . Completed , replication . Failed :
2020-11-20 03:43:58 +01:00
validReplStatus = true
}
2021-02-19 01:35:37 +01:00
if oi . DeleteMarker && ( validReplStatus || replicate ) {
2021-03-31 02:15:36 +02:00
return true , sync
2020-11-20 03:43:58 +01:00
}
2021-02-18 09:33:51 +01:00
// can be the case that other cluster is down and duplicate `mc rm --vid`
// is issued - this still needs to be replicated back to the other target
2021-03-31 02:15:36 +02:00
return oi . VersionPurgeStatus == Pending || oi . VersionPurgeStatus == Failed , sync
2021-02-10 00:11:43 +01:00
}
2021-01-12 07:36:51 +01:00
tgt := globalBucketTargetSys . GetRemoteTargetClient ( ctx , rcfg . RoleArn )
2021-01-27 20:22:34 +01:00
// the target online status should not be used here while deciding
// whether to replicate deletes as the target could be temporarily down
if tgt == nil {
2021-03-31 02:15:36 +02:00
return false , false
2020-11-20 03:43:58 +01:00
}
2021-03-31 02:15:36 +02:00
return replicate , tgt . replicateSync
2020-11-20 03:43:58 +01:00
}
// replicate deletes to the designated replication target if replication configuration
// has delete marker replication or delete replication (MinIO extension to allow deletes where version id
// is specified) enabled.
// Similar to bucket replication for PUT operation, soft delete (a.k.a setting delete marker) and
// permanent deletes (by specifying a version ID in the delete operation) have three states "Pending", "Complete"
// and "Failed" to mark the status of the replication of "DELETE" operation. All failed operations can
// then be retried by healing. In the case of permanent deletes, until the replication is completed on the
// target cluster, the object version is marked deleted on the source and hidden from listing. It is permanently
// deleted from the source when the VersionPurgeStatus changes to "Complete", i.e after replication succeeds
// on target.
func replicateDelete ( ctx context . Context , dobj DeletedObjectVersionInfo , objectAPI ObjectLayer ) {
bucket := dobj . Bucket
2021-02-04 05:41:33 +01:00
versionID := dobj . DeleteMarkerVersionID
if versionID == "" {
versionID = dobj . VersionID
}
2020-11-20 03:43:58 +01:00
rcfg , err := getReplicationConfig ( ctx , bucket )
if err != nil || rcfg == nil {
2021-01-27 20:22:34 +01:00
logger . LogIf ( ctx , err )
2021-02-04 05:41:33 +01:00
sendEvent ( eventArgs {
BucketName : bucket ,
Object : ObjectInfo {
Bucket : bucket ,
Name : dobj . ObjectName ,
VersionID : versionID ,
DeleteMarker : dobj . DeleteMarker ,
} ,
Host : "Internal: [Replication]" ,
EventName : event . ObjectReplicationNotTracked ,
} )
2020-11-20 03:43:58 +01:00
return
}
2021-02-10 00:11:43 +01:00
2020-11-20 03:43:58 +01:00
tgt := globalBucketTargetSys . GetRemoteTargetClient ( ctx , rcfg . RoleArn )
if tgt == nil {
2021-01-27 20:22:34 +01:00
logger . LogIf ( ctx , fmt . Errorf ( "failed to get target for bucket:%s arn:%s" , bucket , rcfg . RoleArn ) )
2021-02-04 05:41:33 +01:00
sendEvent ( eventArgs {
BucketName : bucket ,
Object : ObjectInfo {
Bucket : bucket ,
Name : dobj . ObjectName ,
VersionID : versionID ,
DeleteMarker : dobj . DeleteMarker ,
} ,
Host : "Internal: [Replication]" ,
EventName : event . ObjectReplicationNotTracked ,
} )
2020-11-20 03:43:58 +01:00
return
}
2021-02-04 05:41:33 +01:00
2020-11-20 03:43:58 +01:00
rmErr := tgt . RemoveObject ( ctx , rcfg . GetDestination ( ) . Bucket , dobj . ObjectName , miniogo . RemoveObjectOptions {
VersionID : versionID ,
Internal : miniogo . AdvancedRemoveOptions {
ReplicationDeleteMarker : dobj . DeleteMarkerVersionID != "" ,
2020-11-29 06:15:45 +01:00
ReplicationMTime : dobj . DeleteMarkerMTime . Time ,
2020-11-20 03:43:58 +01:00
ReplicationStatus : miniogo . ReplicationStatusReplica ,
2021-03-03 20:13:31 +01:00
ReplicationRequest : true , // always set this to distinguish between `mc mirror` replication and serverside
2020-11-20 03:43:58 +01:00
} ,
} )
replicationStatus := dobj . DeleteMarkerReplicationStatus
versionPurgeStatus := dobj . VersionPurgeStatus
if rmErr != nil {
if dobj . VersionID == "" {
replicationStatus = string ( replication . Failed )
} else {
versionPurgeStatus = Failed
}
2021-02-11 07:00:42 +01:00
logger . LogIf ( ctx , fmt . Errorf ( "Unable to replicate delete marker to %s/%s(%s): %s" , rcfg . GetDestination ( ) . Bucket , dobj . ObjectName , versionID , rmErr ) )
2020-11-20 03:43:58 +01:00
} else {
if dobj . VersionID == "" {
2021-01-13 20:52:28 +01:00
replicationStatus = string ( replication . Completed )
2020-11-20 03:43:58 +01:00
} else {
versionPurgeStatus = Complete
}
}
2021-04-03 18:03:42 +02:00
prevStatus := dobj . DeleteMarkerReplicationStatus
currStatus := replicationStatus
if dobj . VersionID != "" {
prevStatus = string ( dobj . VersionPurgeStatus )
currStatus = string ( versionPurgeStatus )
}
globalReplicationStats . Update ( ctx , dobj . Bucket , 0 , replication . StatusType ( currStatus ) , replication . StatusType ( prevStatus ) , replication . DeleteReplicationType ) // to decrement pending count
2021-01-25 23:04:41 +01:00
2020-11-22 08:48:50 +01:00
var eventName = event . ObjectReplicationComplete
2020-11-20 03:43:58 +01:00
if replicationStatus == string ( replication . Failed ) || versionPurgeStatus == Failed {
2020-11-22 08:48:50 +01:00
eventName = event . ObjectReplicationFailed
}
2020-11-20 03:43:58 +01:00
// Update metadata on the delete marker or purge permanent delete if replication success.
2021-02-04 05:41:33 +01:00
dobjInfo , err := objectAPI . DeleteObject ( ctx , bucket , dobj . ObjectName , ObjectOptions {
2020-11-20 03:43:58 +01:00
VersionID : versionID ,
DeleteMarkerReplicationStatus : replicationStatus ,
VersionPurgeStatus : versionPurgeStatus ,
2021-02-04 05:41:33 +01:00
Versioned : globalBucketVersioningSys . Enabled ( bucket ) ,
2020-11-20 03:43:58 +01:00
VersionSuspended : globalBucketVersioningSys . Suspended ( bucket ) ,
2021-01-25 23:04:41 +01:00
} )
2021-02-10 00:11:43 +01:00
if err != nil && ! isErrVersionNotFound ( err ) { // VersionNotFound would be reported by pool that object version is missing on.
logger . LogIf ( ctx , fmt . Errorf ( "Unable to update replication metadata for %s/%s(%s): %s" , bucket , dobj . ObjectName , versionID , err ) )
2021-02-04 05:41:33 +01:00
sendEvent ( eventArgs {
BucketName : bucket ,
Object : ObjectInfo {
Bucket : bucket ,
Name : dobj . ObjectName ,
VersionID : versionID ,
DeleteMarker : dobj . DeleteMarker ,
} ,
Host : "Internal: [Replication]" ,
EventName : eventName ,
} )
} else {
sendEvent ( eventArgs {
BucketName : bucket ,
Object : dobjInfo ,
Host : "Internal: [Replication]" ,
EventName : eventName ,
} )
2020-11-20 03:43:58 +01:00
}
}
2020-11-19 20:50:22 +01:00
func getCopyObjMetadata ( oi ObjectInfo , dest replication . Destination ) map [ string ] string {
meta := make ( map [ string ] string , len ( oi . UserDefined ) )
for k , v := range oi . UserDefined {
2021-02-04 05:41:33 +01:00
if strings . HasPrefix ( strings . ToLower ( k ) , ReservedMetadataPrefixLower ) {
2020-11-19 20:50:22 +01:00
continue
}
2021-02-04 05:41:33 +01:00
if equals ( k , xhttp . AmzBucketReplicationStatus ) {
continue
}
// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
if equals ( k , xhttp . AmzMetaUnencryptedContentLength , xhttp . AmzMetaUnencryptedContentMD5 ) {
2020-11-19 20:50:22 +01:00
continue
}
2021-02-04 05:41:33 +01:00
2020-11-19 20:50:22 +01:00
meta [ k ] = v
}
2021-02-04 05:41:33 +01:00
2020-11-19 20:50:22 +01:00
if oi . ContentEncoding != "" {
meta [ xhttp . ContentEncoding ] = oi . ContentEncoding
}
2021-02-04 05:41:33 +01:00
2020-11-19 20:50:22 +01:00
if oi . ContentType != "" {
meta [ xhttp . ContentType ] = oi . ContentType
}
2021-02-04 05:41:33 +01:00
if oi . UserTags != "" {
meta [ xhttp . AmzObjectTagging ] = oi . UserTags
2020-11-19 20:50:22 +01:00
meta [ xhttp . AmzTagDirective ] = "REPLACE"
}
2021-02-04 05:41:33 +01:00
2020-11-19 20:50:22 +01:00
sc := dest . StorageClass
if sc == "" {
sc = oi . StorageClass
}
2021-02-04 05:41:33 +01:00
if sc != "" {
meta [ xhttp . AmzStorageClass ] = sc
2020-11-19 20:50:22 +01:00
}
meta [ xhttp . MinIOSourceETag ] = oi . ETag
2021-02-04 05:41:33 +01:00
meta [ xhttp . MinIOSourceMTime ] = oi . ModTime . Format ( time . RFC3339Nano )
2020-11-19 20:50:22 +01:00
meta [ xhttp . AmzBucketReplicationStatus ] = replication . Replica . String ( )
return meta
}
2021-02-09 03:12:28 +01:00
type caseInsensitiveMap map [ string ] string
// Lookup map entry case insensitively.
func ( m caseInsensitiveMap ) Lookup ( key string ) ( string , bool ) {
if len ( m ) == 0 {
return "" , false
}
for _ , k := range [ ] string {
key ,
strings . ToLower ( key ) ,
http . CanonicalHeaderKey ( key ) ,
} {
v , ok := m [ k ]
if ok {
return v , ok
}
2021-02-09 01:19:05 +01:00
}
2021-02-09 03:12:28 +01:00
return "" , false
2021-02-09 01:19:05 +01:00
}
func putReplicationOpts ( ctx context . Context , dest replication . Destination , objInfo ObjectInfo ) ( putOpts miniogo . PutObjectOptions , err error ) {
2020-07-22 02:49:56 +02:00
meta := make ( map [ string ] string )
for k , v := range objInfo . UserDefined {
2021-01-12 07:36:51 +01:00
if strings . HasPrefix ( strings . ToLower ( k ) , ReservedMetadataPrefixLower ) {
2020-07-22 02:49:56 +02:00
continue
}
2021-01-12 07:36:51 +01:00
if isStandardHeader ( k ) {
2020-08-13 02:32:24 +02:00
continue
}
2020-07-22 02:49:56 +02:00
meta [ k ] = v
}
2021-02-04 05:41:33 +01:00
2020-08-06 05:01:20 +02:00
sc := dest . StorageClass
if sc == "" {
sc = objInfo . StorageClass
}
2020-07-22 02:49:56 +02:00
putOpts = miniogo . PutObjectOptions {
2020-10-06 17:37:09 +02:00
UserMetadata : meta ,
ContentType : objInfo . ContentType ,
ContentEncoding : objInfo . ContentEncoding ,
StorageClass : sc ,
Internal : miniogo . AdvancedPutOptions {
2021-03-03 20:13:31 +01:00
SourceVersionID : objInfo . VersionID ,
ReplicationStatus : miniogo . ReplicationStatusReplica ,
SourceMTime : objInfo . ModTime ,
SourceETag : objInfo . ETag ,
ReplicationRequest : true , // always set this to distinguish between `mc mirror` replication and serverside
2020-10-06 17:37:09 +02:00
} ,
2020-07-22 02:49:56 +02:00
}
2021-02-04 05:41:33 +01:00
if objInfo . UserTags != "" {
tag , _ := tags . ParseObjectTags ( objInfo . UserTags )
if tag != nil {
putOpts . UserTags = tag . ToMap ( )
}
}
2021-02-09 03:12:28 +01:00
lkMap := caseInsensitiveMap ( objInfo . UserDefined )
if lang , ok := lkMap . Lookup ( xhttp . ContentLanguage ) ; ok {
2021-01-27 20:22:34 +01:00
putOpts . ContentLanguage = lang
}
2021-02-09 03:12:28 +01:00
if disp , ok := lkMap . Lookup ( xhttp . ContentDisposition ) ; ok {
2021-01-27 20:22:34 +01:00
putOpts . ContentDisposition = disp
}
2021-02-09 03:12:28 +01:00
if cc , ok := lkMap . Lookup ( xhttp . CacheControl ) ; ok {
2021-01-27 20:22:34 +01:00
putOpts . CacheControl = cc
}
2021-02-09 03:12:28 +01:00
if mode , ok := lkMap . Lookup ( xhttp . AmzObjectLockMode ) ; ok {
2020-07-22 02:49:56 +02:00
rmode := miniogo . RetentionMode ( mode )
putOpts . Mode = rmode
}
2021-02-09 03:12:28 +01:00
if retainDateStr , ok := lkMap . Lookup ( xhttp . AmzObjectLockRetainUntilDate ) ; ok {
rdate , err := time . Parse ( time . RFC3339 , retainDateStr )
2020-07-22 02:49:56 +02:00
if err != nil {
2021-02-09 03:12:28 +01:00
return putOpts , err
2020-07-22 02:49:56 +02:00
}
putOpts . RetainUntilDate = rdate
}
2021-02-09 03:12:28 +01:00
if lhold , ok := lkMap . Lookup ( xhttp . AmzObjectLockLegalHold ) ; ok {
2020-07-22 02:49:56 +02:00
putOpts . LegalHold = miniogo . LegalHoldStatus ( lhold )
}
if crypto . S3 . IsEncrypted ( objInfo . UserDefined ) {
putOpts . ServerSideEncryption = encrypt . NewSSE ( )
}
return
}
2020-11-19 20:50:22 +01:00
type replicationAction string
const (
replicateMetadata replicationAction = "metadata"
replicateNone replicationAction = "none"
replicateAll replicationAction = "all"
)
2021-02-04 05:41:33 +01:00
// matches k1 with all keys, returns 'true' if one of them matches
func equals ( k1 string , keys ... string ) bool {
for _ , k2 := range keys {
if strings . ToLower ( k1 ) == strings . ToLower ( k2 ) {
return true
}
}
return false
}
2020-11-19 20:50:22 +01:00
// returns replicationAction by comparing metadata between source and target
func getReplicationAction ( oi1 ObjectInfo , oi2 minio . ObjectInfo ) replicationAction {
// needs full replication
if oi1 . ETag != oi2 . ETag ||
oi1 . VersionID != oi2 . VersionID ||
oi1 . Size != oi2 . Size ||
2021-01-27 20:22:34 +01:00
oi1 . DeleteMarker != oi2 . IsDeleteMarker ||
2021-02-04 05:41:33 +01:00
oi1 . ModTime . Unix ( ) != oi2 . LastModified . Unix ( ) {
2020-11-19 20:50:22 +01:00
return replicateAll
}
2021-02-04 05:41:33 +01:00
2021-01-27 20:22:34 +01:00
if oi1 . ContentType != oi2 . ContentType {
2020-11-19 20:50:22 +01:00
return replicateMetadata
}
2021-02-04 05:41:33 +01:00
2020-11-19 20:50:22 +01:00
if oi1 . ContentEncoding != "" {
2021-01-27 20:22:34 +01:00
enc , ok := oi2 . Metadata [ xhttp . ContentEncoding ]
2021-02-04 05:41:33 +01:00
if ! ok {
enc , ok = oi2 . Metadata [ strings . ToLower ( xhttp . ContentEncoding ) ]
if ! ok {
return replicateMetadata
}
}
if strings . Join ( enc , "," ) != oi1 . ContentEncoding {
2020-11-19 20:50:22 +01:00
return replicateMetadata
}
}
2021-02-04 05:41:33 +01:00
t , _ := tags . ParseObjectTags ( oi1 . UserTags )
if ! reflect . DeepEqual ( oi2 . UserTags , t . ToMap ( ) ) {
2020-11-19 20:50:22 +01:00
return replicateMetadata
}
2021-02-04 05:41:33 +01:00
// Compare only necessary headers
compareKeys := [ ] string {
"Expires" ,
"Cache-Control" ,
"Content-Language" ,
"Content-Disposition" ,
"X-Amz-Object-Lock-Mode" ,
"X-Amz-Object-Lock-Retain-Until-Date" ,
"X-Amz-Object-Lock-Legal-Hold" ,
"X-Amz-Website-Redirect-Location" ,
"X-Amz-Meta-" ,
}
// compare metadata on both maps to see if meta is identical
compareMeta1 := make ( map [ string ] string )
for k , v := range oi1 . UserDefined {
var found bool
for _ , prefix := range compareKeys {
if ! strings . HasPrefix ( strings . ToLower ( k ) , strings . ToLower ( prefix ) ) {
continue
}
found = true
break
}
if found {
compareMeta1 [ strings . ToLower ( k ) ] = v
2021-01-27 20:22:34 +01:00
}
}
2021-02-04 05:41:33 +01:00
compareMeta2 := make ( map [ string ] string )
for k , v := range oi2 . Metadata {
var found bool
for _ , prefix := range compareKeys {
if ! strings . HasPrefix ( strings . ToLower ( k ) , strings . ToLower ( prefix ) ) {
continue
}
found = true
break
2021-01-27 20:22:34 +01:00
}
2021-02-04 05:41:33 +01:00
if found {
compareMeta2 [ strings . ToLower ( k ) ] = strings . Join ( v , "," )
2020-11-19 20:50:22 +01:00
}
}
2021-02-04 05:41:33 +01:00
if ! reflect . DeepEqual ( compareMeta1 , compareMeta2 ) {
2020-11-19 20:50:22 +01:00
return replicateMetadata
}
2021-02-04 05:41:33 +01:00
2020-11-19 20:50:22 +01:00
return replicateNone
}
2020-07-22 02:49:56 +02:00
// replicateObject replicates the specified version of the object to destination bucket
// The source object is then updated to reflect the replication status.
2020-09-17 01:04:55 +02:00
func replicateObject ( ctx context . Context , objInfo ObjectInfo , objectAPI ObjectLayer ) {
2021-02-04 05:41:33 +01:00
z , ok := objectAPI . ( * erasureServerPools )
if ! ok {
return
}
2020-09-17 01:04:55 +02:00
bucket := objInfo . Bucket
object := objInfo . Name
2020-07-31 04:55:22 +02:00
cfg , err := getReplicationConfig ( ctx , bucket )
2020-07-22 02:49:56 +02:00
if err != nil {
logger . LogIf ( ctx , err )
2021-02-04 05:41:33 +01:00
sendEvent ( eventArgs {
EventName : event . ObjectReplicationNotTracked ,
BucketName : bucket ,
Object : objInfo ,
Host : "Internal: [Replication]" ,
} )
2020-07-22 02:49:56 +02:00
return
}
2020-10-08 19:54:11 +02:00
tgt := globalBucketTargetSys . GetRemoteTargetClient ( ctx , cfg . RoleArn )
2020-07-22 02:49:56 +02:00
if tgt == nil {
2020-10-10 05:36:00 +02:00
logger . LogIf ( ctx , fmt . Errorf ( "failed to get target for bucket:%s arn:%s" , bucket , cfg . RoleArn ) )
2021-02-04 05:41:33 +01:00
sendEvent ( eventArgs {
EventName : event . ObjectReplicationNotTracked ,
BucketName : bucket ,
Object : objInfo ,
Host : "Internal: [Replication]" ,
} )
2020-07-22 02:49:56 +02:00
return
}
2021-04-03 18:03:42 +02:00
gr , err := objectAPI . GetObjectNInfo ( ctx , bucket , object , nil , http . Header { } , writeLock , ObjectOptions {
2020-09-17 01:04:55 +02:00
VersionID : objInfo . VersionID ,
2020-09-16 05:44:48 +02:00
} )
2020-07-22 02:49:56 +02:00
if err != nil {
2021-02-04 05:41:33 +01:00
sendEvent ( eventArgs {
EventName : event . ObjectReplicationNotTracked ,
BucketName : bucket ,
Object : objInfo ,
Host : "Internal: [Replication]" ,
} )
2021-04-03 18:03:42 +02:00
logger . LogIf ( ctx , fmt . Errorf ( "Unable to update replicate for %s/%s(%s): %w" , bucket , object , objInfo . VersionID , err ) )
2020-07-22 02:49:56 +02:00
return
}
2021-04-03 18:03:42 +02:00
defer gr . Close ( ) // hold write lock for entire transaction
2021-02-04 05:41:33 +01:00
2020-09-17 01:04:55 +02:00
objInfo = gr . ObjInfo
2020-07-22 02:49:56 +02:00
size , err := objInfo . GetActualSize ( )
if err != nil {
logger . LogIf ( ctx , err )
2021-02-04 05:41:33 +01:00
sendEvent ( eventArgs {
EventName : event . ObjectReplicationNotTracked ,
BucketName : bucket ,
Object : objInfo ,
Host : "Internal: [Replication]" ,
} )
2020-07-22 02:49:56 +02:00
return
}
dest := cfg . GetDestination ( )
if dest . Bucket == "" {
2021-02-04 05:41:33 +01:00
logger . LogIf ( ctx , fmt . Errorf ( "Unable to replicate object %s(%s), bucket is empty" , objInfo . Name , objInfo . VersionID ) )
sendEvent ( eventArgs {
EventName : event . ObjectReplicationNotTracked ,
BucketName : bucket ,
Object : objInfo ,
Host : "Internal: [Replication]" ,
} )
2020-07-22 02:49:56 +02:00
return
}
2020-09-17 01:04:55 +02:00
2020-11-19 20:50:22 +01:00
rtype := replicateAll
2021-01-27 20:22:34 +01:00
oi , err := tgt . StatObject ( ctx , dest . Bucket , object , miniogo . StatObjectOptions {
VersionID : objInfo . VersionID ,
Internal : miniogo . AdvancedGetOptions {
ReplicationProxyRequest : "false" ,
} } )
2020-11-19 20:50:22 +01:00
if err == nil {
rtype = getReplicationAction ( objInfo , oi )
if rtype == replicateNone {
2020-07-22 02:49:56 +02:00
// object with same VersionID already exists, replication kicked off by
2021-04-03 18:03:42 +02:00
// PutObject might have completed
2020-07-22 02:49:56 +02:00
return
}
}
2021-01-13 20:52:28 +01:00
replicationStatus := replication . Completed
2021-02-20 09:22:17 +01:00
// use core client to avoid doing multipart on PUT
c := & miniogo . Core { Client : tgt . Client }
2021-01-07 01:13:10 +01:00
if rtype != replicateAll {
2020-11-19 20:50:22 +01:00
// replicate metadata for object tagging/copy with metadata replacement
2021-02-11 02:25:04 +01:00
srcOpts := miniogo . CopySrcOptions {
Bucket : dest . Bucket ,
Object : object ,
2021-04-03 18:03:42 +02:00
VersionID : objInfo . VersionID ,
}
2021-03-03 20:13:31 +01:00
dstOpts := miniogo . PutObjectOptions {
Internal : miniogo . AdvancedPutOptions {
SourceVersionID : objInfo . VersionID ,
ReplicationRequest : true , // always set this to distinguish between `mc mirror` replication and serverside
} }
2021-02-11 02:25:04 +01:00
if _ , err = c . CopyObject ( ctx , dest . Bucket , object , dest . Bucket , object , getCopyObjMetadata ( objInfo , dest ) , srcOpts , dstOpts ) ; err != nil {
2021-01-07 01:13:10 +01:00
replicationStatus = replication . Failed
2021-02-10 00:11:43 +01:00
logger . LogIf ( ctx , fmt . Errorf ( "Unable to replicate metadata for object %s/%s(%s): %s" , bucket , objInfo . Name , objInfo . VersionID , err ) )
2021-01-07 01:13:10 +01:00
}
} else {
target , err := globalBucketMetadataSys . GetBucketTarget ( bucket , cfg . RoleArn )
if err != nil {
2021-02-10 00:11:43 +01:00
logger . LogIf ( ctx , fmt . Errorf ( "failed to get target for replication bucket:%s cfg:%s err:%s" , bucket , cfg . RoleArn , err ) )
2021-02-04 05:41:33 +01:00
sendEvent ( eventArgs {
EventName : event . ObjectReplicationNotTracked ,
BucketName : bucket ,
Object : objInfo ,
Host : "Internal: [Replication]" ,
} )
2021-01-07 01:13:10 +01:00
return
}
2020-11-25 20:24:50 +01:00
2021-02-09 01:19:05 +01:00
putOpts , err := putReplicationOpts ( ctx , dest , objInfo )
if err != nil {
2021-02-09 03:12:28 +01:00
logger . LogIf ( ctx , fmt . Errorf ( "failed to get target for replication bucket:%s cfg:%s err:%w" , bucket , cfg . RoleArn , err ) )
sendEvent ( eventArgs {
EventName : event . ObjectReplicationNotTracked ,
BucketName : bucket ,
Object : objInfo ,
Host : "Internal: [Replication]" ,
} )
2021-02-09 01:19:05 +01:00
return
}
2021-02-09 03:12:28 +01:00
2021-01-07 01:13:10 +01:00
// Setup bandwidth throttling
peers , _ := globalEndpoints . peers ( )
totalNodesCount := len ( peers )
if totalNodesCount == 0 {
totalNodesCount = 1 // For standalone erasure coding
}
b := target . BandwidthLimit / int64 ( totalNodesCount )
var headerSize int
for k , v := range putOpts . Header ( ) {
headerSize += len ( k ) + len ( v )
}
2021-01-08 19:12:26 +01:00
// r takes over closing gr.
2021-01-07 01:13:10 +01:00
r := bandwidth . NewMonitoredReader ( ctx , globalBucketMonitor , objInfo . Bucket , objInfo . Name , gr , headerSize , b , target . BandwidthLimit )
2021-02-20 09:22:17 +01:00
if _ , err = c . PutObject ( ctx , dest . Bucket , object , r , size , "" , "" , putOpts ) ; err != nil {
2021-01-07 01:13:10 +01:00
replicationStatus = replication . Failed
2021-02-09 03:12:28 +01:00
logger . LogIf ( ctx , fmt . Errorf ( "Unable to replicate for object %s/%s(%s): %w" , bucket , objInfo . Name , objInfo . VersionID , err ) )
2021-01-07 01:13:10 +01:00
}
2021-02-04 05:41:33 +01:00
defer r . Close ( )
2020-07-22 02:49:56 +02:00
}
2021-01-25 23:04:41 +01:00
2021-04-03 18:03:42 +02:00
prevReplStatus := objInfo . ReplicationStatus
2020-07-22 02:49:56 +02:00
objInfo . UserDefined [ xhttp . AmzBucketReplicationStatus ] = replicationStatus . String ( )
if objInfo . UserTags != "" {
objInfo . UserDefined [ xhttp . AmzObjectTagging ] = objInfo . UserTags
}
2020-09-17 01:04:55 +02:00
// FIXME: add support for missing replication events
// - event.ObjectReplicationMissedThreshold
// - event.ObjectReplicationReplicatedAfterThreshold
2020-11-16 07:16:41 +01:00
var eventName = event . ObjectReplicationComplete
2020-09-17 01:04:55 +02:00
if replicationStatus == replication . Failed {
2020-11-16 07:16:41 +01:00
eventName = event . ObjectReplicationFailed
2020-09-17 01:04:55 +02:00
}
2021-01-25 23:04:41 +01:00
2021-02-04 05:41:33 +01:00
// This lower level implementation is necessary to avoid write locks from CopyObject.
2021-02-10 20:45:02 +01:00
poolIdx , err := z . getPoolIdx ( ctx , bucket , object , objInfo . Size )
2021-01-25 23:04:41 +01:00
if err != nil {
2021-02-04 05:41:33 +01:00
logger . LogIf ( ctx , fmt . Errorf ( "Unable to update replication metadata for %s/%s(%s): %w" , bucket , objInfo . Name , objInfo . VersionID , err ) )
} else {
if err = z . serverPools [ poolIdx ] . getHashedSet ( object ) . updateObjectMeta ( ctx , bucket , object , objInfo . UserDefined , ObjectOptions {
VersionID : objInfo . VersionID ,
} ) ; err != nil {
logger . LogIf ( ctx , fmt . Errorf ( "Unable to update replication metadata for %s/%s(%s): %w" , bucket , objInfo . Name , objInfo . VersionID , err ) )
}
2020-07-22 02:49:56 +02:00
}
2021-04-03 18:03:42 +02:00
opType := replication . MetadataReplicationType
if rtype == replicateAll {
opType = replication . ObjectReplicationType
}
globalReplicationStats . Update ( ctx , bucket , size , replicationStatus , prevReplStatus , opType )
2021-01-25 23:04:41 +01:00
sendEvent ( eventArgs {
EventName : eventName ,
BucketName : bucket ,
Object : objInfo ,
Host : "Internal: [Replication]" ,
} )
2020-07-22 02:49:56 +02:00
}
2020-08-13 02:32:24 +02:00
// filterReplicationStatusMetadata filters replication status metadata for COPY
func filterReplicationStatusMetadata ( metadata map [ string ] string ) map [ string ] string {
// Copy on write
dst := metadata
var copied bool
delKey := func ( key string ) {
if _ , ok := metadata [ key ] ; ! ok {
return
}
if ! copied {
dst = make ( map [ string ] string , len ( metadata ) )
for k , v := range metadata {
dst [ k ] = v
}
copied = true
}
delete ( dst , key )
}
delKey ( xhttp . AmzBucketReplicationStatus )
return dst
}
2020-09-17 01:04:55 +02:00
2020-11-20 03:43:58 +01:00
// DeletedObjectVersionInfo has info on deleted object
type DeletedObjectVersionInfo struct {
DeletedObject
Bucket string
}
2020-09-21 22:43:29 +02:00
var (
2021-04-03 18:03:42 +02:00
globalReplicationPool * ReplicationPool
globalReplicationStats * ReplicationStats
2020-09-21 22:43:29 +02:00
)
2020-09-17 01:04:55 +02:00
2021-03-09 11:56:42 +01:00
// ReplicationPool describes replication pool
type ReplicationPool struct {
2021-04-03 18:03:42 +02:00
mu sync . Mutex
size int
replicaCh chan ObjectInfo
replicaDeleteCh chan DeletedObjectVersionInfo
mrfReplicaCh chan ObjectInfo
mrfReplicaDeleteCh chan DeletedObjectVersionInfo
killCh chan struct { }
wg sync . WaitGroup
ctx context . Context
objLayer ObjectLayer
2021-03-09 11:56:42 +01:00
}
// NewReplicationPool creates a pool of replication workers of specified size
func NewReplicationPool ( ctx context . Context , o ObjectLayer , sz int ) * ReplicationPool {
pool := & ReplicationPool {
2021-04-03 18:03:42 +02:00
replicaCh : make ( chan ObjectInfo , 1000 ) ,
replicaDeleteCh : make ( chan DeletedObjectVersionInfo , 1000 ) ,
mrfReplicaCh : make ( chan ObjectInfo , 100000 ) ,
mrfReplicaDeleteCh : make ( chan DeletedObjectVersionInfo , 100000 ) ,
ctx : ctx ,
objLayer : o ,
}
2021-03-09 11:56:42 +01:00
pool . Resize ( sz )
2021-04-03 18:03:42 +02:00
// add long running worker for handling most recent failures/pending replications
go pool . AddMRFWorker ( )
2021-03-09 11:56:42 +01:00
return pool
2020-09-17 01:04:55 +02:00
}
2021-04-03 18:03:42 +02:00
// AddMRFWorker adds a pending/failed replication worker to handle requests that could not be queued
// to the other workers
func ( p * ReplicationPool ) AddMRFWorker ( ) {
for {
select {
case <- p . ctx . Done ( ) :
return
case oi , ok := <- p . mrfReplicaCh :
if ! ok {
return
}
replicateObject ( p . ctx , oi , p . objLayer )
case doi , ok := <- p . mrfReplicaDeleteCh :
if ! ok {
return
}
replicateDelete ( p . ctx , doi , p . objLayer )
}
}
}
2021-03-09 11:56:42 +01:00
// AddWorker adds a replication worker to the pool
func ( p * ReplicationPool ) AddWorker ( ) {
defer p . wg . Done ( )
for {
select {
case <- p . ctx . Done ( ) :
return
case oi , ok := <- p . replicaCh :
if ! ok {
2020-09-17 01:04:55 +02:00
return
}
2021-03-09 11:56:42 +01:00
replicateObject ( p . ctx , oi , p . objLayer )
case doi , ok := <- p . replicaDeleteCh :
if ! ok {
return
}
replicateDelete ( p . ctx , doi , p . objLayer )
case <- p . killCh :
return
2020-09-17 01:04:55 +02:00
}
2021-03-09 11:56:42 +01:00
}
2020-09-17 01:04:55 +02:00
}
2020-09-21 22:43:29 +02:00
2021-03-09 11:56:42 +01:00
//Resize replication pool to new size
func ( p * ReplicationPool ) Resize ( n int ) {
p . mu . Lock ( )
defer p . mu . Unlock ( )
for p . size < n {
p . size ++
p . wg . Add ( 1 )
go p . AddWorker ( )
}
for p . size > n {
p . size --
go func ( ) { p . killCh <- struct { } { } } ( )
}
}
2021-04-03 18:03:42 +02:00
func ( p * ReplicationPool ) queueReplicaTask ( ctx context . Context , oi ObjectInfo ) {
2021-03-09 11:56:42 +01:00
if p == nil {
2020-09-21 22:43:29 +02:00
return
}
2021-03-09 11:56:42 +01:00
select {
2021-04-03 18:03:42 +02:00
case <- ctx . Done ( ) :
close ( p . replicaCh )
close ( p . mrfReplicaCh )
2021-03-09 11:56:42 +01:00
case p . replicaCh <- oi :
2021-04-03 18:03:42 +02:00
case p . mrfReplicaCh <- oi :
// queue all overflows into the mrfReplicaCh to handle incoming pending/failed operations
2021-03-09 11:56:42 +01:00
default :
}
}
2020-09-21 22:43:29 +02:00
2021-04-03 18:03:42 +02:00
func ( p * ReplicationPool ) queueReplicaDeleteTask ( ctx context . Context , doi DeletedObjectVersionInfo ) {
2021-03-09 11:56:42 +01:00
if p == nil {
return
2020-09-21 22:43:29 +02:00
}
2021-03-09 11:56:42 +01:00
select {
2021-04-03 18:03:42 +02:00
case <- ctx . Done ( ) :
close ( p . replicaDeleteCh )
close ( p . mrfReplicaDeleteCh )
2021-03-09 11:56:42 +01:00
case p . replicaDeleteCh <- doi :
2021-04-03 18:03:42 +02:00
case p . mrfReplicaDeleteCh <- doi :
// queue all overflows into the mrfReplicaDeleteCh to handle incoming pending/failed operations
2021-03-09 11:56:42 +01:00
default :
}
}
func initBackgroundReplication ( ctx context . Context , objectAPI ObjectLayer ) {
globalReplicationPool = NewReplicationPool ( ctx , objectAPI , globalAPIConfig . getReplicationWorkers ( ) )
2021-04-03 18:03:42 +02:00
globalReplicationStats = NewReplicationStats ( ctx , objectAPI )
2020-09-21 22:43:29 +02:00
}
2021-01-12 07:36:51 +01:00
// get Reader from replication target if active-active replication is in place and
// this node returns a 404
func proxyGetToReplicationTarget ( ctx context . Context , bucket , object string , rs * HTTPRangeSpec , h http . Header , opts ObjectOptions ) ( gr * GetObjectReader , proxy bool ) {
tgt , oi , proxy , err := proxyHeadToRepTarget ( ctx , bucket , object , opts )
if ! proxy || err != nil {
return nil , false
}
fn , off , length , err := NewGetObjectReader ( rs , oi , opts )
if err != nil {
return nil , false
}
gopts := miniogo . GetObjectOptions {
VersionID : opts . VersionID ,
ServerSideEncryption : opts . ServerSideEncryption ,
Internal : miniogo . AdvancedGetOptions {
2021-01-27 20:22:34 +01:00
ReplicationProxyRequest : "true" ,
2021-01-12 07:36:51 +01:00
} ,
}
// get correct offsets for encrypted object
if off >= 0 && length >= 0 {
if err := gopts . SetRange ( off , off + length - 1 ) ; err != nil {
return nil , false
}
}
2021-02-04 05:41:33 +01:00
// Make sure to match ETag when proxying.
if err = gopts . SetMatchETag ( oi . ETag ) ; err != nil {
return nil , false
}
2021-01-12 07:36:51 +01:00
c := miniogo . Core { Client : tgt . Client }
obj , _ , _ , err := c . GetObject ( ctx , bucket , object , gopts )
if err != nil {
return nil , false
}
closeReader := func ( ) { obj . Close ( ) }
reader , err := fn ( obj , h , opts . CheckPrecondFn , closeReader )
if err != nil {
return nil , false
}
2021-02-11 02:25:04 +01:00
reader . ObjInfo = oi . Clone ( )
2021-01-12 07:36:51 +01:00
return reader , true
}
// isProxyable returns true if replication config found for this bucket
func isProxyable ( ctx context . Context , bucket string ) bool {
cfg , err := getReplicationConfig ( ctx , bucket )
if err != nil {
return false
}
dest := cfg . GetDestination ( )
return dest . Bucket == bucket
}
2021-01-27 20:22:34 +01:00
2021-01-12 07:36:51 +01:00
func proxyHeadToRepTarget ( ctx context . Context , bucket , object string , opts ObjectOptions ) ( tgt * TargetClient , oi ObjectInfo , proxy bool , err error ) {
// this option is set when active-active replication is in place between site A -> B,
// and site B does not have the object yet.
2021-01-27 20:22:34 +01:00
if opts . ProxyRequest || ( opts . ProxyHeaderSet && ! opts . ProxyRequest ) { // true only when site B sets MinIOSourceProxyRequest header
2021-01-12 07:36:51 +01:00
return nil , oi , false , nil
}
cfg , err := getReplicationConfig ( ctx , bucket )
if err != nil {
return nil , oi , false , err
}
dest := cfg . GetDestination ( )
if dest . Bucket != bucket { // not active-active
return nil , oi , false , err
}
ssec := false
if opts . ServerSideEncryption != nil {
ssec = opts . ServerSideEncryption . Type ( ) == encrypt . SSEC
}
ropts := replication . ObjectOpts {
Name : object ,
SSEC : ssec ,
}
if ! cfg . Replicate ( ropts ) { // no matching rule for object prefix
return nil , oi , false , nil
}
tgt = globalBucketTargetSys . GetRemoteTargetClient ( ctx , cfg . RoleArn )
if tgt == nil || tgt . isOffline ( ) {
2021-01-27 20:22:34 +01:00
return nil , oi , false , fmt . Errorf ( "target is offline or not configured" )
2021-01-12 07:36:51 +01:00
}
gopts := miniogo . GetObjectOptions {
VersionID : opts . VersionID ,
ServerSideEncryption : opts . ServerSideEncryption ,
Internal : miniogo . AdvancedGetOptions {
2021-01-27 20:22:34 +01:00
ReplicationProxyRequest : "true" ,
2021-01-12 07:36:51 +01:00
} ,
}
objInfo , err := tgt . StatObject ( ctx , dest . Bucket , object , gopts )
if err != nil {
return nil , oi , false , err
}
2021-02-04 05:41:33 +01:00
2021-01-12 07:36:51 +01:00
tags , _ := tags . MapToObjectTags ( objInfo . UserTags )
oi = ObjectInfo {
Bucket : bucket ,
Name : object ,
ModTime : objInfo . LastModified ,
Size : objInfo . Size ,
ETag : objInfo . ETag ,
VersionID : objInfo . VersionID ,
IsLatest : objInfo . IsLatest ,
DeleteMarker : objInfo . IsDeleteMarker ,
ContentType : objInfo . ContentType ,
Expires : objInfo . Expires ,
StorageClass : objInfo . StorageClass ,
ReplicationStatus : replication . StatusType ( objInfo . ReplicationStatus ) ,
UserTags : tags . String ( ) ,
}
2021-02-08 07:01:10 +01:00
oi . UserDefined = make ( map [ string ] string , len ( objInfo . Metadata ) )
2021-02-04 05:41:33 +01:00
for k , v := range objInfo . Metadata {
oi . UserDefined [ k ] = v [ 0 ]
}
ce , ok := oi . UserDefined [ xhttp . ContentEncoding ]
if ! ok {
ce , ok = oi . UserDefined [ strings . ToLower ( xhttp . ContentEncoding ) ]
}
if ok {
2021-01-12 07:36:51 +01:00
oi . ContentEncoding = ce
}
return tgt , oi , true , nil
}
// get object info from replication target if active-active replication is in place and
// this node returns a 404
func proxyHeadToReplicationTarget ( ctx context . Context , bucket , object string , opts ObjectOptions ) ( oi ObjectInfo , proxy bool , err error ) {
_ , oi , proxy , err = proxyHeadToRepTarget ( ctx , bucket , object , opts )
return oi , proxy , err
}
2021-04-03 18:03:42 +02:00
func scheduleReplication ( ctx context . Context , objInfo ObjectInfo , o ObjectLayer , sync bool , opType replication . Type ) {
2021-01-12 07:36:51 +01:00
if sync {
replicateObject ( ctx , objInfo , o )
} else {
2021-04-03 18:03:42 +02:00
globalReplicationPool . queueReplicaTask ( GlobalContext , objInfo )
}
if sz , err := objInfo . GetActualSize ( ) ; err == nil {
globalReplicationStats . Update ( ctx , objInfo . Bucket , sz , objInfo . ReplicationStatus , replication . StatusType ( "" ) , opType )
2021-01-12 07:36:51 +01:00
}
}
func scheduleReplicationDelete ( ctx context . Context , dv DeletedObjectVersionInfo , o ObjectLayer , sync bool ) {
if sync {
replicateDelete ( ctx , dv , o )
} else {
2021-04-03 18:03:42 +02:00
globalReplicationPool . queueReplicaDeleteTask ( GlobalContext , dv )
2021-01-12 07:36:51 +01:00
}
2021-04-03 18:03:42 +02:00
globalReplicationStats . Update ( ctx , dv . Bucket , 0 , replication . Pending , replication . StatusType ( "" ) , replication . DeleteReplicationType )
2021-01-12 07:36:51 +01:00
}