maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add support for object locking with legal hold. (#8634)

Browse source
This commit is contained in:
poornas 2020-01-16 15:41:56 -08:00 committed by kannappanr
parent ba758361b3
commit 60e60f68dd
21 changed files with 1559 additions and 517 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
cmd/api-errors.go
View file

@ -34,6 +34,7 @@ import (
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/event"
"github.com/minio/minio/pkg/hash"
"github.com/minio/minio/pkg/objectlock"
"github.com/minio/minio/pkg/policy"
)
@ -1611,14 +1612,16 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
apiErr = ErrOperationTimedOut
case errDiskNotFound:
apiErr = ErrSlowDown
case errInvalidRetentionDate:
case objectlock.ErrInvalidRetentionDate:
apiErr = ErrInvalidRetentionDate
case errPastObjectLockRetainDate:
case objectlock.ErrPastObjectLockRetainDate:
apiErr = ErrPastObjectLockRetainDate
case errUnknownWORMModeDirective:
case objectlock.ErrUnknownWORMModeDirective:
apiErr = ErrUnknownWORMModeDirective
case errObjectLockInvalidHeaders:
case objectlock.ErrObjectLockInvalidHeaders:
apiErr = ErrObjectLockInvalidHeaders
case objectlock.ErrMalformedXML:
apiErr = ErrMalformedXML
}
// Compression errors

9
cmd/api-router.go
View file

@ -111,20 +111,21 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(collectAPIStats("selectobjectcontent", httpTraceHdrs(api.SelectObjectContentHandler))).Queries("select", "").Queries("select-type", "2")
// GetObjectRetention
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectretention", httpTraceHdrs(api.GetObjectRetentionHandler))).Queries("retention", "")
// GetObjectLegalHold
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectlegalhold", httpTraceHdrs(api.GetObjectLegalHoldHandler))).Queries("legal-hold", "")
// GetObject
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobject", httpTraceHdrs(api.GetObjectHandler)))
// CopyObject
bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(collectAPIStats("copyobject", httpTraceAll(api.CopyObjectHandler)))
// PutObjectRetention
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectretention", httpTraceHdrs(api.PutObjectRetentionHandler))).Queries("retention", "")
// PutObjectLegalHold
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectlegalhold", httpTraceHdrs(api.PutObjectLegalHoldHandler))).Queries("legal-hold", "")
// PutObject
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobject", httpTraceHdrs(api.PutObjectHandler)))
// DeleteObject
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(collectAPIStats("deleteobject", httpTraceAll(api.DeleteObjectHandler)))
// PutObjectLegalHold
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectlegalhold", httpTraceHdrs(api.PutObjectLegalHoldHandler))).Queries("legal-hold", "")
// GetObjectLegalHold
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectlegalhold", httpTraceHdrs(api.GetObjectLegalHoldHandler))).Queries("legal-hold", "")
/// Bucket operations
// GetBucketLocation

11
cmd/bucket-handlers.go
View file

@ -40,6 +40,7 @@ import (
"github.com/minio/minio/pkg/handlers"
"github.com/minio/minio/pkg/hash"
iampolicy "github.com/minio/minio/pkg/iam/policy"
"github.com/minio/minio/pkg/objectlock"
"github.com/minio/minio/pkg/policy"
"github.com/minio/minio/pkg/sync/errgroup"
)
@ -577,14 +578,14 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
return
}
if objectLockEnabled {
if objectLockEnabled && !globalIsGateway {
configFile := path.Join(bucketConfigPrefix, bucket, bucketObjectLockEnabledConfigFile)
if err = saveConfig(ctx, objectAPI, configFile, []byte(bucketObjectLockEnabledConfig)); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
globalBucketObjectLockConfig.Set(bucket, Retention{})
globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, Retention{})
globalBucketObjectLockConfig.Set(bucket, objectlock.Retention{})
globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, objectlock.Retention{})
}
// Make sure to add Location information here only for bucket
@ -1005,7 +1006,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
return
}
config, err := parseObjectLockConfig(r.Body)
config, err := objectlock.ParseObjectLockConfig(r.Body)
if err != nil {
apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
apiErr.Description = err.Error()
@ -1099,7 +1100,7 @@ func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWri
return
}
if configData, err = xml.Marshal(newObjectLockConfig()); err != nil {
if configData, err = xml.Marshal(objectlock.NewObjectLockConfig()); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}

14
cmd/disk-cache.go
View file

@ -32,6 +32,7 @@ import (
"github.com/minio/minio/cmd/config/cache"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/color"
"github.com/minio/minio/pkg/objectlock"
"github.com/minio/minio/pkg/sync/errgroup"
"github.com/minio/minio/pkg/wildcard"
)
@ -251,7 +252,13 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
c.cacheStats.incMiss()
return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
}
// skip cache for objects with locks
objRetention := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
legalHold := objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined)
if objRetention.Mode != objectlock.Invalid || legalHold.Status != "" {
c.cacheStats.incMiss()
return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
}
if cacheErr == nil {
// if ETag matches for stale cache entry, serve from cache
if cacheReader.ObjInfo.ETag == objInfo.ETag {
@ -596,8 +603,9 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
}
// skip cache for objects with locks
objRetention := getObjectRetentionMeta(opts.UserDefined)
if objRetention.Mode == Governance || objRetention.Mode == Compliance {
objRetention := objectlock.GetObjectRetentionMeta(opts.UserDefined)
legalHold := objectlock.GetObjectLegalHoldMeta(opts.UserDefined)
if objRetention.Mode != objectlock.Invalid || legalHold.Status != "" {
dcache.Delete(ctx, bucket, object)
return putObjectFn(ctx, bucket, object, r, opts)
}

3
cmd/globals.go
View file

@ -37,6 +37,7 @@ import (
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/certs"
"github.com/minio/minio/pkg/event"
"github.com/minio/minio/pkg/objectlock"
"github.com/minio/minio/pkg/pubsub"
)
@ -201,7 +202,7 @@ var (
// Is worm enabled
globalWORMEnabled bool
globalBucketObjectLockConfig = newBucketObjectLockConfig()
globalBucketObjectLockConfig = objectlock.NewBucketObjectLockConfig()
// Disk cache drives
globalCacheConfig cache.Config

11
cmd/notification.go
View file

@ -36,6 +36,7 @@ import (
"github.com/minio/minio/pkg/lifecycle"
"github.com/minio/minio/pkg/madmin"
xnet "github.com/minio/minio/pkg/net"
"github.com/minio/minio/pkg/objectlock"
"github.com/minio/minio/pkg/policy"
"github.com/minio/minio/pkg/sync/errgroup"
)
@ -668,7 +669,7 @@ func (sys *NotificationSys) initBucketObjectLockConfig(objAPI ObjectLayer) error
if string(bucketObjLockData) != bucketObjectLockEnabledConfig {
// this should never happen
logger.LogIf(ctx, errMalformedBucketObjectConfig)
logger.LogIf(ctx, objectlock.ErrMalformedBucketObjectConfig)
continue
}
@ -677,17 +678,17 @@ func (sys *NotificationSys) initBucketObjectLockConfig(objAPI ObjectLayer) error
if err != nil {
if err == errConfigNotFound {
globalBucketObjectLockConfig.Set(bucket.Name, Retention{})
globalBucketObjectLockConfig.Set(bucket.Name, objectlock.Retention{})
continue
}
return err
}
config, err := parseObjectLockConfig(bytes.NewReader(configData))
config, err := objectlock.ParseObjectLockConfig(bytes.NewReader(configData))
if err != nil {
return err
}
retention := Retention{}
retention := objectlock.Retention{}
if config.Rule != nil {
retention = config.ToRetention()
}
@ -874,7 +875,7 @@ func (sys *NotificationSys) Send(args eventArgs) []event.TargetIDErr {
}
// PutBucketObjectLockConfig - put bucket object lock configuration to all peers.
func (sys *NotificationSys) PutBucketObjectLockConfig(ctx context.Context, bucketName string, retention Retention) {
func (sys *NotificationSys) PutBucketObjectLockConfig(ctx context.Context, bucketName string, retention objectlock.Retention) {
g := errgroup.WithNErrs(len(sys.peerClients))
for index, client := range sys.peerClients {
if client == nil {

138
cmd/object-handlers.go
View file

@ -44,6 +44,8 @@ import (
"github.com/minio/minio/pkg/event"
"github.com/minio/minio/pkg/handlers"
"github.com/minio/minio/pkg/hash"
"github.com/minio/minio/pkg/objectlock"
iampolicy "github.com/minio/minio/pkg/iam/policy"
"github.com/minio/minio/pkg/ioutil"
"github.com/minio/minio/pkg/policy"
@ -209,8 +211,10 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
return
}
getRetPerms := checkRequestAuthType(ctx, r, policy.GetObjectRetentionAction, bucket, object)
legalHoldPerms := checkRequestAuthType(ctx, r, policy.GetObjectLegalHoldAction, bucket, object)
// filter object lock metadata if permission does not permit
objInfo.UserDefined = filterObjectLockMetadata(ctx, r, bucket, object, objInfo.UserDefined, false, getRetPerms)
objInfo.UserDefined = objectlock.FilterObjectLockMetadata(objInfo.UserDefined, getRetPerms != ErrNone, legalHoldPerms != ErrNone)
if err = s3Select.Open(getObject); err != nil {
if serr, ok := err.(s3select.SelectError); ok {
@ -353,7 +357,10 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
// filter object lock metadata if permission does not permit
getRetPerms := checkRequestAuthType(ctx, r, policy.GetObjectRetentionAction, bucket, object)
objInfo.UserDefined = filterObjectLockMetadata(ctx, r, bucket, object, objInfo.UserDefined, false, getRetPerms)
legalHoldPerms := checkRequestAuthType(ctx, r, policy.GetObjectLegalHoldAction, bucket, object)
// filter object lock metadata if permission does not permit
objInfo.UserDefined = objectlock.FilterObjectLockMetadata(objInfo.UserDefined, getRetPerms != ErrNone, legalHoldPerms != ErrNone)
if objectAPI.IsEncryptionSupported() {
objInfo.UserDefined = CleanMinioInternalMetadataKeys(objInfo.UserDefined)
@ -518,7 +525,10 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
// filter object lock metadata if permission does not permit
getRetPerms := checkRequestAuthType(ctx, r, policy.GetObjectRetentionAction, bucket, object)
objInfo.UserDefined = filterObjectLockMetadata(ctx, r, bucket, object, objInfo.UserDefined, false, getRetPerms)
legalHoldPerms := checkRequestAuthType(ctx, r, policy.GetObjectLegalHoldAction, bucket, object)
// filter object lock metadata if permission does not permit
objInfo.UserDefined = objectlock.FilterObjectLockMetadata(objInfo.UserDefined, getRetPerms != ErrNone, legalHoldPerms != ErrNone)
if objectAPI.IsEncryptionSupported() {
if _, err = DecryptObjectInfo(&objInfo, r.Header); err != nil {
@ -962,16 +972,19 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
if api.CacheAPI() != nil {
getObjectInfo = api.CacheAPI().GetObjectInfo
}
isCpy := true
getRetPerms := checkRequestAuthType(ctx, r, policy.GetObjectRetentionAction, srcBucket, srcObject)
srcInfo.UserDefined = filterObjectLockMetadata(ctx, r, srcBucket, srcObject, srcInfo.UserDefined, isCpy, getRetPerms)
srcInfo.UserDefined = objectlock.FilterObjectLockMetadata(srcInfo.UserDefined, true, true)
retPerms := isPutActionAllowed(getRequestAuthType(r), dstBucket, dstObject, r, iampolicy.PutObjectRetentionAction)
holdPerms := isPutActionAllowed(getRequestAuthType(r), dstBucket, dstObject, r, iampolicy.PutObjectLegalHoldAction)
// apply default bucket configuration/governance headers for dest side.
retentionMode, retentionDate, s3Err := checkPutObjectRetentionAllowed(ctx, r, dstBucket, dstObject, getObjectInfo, retPerms)
retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, dstBucket, dstObject, getObjectInfo, retPerms, holdPerms)
if s3Err == ErrNone && retentionMode != "" {
srcInfo.UserDefined[xhttp.AmzObjectLockMode] = string(retentionMode)
srcInfo.UserDefined[xhttp.AmzObjectLockRetainUntilDate] = retentionDate.UTC().Format(time.RFC3339)
}
if s3Err == ErrNone && legalHold.Status != "" {
srcInfo.UserDefined[xhttp.AmzObjectLockLegalHold] = string(legalHold.Status)
}
if s3Err != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
return
@ -1251,11 +1264,16 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
putObject = api.CacheAPI().PutObject
}
retPerms := isPutActionAllowed(rAuthType, bucket, object, r, iampolicy.PutObjectRetentionAction)
retentionMode, retentionDate, s3Err := checkPutObjectRetentionAllowed(ctx, r, bucket, object, getObjectInfo, retPerms)
holdPerms := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, iampolicy.PutObjectLegalHoldAction)
retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms)
if s3Err == ErrNone && retentionMode != "" {
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(time.RFC3339)
}
if s3Err == ErrNone && legalHold.Status != "" {
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
}
if s3Err != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
return
@ -1409,11 +1427,16 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
return
}
retPerms := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, iampolicy.PutObjectRetentionAction)
retentionMode, retentionDate, s3Err := checkPutObjectRetentionAllowed(ctx, r, bucket, object, objectAPI.GetObjectInfo, retPerms)
holdPerms := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, iampolicy.PutObjectLegalHoldAction)
retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, objectAPI.GetObjectInfo, retPerms, holdPerms)
if s3Err == ErrNone && retentionMode != "" {
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(time.RFC3339)
}
if s3Err == ErrNone && legalHold.Status != "" {
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
}
if s3Err != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
return
@ -2215,14 +2238,16 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
}
// Reject retention or governance headers if set, CompleteMultipartUpload spec
// does not use these headers, and should not be passed down to checkPutObjectRetentionAllowed
if isObjectLockRequested(r.Header) || isObjectLockGovernanceBypassSet(r.Header) {
// does not use these headers, and should not be passed down to checkPutObjectLockAllowed
if objectlock.IsObjectLockRequested(r.Header) || objectlock.IsObjectLockGovernanceBypassSet(r.Header) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL, guessIsBrowserReq(r))
return
}
// Enforce object lock governance in case a competing upload finalized first.
retPerms := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, iampolicy.PutObjectRetentionAction)
if _, _, s3Err := checkPutObjectRetentionAllowed(ctx, r, bucket, object, objectAPI.GetObjectInfo, retPerms); s3Err != ErrNone {
holdPerms := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, iampolicy.PutObjectLegalHoldAction)
if _, _, _, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, objectAPI.GetObjectInfo, retPerms, holdPerms); s3Err != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
return
}
@ -2467,23 +2492,81 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
return
}
// Check permissions to perform this legal hold operation
if s3Err := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, policy.PutObjectLegalHoldAction); s3Err != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
return
}
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
// Get Content-Md5 sent by client and verify if valid
md5Bytes, err := checkValidMD5(r.Header)
if err != nil {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDigest), r.URL, guessIsBrowserReq(r))
return
}
if _, ok := globalBucketObjectLockConfig.Get(bucket); !ok {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketObjectLockConfiguration), r.URL, guessIsBrowserReq(r))
return
}
legalHold, err := objectlock.ParseObjectLegalHold(r.Body)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
// verify Content-MD5 sum of request body if this header set
if len(md5Bytes) > 0 {
data, err := xml.Marshal(legalHold)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
if hex.EncodeToString(md5Bytes) != getMD5Hash(data) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDigest), r.URL, guessIsBrowserReq(r))
return
}
}
getObjectInfo := objectAPI.GetObjectInfo
if api.CacheAPI() != nil {
getObjectInfo = api.CacheAPI().GetObjectInfo
}
opts, err := getOpts(ctx, r, bucket, object)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
if _, err = getObjectInfo(ctx, bucket, object, opts); err != nil {
objInfo, err := getObjectInfo(ctx, bucket, object, opts)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
objInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = strings.ToUpper(string(legalHold.Status))
objInfo.metadataOnly = true
if _, err = objectAPI.CopyObject(ctx, bucket, object, bucket, object, objInfo, ObjectOptions{}, ObjectOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
writeSuccessNoContent(w)
// Notify object event.
sendEvent(eventArgs{
EventName: event.ObjectCreatedPutLegalHold,
BucketName: bucket,
Object: objInfo,
ReqParams: extractReqParams(r),
RespElements: extractRespElements(w),
UserAgent: r.UserAgent(),
Host: handlers.GetSourceIP(r),
})
}
// GetObjectLegalHoldHandler - get legal hold configuration to object,
@ -2506,6 +2589,10 @@ func (api objectAPIHandlers) GetObjectLegalHoldHandler(w http.ResponseWriter, r
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
return
}
if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectLegalHoldAction, bucket, object); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
return
}
getObjectInfo := objectAPI.GetObjectInfo
if api.CacheAPI() != nil {
@ -2518,12 +2605,25 @@ func (api objectAPIHandlers) GetObjectLegalHoldHandler(w http.ResponseWriter, r
return
}
if _, err = getObjectInfo(ctx, bucket, object, opts); err != nil {
objInfo, err := getObjectInfo(ctx, bucket, object, opts)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
legalHold := objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined)
writeSuccessResponseXML(w, encodeResponse(legalHold))
// Notify object legal hold accessed via a GET request.
sendEvent(eventArgs{
EventName: event.ObjectAccessedGetLegalHold,
BucketName: bucket,
Object: objInfo,
ReqParams: extractReqParams(r),
RespElements: extractRespElements(w),
UserAgent: r.UserAgent(),
Host: handlers.GetSourceIP(r),
})
}
// PutObjectRetentionHandler - set object hold configuration to object,
@ -2568,7 +2668,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
return
}
objRetention, err := parseObjectRetention(r.Body)
objRetention, err := objectlock.ParseObjectRetention(r.Body)
if err != nil {
apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
apiErr.Description = err.Error()
@ -2660,7 +2760,7 @@ func (api objectAPIHandlers) GetObjectRetentionHandler(w http.ResponseWriter, r
return
}
retention := getObjectRetentionMeta(objInfo.UserDefined)
retention := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
writeSuccessResponseXML(w, encodeResponse(retention))
// Notify object retention accessed via a GET request.

521
cmd/object-lock.go
View file

@ -18,387 +18,12 @@ package cmd
import (
"context"
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
"strings"
"sync"
"time"
xhttp "github.com/minio/minio/cmd/http"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/env"
"github.com/minio/minio/pkg/objectlock"
)
// RetentionMode - object retention mode.
type RetentionMode string
const (
// Governance - governance mode.
Governance RetentionMode = "GOVERNANCE"
// Compliance - compliance mode.
Compliance RetentionMode = "COMPLIANCE"
// Invalid - invalid retention mode.
Invalid RetentionMode = ""
)
func parseRetentionMode(modeStr string) (mode RetentionMode) {
switch strings.ToUpper(modeStr) {
case "GOVERNANCE":
mode = Governance
case "COMPLIANCE":
mode = Compliance
default:
mode = Invalid
}
return mode
}
var (
errMalformedBucketObjectConfig = errors.New("Invalid bucket object lock config")
errInvalidRetentionDate = errors.New("Date must be provided in ISO 8601 format")
errPastObjectLockRetainDate = errors.New("the retain until date must be in the future")
errUnknownWORMModeDirective = errors.New("unknown WORM mode directive")
errObjectLockMissingContentMD5 = errors.New("Content-MD5 HTTP header is required for Put Object requests with Object Lock parameters")
errObjectLockInvalidHeaders = errors.New("x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied")
)
const (
ntpServerEnv = "MINIO_NTP_SERVER"
)
var (
ntpServer = env.Get(ntpServerEnv, "")
)
// Retention - bucket level retention configuration.
type Retention struct {
Mode RetentionMode
Validity time.Duration
}
// IsEmpty - returns whether retention is empty or not.
func (r Retention) IsEmpty() bool {
return r.Mode == "" || r.Validity == 0
}
// Retain - check whether given date is retainable by validity time.
func (r Retention) Retain(created time.Time) bool {
t, err := UTCNowNTP()
if err != nil {
logger.LogIf(context.Background(), err)
// Retain
return true
}
return globalWORMEnabled || created.Add(r.Validity).After(t)
}
// BucketObjectLockConfig - map of bucket and retention configuration.
type BucketObjectLockConfig struct {
sync.RWMutex
retentionMap map[string]Retention
}
// Set - set retention configuration.
func (config *BucketObjectLockConfig) Set(bucketName string, retention Retention) {
config.Lock()
config.retentionMap[bucketName] = retention
config.Unlock()
}
// Get - Get retention configuration.
func (config *BucketObjectLockConfig) Get(bucketName string) (r Retention, ok bool) {
config.RLock()
defer config.RUnlock()
r, ok = config.retentionMap[bucketName]
return r, ok
}
// Remove - removes retention configuration.
func (config *BucketObjectLockConfig) Remove(bucketName string) {
config.Lock()
delete(config.retentionMap, bucketName)
config.Unlock()
}
func newBucketObjectLockConfig() *BucketObjectLockConfig {
return &BucketObjectLockConfig{
retentionMap: map[string]Retention{},
}
}
// DefaultRetention - default retention configuration.
type DefaultRetention struct {
XMLName xml.Name `xml:"DefaultRetention"`
Mode RetentionMode `xml:"Mode"`
Days *uint64 `xml:"Days"`
Years *uint64 `xml:"Years"`
}
// Maximum support retention days and years supported by AWS S3.
const (
// This tested by using `mc lock` command
maximumRetentionDays = 36500
maximumRetentionYears = 100
)
// UnmarshalXML - decodes XML data.
func (dr *DefaultRetention) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
// Make subtype to avoid recursive UnmarshalXML().
type defaultRetention DefaultRetention
retention := defaultRetention{}
if err := d.DecodeElement(&retention, &start); err != nil {
return err
}
switch string(retention.Mode) {
case "GOVERNANCE", "COMPLIANCE":
default:
return fmt.Errorf("unknown retention mode %v", retention.Mode)
}
if retention.Days == nil && retention.Years == nil {
return fmt.Errorf("either Days or Years must be specified")
}
if retention.Days != nil && retention.Years != nil {
return fmt.Errorf("either Days or Years must be specified, not both")
}
if retention.Days != nil {
if *retention.Days == 0 {
return fmt.Errorf("Default retention period must be a positive integer value for 'Days'")
}
if *retention.Days > maximumRetentionDays {
return fmt.Errorf("Default retention period too large for 'Days' %d", *retention.Days)
}
} else if *retention.Years == 0 {
return fmt.Errorf("Default retention period must be a positive integer value for 'Years'")
} else if *retention.Years > maximumRetentionYears {
return fmt.Errorf("Default retention period too large for 'Years' %d", *retention.Years)
}
*dr = DefaultRetention(retention)
return nil
}
// ObjectLockConfig - object lock configuration specified in
// https://docs.aws.amazon.com/AmazonS3/latest/API/Type_API_ObjectLockConfiguration.html
type ObjectLockConfig struct {
XMLNS string `xml:"xmlns,attr,omitempty"`
XMLName xml.Name `xml:"ObjectLockConfiguration"`
ObjectLockEnabled string `xml:"ObjectLockEnabled"`
Rule *struct {
DefaultRetention DefaultRetention `xml:"DefaultRetention"`
} `xml:"Rule,omitempty"`
}
// UnmarshalXML - decodes XML data.
func (config *ObjectLockConfig) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
// Make subtype to avoid recursive UnmarshalXML().
type objectLockConfig ObjectLockConfig
parsedConfig := objectLockConfig{}
if err := d.DecodeElement(&parsedConfig, &start); err != nil {
return err
}
if parsedConfig.ObjectLockEnabled != "Enabled" {
return fmt.Errorf("only 'Enabled' value is allowd to ObjectLockEnabled element")
}
*config = ObjectLockConfig(parsedConfig)
return nil
}
// ToRetention - convert to Retention type.
func (config *ObjectLockConfig) ToRetention() (r Retention) {
if config.Rule != nil {
r.Mode = config.Rule.DefaultRetention.Mode
t, err := UTCNowNTP()
if err != nil {
logger.LogIf(context.Background(), err)
// Do not change any configuration
// upon NTP failure.
return r
}
if config.Rule.DefaultRetention.Days != nil {
r.Validity = t.AddDate(0, 0, int(*config.Rule.DefaultRetention.Days)).Sub(t)
} else {
r.Validity = t.AddDate(int(*config.Rule.DefaultRetention.Years), 0, 0).Sub(t)
}
}
return r
}
func parseObjectLockConfig(reader io.Reader) (*ObjectLockConfig, error) {
config := ObjectLockConfig{}
if err := xml.NewDecoder(reader).Decode(&config); err != nil {
return nil, err
}
return &config, nil
}
func newObjectLockConfig() *ObjectLockConfig {
return &ObjectLockConfig{
ObjectLockEnabled: "Enabled",
}
}
// RetentionDate is a embedded type containing time.Time to unmarshal
// Date in Retention
type RetentionDate struct {
time.Time
}
// UnmarshalXML parses date from Expiration and validates date format
func (rDate *RetentionDate) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
var dateStr string
err := d.DecodeElement(&dateStr, &startElement)
if err != nil {
return err
}
// While AWS documentation mentions that the date specified
// must be present in ISO 8601 format, in reality they allow
// users to provide RFC 3339 compliant dates.
retDate, err := time.Parse(time.RFC3339, dateStr)
if err != nil {
return errInvalidRetentionDate
}
*rDate = RetentionDate{retDate}
return nil
}
// MarshalXML encodes expiration date if it is non-zero and encodes
// empty string otherwise
func (rDate *RetentionDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
if *rDate == (RetentionDate{time.Time{}}) {
return nil
}
return e.EncodeElement(rDate.Format(time.RFC3339), startElement)
}
// ObjectRetention specified in
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectRetention.html
type ObjectRetention struct {
XMLNS string `xml:"xmlns,attr,omitempty"`
XMLName xml.Name `xml:"Retention"`
Mode RetentionMode `xml:"Mode,omitempty"`
RetainUntilDate RetentionDate `xml:"RetainUntilDate,omitempty"`
}
func parseObjectRetention(reader io.Reader) (*ObjectRetention, error) {
ret := ObjectRetention{}
if err := xml.NewDecoder(reader).Decode(&ret); err != nil {
return nil, err
}
if ret.Mode != Compliance && ret.Mode != Governance {
return &ret, errUnknownWORMModeDirective
}
t, err := UTCNowNTP()
if err != nil {
logger.LogIf(context.Background(), err)
return &ret, errPastObjectLockRetainDate
}
if ret.RetainUntilDate.Before(t) {
return &ret, errPastObjectLockRetainDate
}
return &ret, nil
}
func isObjectLockRetentionRequested(h http.Header) bool {
if _, ok := h[xhttp.AmzObjectLockMode]; ok {
return true
}
if _, ok := h[xhttp.AmzObjectLockRetainUntilDate]; ok {
return true
}
return false
}
func isObjectLockLegalHoldRequested(h http.Header) bool {
_, ok := h[xhttp.AmzObjectLockLegalHold]
return ok
}
func isObjectLockGovernanceBypassSet(h http.Header) bool {
v, ok := h[xhttp.AmzObjectLockBypassGovernance]
if !ok {
return false
}
val := strings.Join(v, "")
return strings.ToLower(val) == "true"
}
func isObjectLockRequested(h http.Header) bool {
return isObjectLockLegalHoldRequested(h) || isObjectLockRetentionRequested(h)
}
func parseObjectLockRetentionHeaders(h http.Header) (rmode RetentionMode, r RetentionDate, err error) {
retMode, ok := h[xhttp.AmzObjectLockMode]
if ok {
rmode = parseRetentionMode(strings.Join(retMode, ""))
if rmode == Invalid {
return rmode, r, errUnknownWORMModeDirective
}
}
var retDate time.Time
dateStr, ok := h[xhttp.AmzObjectLockRetainUntilDate]
if ok {
// While AWS documentation mentions that the date specified
// must be present in ISO 8601 format, in reality they allow
// users to provide RFC 3339 compliant dates.
retDate, err = time.Parse(time.RFC3339, strings.Join(dateStr, ""))
if err != nil {
return rmode, r, errInvalidRetentionDate
}
t, err := UTCNowNTP()
if err != nil {
logger.LogIf(context.Background(), err)
return rmode, r, errPastObjectLockRetainDate
}
if retDate.Before(t) {
return rmode, r, errPastObjectLockRetainDate
}
}
if len(retMode) == 0 || len(dateStr) == 0 {
return rmode, r, errObjectLockInvalidHeaders
}
return rmode, RetentionDate{retDate}, nil
}
func getObjectRetentionMeta(meta map[string]string) ObjectRetention {
var mode RetentionMode
var retainTill RetentionDate
if modeStr, ok := meta[strings.ToLower(xhttp.AmzObjectLockMode)]; ok {
mode = parseRetentionMode(modeStr)
}
if tillStr, ok := meta[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)]; ok {
if t, e := time.Parse(time.RFC3339, tillStr); e == nil {
retainTill = RetentionDate{t.UTC()}
}
}
return ObjectRetention{Mode: mode, RetainUntilDate: retainTill}
}
// enforceRetentionBypassForDelete enforces whether an existing object under governance can be deleted
// with governance bypass headers set in the request.
// Objects under site wide WORM can never be overwritten.
@ -424,16 +49,19 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
}
return oi, toAPIErrorCode(ctx, err)
}
ret := getObjectRetentionMeta(oi.UserDefined)
ret := objectlock.GetObjectRetentionMeta(oi.UserDefined)
lhold := objectlock.GetObjectLegalHoldMeta(oi.UserDefined)
if lhold.Status == objectlock.ON {
return oi, ErrObjectLocked
}
// Here bucket does not support object lock
if ret.Mode == Invalid {
if ret.Mode == objectlock.Invalid {
return oi, ErrNone
}
if ret.Mode != Compliance && ret.Mode != Governance {
if ret.Mode != objectlock.Compliance && ret.Mode != objectlock.Governance {
return oi, ErrUnknownWORMModeDirective
}
t, err := UTCNowNTP()
t, err := objectlock.UTCNowNTP()
if err != nil {
logger.LogIf(ctx, err)
return oi, ErrObjectLocked
@ -441,7 +69,7 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
if ret.RetainUntilDate.Before(t) {
return oi, ErrNone
}
if isObjectLockGovernanceBypassSet(r.Header) && ret.Mode == Governance && govBypassPerm == ErrNone {
if objectlock.IsObjectLockGovernanceBypassSet(r.Header) && ret.Mode == objectlock.Governance && govBypassPerm == ErrNone {
return oi, ErrNone
}
return oi, ErrObjectLocked
@ -453,7 +81,7 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
// For objects in "Governance" mode, overwrite is allowed if a) object retention date is past OR
// governance bypass headers are set and user has governance bypass permissions.
// Objects in compliance mode can be overwritten only if retention date is being extended. No mode change is permitted.
func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, govBypassPerm APIErrorCode, objRetention *ObjectRetention) (oi ObjectInfo, s3Err APIErrorCode) {
func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, govBypassPerm APIErrorCode, objRetention *objectlock.ObjectRetention) (oi ObjectInfo, s3Err APIErrorCode) {
if globalWORMEnabled {
return oi, ErrObjectLocked
}
@ -474,24 +102,24 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket,
return oi, toAPIErrorCode(ctx, err)
}
ret := getObjectRetentionMeta(oi.UserDefined)
ret := objectlock.GetObjectRetentionMeta(oi.UserDefined)
// no retention metadata on object
if ret.Mode == Invalid {
if ret.Mode == objectlock.Invalid {
if _, isWORMBucket := globalBucketObjectLockConfig.Get(bucket); !isWORMBucket {
return oi, ErrInvalidBucketObjectLockConfiguration
}
return oi, ErrNone
}
t, err := UTCNowNTP()
t, err := objectlock.UTCNowNTP()
if err != nil {
logger.LogIf(ctx, err)
return oi, ErrObjectLocked
}
if ret.Mode == Compliance {
if ret.Mode == objectlock.Compliance {
// Compliance retention mode cannot be changed and retention period cannot be shortened as per
// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
if objRetention.Mode != Compliance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) {
if objRetention.Mode != objectlock.Compliance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) {
return oi, ErrObjectLocked
}
if objRetention.RetainUntilDate.Before(t) {
@ -500,8 +128,8 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket,
return oi, ErrNone
}
if ret.Mode == Governance {
if !isObjectLockGovernanceBypassSet(r.Header) {
if ret.Mode == objectlock.Governance {
if !objectlock.IsObjectLockGovernanceBypassSet(r.Header) {
if objRetention.RetainUntilDate.Before(t) {
return oi, ErrInvalidRetentionDate
}
@ -515,111 +143,104 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket,
return oi, ErrNone
}
// checkPutObjectRetentionAllowed enforces object retention policy for requests with WORM headers
// checkPutObjectLockAllowed enforces object retention policy and legal hold policy
// for requests with WORM headers
// See https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-managing.html for the spec.
// For non-existing objects with object retention headers set, this method returns ErrNone if bucket has
// locking enabled and user has requisite permissions (s3:PutObjectRetention)
// If object exists on object store and site wide WORM enabled - this method
// returns an error. For objects in "Governance" mode, overwrite is allowed if the retention date has expired.
// For objects in "Compliance" mode, retention date cannot be shortened, and mode cannot be altered.
func checkPutObjectRetentionAllowed(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, retentionPermErr APIErrorCode) (RetentionMode, RetentionDate, APIErrorCode) {
var mode RetentionMode
var retainDate RetentionDate
// For objects with legal hold header set, the s3:PutObjectLegalHold permission is expected to be set
// Both legal hold and retention can be applied independently on an object
func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, retentionPermErr, legalHoldPermErr APIErrorCode) (objectlock.Mode, objectlock.RetentionDate, objectlock.ObjectLegalHold, APIErrorCode) {
var mode objectlock.Mode
var retainDate objectlock.RetentionDate
var legalHold objectlock.ObjectLegalHold
retention, isWORMBucket := globalBucketObjectLockConfig.Get(bucket)
retentionRequested := isObjectLockRequested(r.Header)
retentionRequested := objectlock.IsObjectLockRetentionRequested(r.Header)
legalHoldRequested := objectlock.IsObjectLockLegalHoldRequested(r.Header)
var objExists bool
opts, err := getOpts(ctx, r, bucket, object)
if err != nil {
return mode, retainDate, toAPIErrorCode(ctx, err)
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
}
if objInfo, err := getObjectInfoFn(ctx, bucket, object, opts); err == nil {
objExists = true
r := getObjectRetentionMeta(objInfo.UserDefined)
if globalWORMEnabled || r.Mode == Compliance {
return mode, retainDate, ErrObjectLocked
r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
if globalWORMEnabled || r.Mode == objectlock.Compliance {
return mode, retainDate, legalHold, ErrObjectLocked
}
mode = r.Mode
retainDate = r.RetainUntilDate
legalHold = objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined)
// Disallow overwriting an object on legal hold
if legalHold.Status == "ON" {
return mode, retainDate, legalHold, ErrObjectLocked
}
}
if legalHoldRequested {
if !isWORMBucket {
return mode, retainDate, legalHold, ErrInvalidBucketObjectLockConfiguration
}
var lerr error
if legalHold, lerr = objectlock.ParseObjectLockLegalHoldHeaders(r.Header); lerr != nil {
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
}
}
if retentionRequested {
if !isWORMBucket {
return mode, retainDate, ErrInvalidBucketObjectLockConfiguration
return mode, retainDate, legalHold, ErrInvalidBucketObjectLockConfiguration
}
rMode, rDate, err := parseObjectLockRetentionHeaders(r.Header)
legalHold, err := objectlock.ParseObjectLockLegalHoldHeaders(r.Header)
if err != nil {
return mode, retainDate, toAPIErrorCode(ctx, err)
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
}
rMode, rDate, err := objectlock.ParseObjectLockRetentionHeaders(r.Header)
if err != nil {
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
}
// AWS S3 just creates a new version of object when an object is being overwritten.
t, err := UTCNowNTP()
t, err := objectlock.UTCNowNTP()
if err != nil {
logger.LogIf(ctx, err)
return mode, retainDate, ErrObjectLocked
return mode, retainDate, legalHold, ErrObjectLocked
}
if objExists && retainDate.After(t) {
return mode, retainDate, ErrObjectLocked
return mode, retainDate, legalHold, ErrObjectLocked
}
if rMode == Invalid {
return mode, retainDate, toAPIErrorCode(ctx, errObjectLockInvalidHeaders)
if rMode == objectlock.Invalid {
return mode, retainDate, legalHold, toAPIErrorCode(ctx, objectlock.ErrObjectLockInvalidHeaders)
}
if retentionPermErr != ErrNone {
return mode, retainDate, retentionPermErr
return mode, retainDate, legalHold, retentionPermErr
}
return rMode, rDate, ErrNone
return rMode, rDate, legalHold, ErrNone
}
if !retentionRequested && isWORMBucket {
if retention.IsEmpty() && (mode == Compliance || mode == Governance) {
return mode, retainDate, ErrObjectLocked
if retention.IsEmpty() && (mode == objectlock.Compliance || mode == objectlock.Governance) {
return mode, retainDate, legalHold, ErrObjectLocked
}
if retentionPermErr != ErrNone {
return mode, retainDate, retentionPermErr
return mode, retainDate, legalHold, retentionPermErr
}
t, err := UTCNowNTP()
t, err := objectlock.UTCNowNTP()
if err != nil {
logger.LogIf(ctx, err)
return mode, retainDate, ErrObjectLocked
return mode, retainDate, legalHold, ErrObjectLocked
}
// AWS S3 just creates a new version of object when an object is being overwritten.
if objExists && retainDate.After(t) {
return mode, retainDate, ErrObjectLocked
return mode, retainDate, legalHold, ErrObjectLocked
}
if !legalHoldRequested {
// inherit retention from bucket configuration
return retention.Mode, objectlock.RetentionDate{Time: t.Add(retention.Validity)}, legalHold, ErrNone
}
// inherit retention from bucket configuration
return retention.Mode, RetentionDate{t.Add(retention.Validity)}, ErrNone
}
return mode, retainDate, ErrNone
}
// filter object lock metadata if s3:GetObjectRetention permission is denied or if isCopy flag set.
func filterObjectLockMetadata(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string, isCopy bool, getRetPerms APIErrorCode) map[string]string {
// Copy on write
dst := metadata
var copied bool
delKey := func(key string) {
if _, ok := metadata[key]; !ok {
return
}
if !copied {
dst = make(map[string]string, len(metadata))
for k, v := range metadata {
dst[k] = v
}
copied = true
}
delete(dst, key)
}
ret := getObjectRetentionMeta(metadata)
if ret.Mode == Invalid || isCopy {
delKey(xhttp.AmzObjectLockMode)
delKey(xhttp.AmzObjectLockRetainUntilDate)
return metadata
}
if getRetPerms == ErrNone {
return dst
}
delKey(xhttp.AmzObjectLockMode)
delKey(xhttp.AmzObjectLockRetainUntilDate)
return dst
return mode, retainDate, legalHold, ErrNone
}

3
cmd/peer-rest-client.go
View file

@ -35,6 +35,7 @@ import (
"github.com/minio/minio/pkg/lifecycle"
"github.com/minio/minio/pkg/madmin"
xnet "github.com/minio/minio/pkg/net"
"github.com/minio/minio/pkg/objectlock"
"github.com/minio/minio/pkg/policy"
trace "github.com/minio/minio/pkg/trace"
)
@ -430,7 +431,7 @@ func (client *peerRESTClient) PutBucketNotification(bucket string, rulesMap even
}
// PutBucketObjectLockConfig - PUT bucket object lock configuration.
func (client *peerRESTClient) PutBucketObjectLockConfig(bucket string, retention Retention) error {
func (client *peerRESTClient) PutBucketObjectLockConfig(bucket string, retention objectlock.Retention) error {
values := make(url.Values)
values.Set(peerRESTBucket, bucket)

3
cmd/peer-rest-server.go
View file

@ -33,6 +33,7 @@ import (
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/event"
"github.com/minio/minio/pkg/lifecycle"
objectlock "github.com/minio/minio/pkg/objectlock"
"github.com/minio/minio/pkg/policy"
trace "github.com/minio/minio/pkg/trace"
)
@ -845,7 +846,7 @@ func (s *peerRESTServer) PutBucketObjectLockConfigHandler(w http.ResponseWriter,
return
}
var retention Retention
var retention objectlock.Retention
if r.ContentLength < 0 {
s.writeErrorResponse(w, errInvalidArgument)
return

12
cmd/utils.go
View file

@ -39,7 +39,6 @@ import (
"sync"
"time"
"github.com/beevik/ntp"
xhttp "github.com/minio/minio/cmd/http"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/handlers"
@ -354,17 +353,6 @@ func UTCNow() time.Time {
return time.Now().UTC()
}
// UTCNowNTP - is similar in functionality to UTCNow()
// but only used when we do not wish to rely on system
// time.
func UTCNowNTP() (time.Time, error) {
// ntp server is disabled
if ntpServer == "" {
return UTCNow(), nil
}
return ntp.Time(ntpServer)
}
// GenETag - generate UUID based ETag
func GenETag() string {
return ToS3ETag(getMD5Hash([]byte(mustGetUUID())))

74
cmd/web-handlers.go
View file

@ -50,6 +50,7 @@ import (
"github.com/minio/minio/pkg/hash"
iampolicy "github.com/minio/minio/pkg/iam/policy"
"github.com/minio/minio/pkg/ioutil"
"github.com/minio/minio/pkg/objectlock"
"github.com/minio/minio/pkg/policy"
)
@ -928,6 +929,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
object := vars["object"]
retPerms := ErrAccessDenied
holdPerms := ErrAccessDenied
claims, owner, authErr := webRequestAuthenticate(r)
if authErr != nil {
@ -974,7 +976,17 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
}) {
retPerms = ErrNone
}
if globalIAMSys.IsAllowed(iampolicy.Args{
AccountName: claims.AccessKey(),
Action: iampolicy.PutObjectLegalHoldAction,
BucketName: bucket,
ConditionValues: getConditionValues(r, "", claims.AccessKey(), claims.Map()),
IsOwner: owner,
ObjectName: object,
Claims: claims.Map(),
}) {
holdPerms = ErrNone
}
}
// Check if bucket is a reserved bucket name or invalid.
@ -1068,11 +1080,14 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
getObjectInfo = web.CacheAPI().GetObjectInfo
}
// enforce object retention rules
retentionMode, retentionDate, s3Err := checkPutObjectRetentionAllowed(ctx, r, bucket, object, getObjectInfo, retPerms)
retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms)
if s3Err == ErrNone && retentionMode != "" {
opts.UserDefined[xhttp.AmzObjectLockMode] = string(retentionMode)
opts.UserDefined[xhttp.AmzObjectLockRetainUntilDate] = retentionDate.UTC().Format(time.RFC3339)
}
if s3Err == ErrNone && legalHold.Status != "" {
opts.UserDefined[xhttp.AmzObjectLockLegalHold] = string(legalHold.Status)
}
if s3Err != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
return
@ -1130,6 +1145,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
token := r.URL.Query().Get("token")
getRetPerms := ErrAccessDenied
legalHoldPerms := ErrAccessDenied
claims, owner, authErr := webTokenAuthenticate(token)
if authErr != nil {
@ -1154,6 +1170,15 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
}) {
getRetPerms = ErrNone
}
if globalPolicySys.IsAllowed(policy.Args{
Action: policy.GetObjectLegalHoldAction,
BucketName: bucket,
ConditionValues: getConditionValues(r, "", "", nil),
IsOwner: false,
ObjectName: object,
}) {
legalHoldPerms = ErrNone
}
} else {
writeWebErrorResponse(w, authErr)
return
@ -1185,6 +1210,17 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
}) {
getRetPerms = ErrNone
}
if globalIAMSys.IsAllowed(iampolicy.Args{
AccountName: claims.AccessKey(),
Action: iampolicy.GetObjectLegalHoldAction,
BucketName: bucket,
ConditionValues: getConditionValues(r, "", claims.AccessKey(), claims.Map()),
IsOwner: owner,
ObjectName: object,
Claims: claims.Map(),
}) {
legalHoldPerms = ErrNone
}
}
// Check if bucket is a reserved bucket name or invalid.
@ -1209,7 +1245,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
objInfo := gr.ObjInfo
// filter object lock metadata if permission does not permit
objInfo.UserDefined = filterObjectLockMetadata(ctx, r, bucket, object, objInfo.UserDefined, false, getRetPerms)
objInfo.UserDefined = objectlock.FilterObjectLockMetadata(objInfo.UserDefined, getRetPerms != ErrNone, legalHoldPerms != ErrNone)
if objectAPI.IsEncryptionSupported() {
if _, err = DecryptObjectInfo(&objInfo, r.Header); err != nil {
@ -1304,6 +1340,8 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
token := r.URL.Query().Get("token")
claims, owner, authErr := webTokenAuthenticate(token)
var getRetPerms []APIErrorCode
var legalHoldPerms []APIErrorCode
if authErr != nil {
if authErr == errNoAuthToken {
for _, object := range args.Objects {
@ -1329,6 +1367,18 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
retentionPerm = ErrNone
}
getRetPerms = append(getRetPerms, retentionPerm)
legalHoldPerm := ErrAccessDenied
if globalPolicySys.IsAllowed(policy.Args{
Action: policy.GetObjectLegalHoldAction,
BucketName: args.BucketName,
ConditionValues: getConditionValues(r, "", "", nil),
IsOwner: false,
ObjectName: pathJoin(args.Prefix, object),
}) {
legalHoldPerm = ErrNone
}
legalHoldPerms = append(legalHoldPerms, legalHoldPerm)
}
} else {
writeWebErrorResponse(w, authErr)
@ -1354,7 +1404,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
retentionPerm := ErrAccessDenied
if globalIAMSys.IsAllowed(iampolicy.Args{
AccountName: claims.AccessKey(),
Action: iampolicy.GetObjectAction,
Action: iampolicy.GetObjectRetentionAction,
BucketName: args.BucketName,
ConditionValues: getConditionValues(r, "", claims.AccessKey(), claims.Map()),
IsOwner: owner,
@ -1364,6 +1414,20 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
retentionPerm = ErrNone
}
getRetPerms = append(getRetPerms, retentionPerm)
legalHoldPerm := ErrAccessDenied
if globalIAMSys.IsAllowed(iampolicy.Args{
AccountName: claims.AccessKey(),
Action: iampolicy.GetObjectLegalHoldAction,
BucketName: args.BucketName,
ConditionValues: getConditionValues(r, "", claims.AccessKey(), claims.Map()),
IsOwner: owner,
ObjectName: pathJoin(args.Prefix, object),
Claims: claims.Map(),
}) {
legalHoldPerm = ErrNone
}
legalHoldPerms = append(legalHoldPerms, legalHoldPerm)
}
}
@ -1394,7 +1458,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
info := gr.ObjInfo
// filter object lock metadata if permission does not permit
info.UserDefined = filterObjectLockMetadata(ctx, r, args.BucketName, objectName, info.UserDefined, false, getRetPerms[i])
info.UserDefined = objectlock.FilterObjectLockMetadata(info.UserDefined, getRetPerms[i] != ErrNone, legalHoldPerms[i] != ErrNone)
if info.IsCompressed() {
// For reporting, set the file size to the uncompressed size.

15
docs/retention/README.md
View file

@ -5,6 +5,9 @@ MinIO server allows selectively specify WORM for specific objects or configuring
Object locking requires locking to be enabled on a bucket at the time of bucket creation. In addition, a default retention period and retention mode can be configured on a bucket to be
applied to objects created in that bucket.
Independently of retention, an object can also be under legal hold. This effectively disallows
all deletes and overwrites of an object under legal hold until the hold is lifted.
## Get Started
### 1. Prerequisites
@ -29,10 +32,22 @@ aws s3api put-object --bucket testbucket --key lockme --object-lock-mode GOVERNA
See https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html for AWS S3 spec on
object locking and permissions required for object retention and governance bypass overrides.
### Set legal hold on an object
PutObject API allows setting legal hold using `x-amz-object-lock-legal-hold` header.
```sh
aws s3api put-object --bucket testbucket --key legalhold --object-lock-legal-hold-status ON --body /etc/issue
```
See https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html for AWS S3 spec on
object locking and permissions required for specifying legal hold.
### 3. Note
- When global WORM is enabled by `MINIO_WORM` environment variable or `worm` field in configuration file supersedes bucket level WORM and `PUT object lock configuration` REST API is disabled.
- In global WORM mode objects can never be overwritten
- If an object is under legal hold, it cannot be overwritten unless the legal hold is explicitly removed.
- In `Compliance` mode, objects cannot be overwritten or deleted by anyone until retention period
is expired. If user has requisite governance bypass permissions, an object's retention date can
be extended in `Compliance` mode.

114
mint/run/core/awscli/test.sh
View file

@ -69,6 +69,24 @@ function make_bucket() {
return $rv
}
function make_bucket_with_lock() {
# Make bucket
bucket_name="awscli-mint-test-bucket-$RANDOM"
function="${AWS} s3api create-bucket --bucket ${bucket_name} --object-lock-enabled-for-bucket"
# execute the test
out=$($function 2>&1)
rv=$?
# if command is successful print bucket_name or print error
if [ $rv -eq 0 ]; then
echo "${bucket_name}"
else
echo "${out}"
fi
return $rv
}
function delete_bucket() {
# Delete bucket
function="${AWS} s3 rb s3://${1} --force"
@ -1430,6 +1448,98 @@ function test_worm_bucket() {
return $rv
}
# Tests creating and deleting an object with legal hold.
function test_legal_hold() {
# log start time
start_time=$(get_time)
function="make_bucket_with_lock"
bucket_name=$(make_bucket_with_lock)
rv=$?
# if make bucket succeeds upload a file
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --object-lock-legal-hold-status ON"
out=$($function 2>&1)
errcnt=$(echo "$out" | sed -n '/Bucket is missing ObjectLockConfiguration/p' | wc -l)
# skip test for gateways
if [ "$errcnt" -eq 1 ]; then
return 0
fi
rv=$?
else
# if make bucket fails, $bucket_name has the error output
out="${bucket_name}"
fi
# if upload succeeds download the file
if [ $rv -eq 0 ]; then
function="${AWS} s3api head-object --bucket ${bucket_name} --key datafile-1-kB"
# save the ref to function being tested, so it can be logged
test_function=${function}
out=$($function 2>&1)
lhold=$(echo "$out" | jq -r .ObjectLockLegalHoldStatus)
rv=$?
fi
# if head-object succeeds, verify metadata has legal hold status
if [ $rv -eq 0 ]; then
if [ "${lhold}" == "" ]; then
rv=1
out="Legal hold was not applied"
fi
if [ "${lhold}" == "OFF" ]; then
rv=1
out="Legal hold was not applied"
fi
fi
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object-legal-hold --bucket ${bucket_name} --key datafile-1-kB --legal-hold Status=OFF"
out=$($function 2>&1)
rv=$?
else
# if make bucket fails, $bucket_name has the error output
out="${bucket_name}"
fi
# if upload succeeds download the file
if [ $rv -eq 0 ]; then
function="${AWS} s3api get-object-legal-hold --bucket ${bucket_name} --key datafile-1-kB"
# save the ref to function being tested, so it can be logged
test_function=${function}
out=$($function 2>&1)
lhold=$(echo "$out" | jq -r .LegalHold.Status)
rv=$?
fi
# if head-object succeeds, verify metadata has legal hold status
if [ $rv -eq 0 ]; then
if [ "${lhold}" == "" ]; then
rv=1
out="Legal hold was not applied"
fi
if [ "${lhold}" == "ON" ]; then
rv=1
out="Legal hold status not turned off"
fi
fi
# Attempt a delete on prefix shouldn't delete the directory since we have an object inside it.
if [ $rv -eq 0 ]; then
function="${AWS} s3api delete-object --bucket ${bucket_name} --key datafile-1-kB"
# save the ref to function being tested, so it can be logged
test_function=${function}
out=$($function 2>&1)
rv=$?
fi
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# main handler for all the tests.
main() {
# Success tests
@ -1455,7 +1565,9 @@ main() {
test_list_objects_error && \
test_put_object_error && \
test_serverside_encryption_error && \
test_worm_bucket
test_worm_bucket && \
test_legal_hold
return $?
}

14
pkg/event/name.go
View file

@ -30,6 +30,7 @@ const (
ObjectAccessedAll Name = 1 + iota
ObjectAccessedGet
ObjectAccessedGetRetention
ObjectAccessedGetLegalHold
ObjectAccessedHead
ObjectCreatedAll
ObjectCreatedCompleteMultipartUpload
@ -37,6 +38,7 @@ const (
ObjectCreatedPost
ObjectCreatedPut
ObjectCreatedPutRetention
ObjectCreatedPutLegalHold
ObjectRemovedAll
ObjectRemovedDelete
)
@ -45,9 +47,9 @@ const (
func (name Name) Expand() []Name {
switch name {
case ObjectAccessedAll:
return []Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention}
return []Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention, ObjectAccessedGetLegalHold}
case ObjectCreatedAll:
return []Name{ObjectCreatedCompleteMultipartUpload, ObjectCreatedCopy, ObjectCreatedPost, ObjectCreatedPut, ObjectCreatedPutRetention}
return []Name{ObjectCreatedCompleteMultipartUpload, ObjectCreatedCopy, ObjectCreatedPost, ObjectCreatedPut, ObjectCreatedPutRetention, ObjectCreatedPutLegalHold}
case ObjectRemovedAll:
return []Name{ObjectRemovedDelete}
default:
@ -64,6 +66,8 @@ func (name Name) String() string {
return "s3:ObjectAccessed:Get"
case ObjectAccessedGetRetention:
return "s3:ObjectAccessed:GetRetention"
case ObjectAccessedGetLegalHold:
return "s3:ObjectAccessed:GetLegalHold"
case ObjectAccessedHead:
return "s3:ObjectAccessed:Head"
case ObjectCreatedAll:
@ -78,6 +82,8 @@ func (name Name) String() string {
return "s3:ObjectCreated:Put"
case ObjectCreatedPutRetention:
return "s3:ObjectCreated:PutRetention"
case ObjectCreatedPutLegalHold:
return "s3:ObjectCreated:PutLegalHold"
case ObjectRemovedAll:
return "s3:ObjectRemoved:*"
case ObjectRemovedDelete:
@ -138,6 +144,8 @@ func ParseName(s string) (Name, error) {
return ObjectAccessedGet, nil
case "s3:ObjectAccessed:GetRetention":
return ObjectAccessedGetRetention, nil
case "s3:ObjectAccessed:GetLegalHold":
return ObjectAccessedGetLegalHold, nil
case "s3:ObjectAccessed:Head":
return ObjectAccessedHead, nil
case "s3:ObjectCreated:*":
@ -152,6 +160,8 @@ func ParseName(s string) (Name, error) {
return ObjectCreatedPut, nil
case "s3:ObjectCreated:PutRetention":
return ObjectCreatedPutRetention, nil
case "s3:ObjectCreated:PutLegalHold":
return ObjectCreatedPutLegalHold, nil
case "s3:ObjectRemoved:*":
return ObjectRemovedAll, nil
case "s3:ObjectRemoved:Delete":

9
pkg/event/name_test.go
View file

@ -28,8 +28,8 @@ func TestNameExpand(t *testing.T) {
name Name
expectedResult []Name
}{
{ObjectAccessedAll, []Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention}},
{ObjectCreatedAll, []Name{ObjectCreatedCompleteMultipartUpload, ObjectCreatedCopy, ObjectCreatedPost, ObjectCreatedPut, ObjectCreatedPutRetention}},
{ObjectAccessedAll, []Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention, ObjectAccessedGetLegalHold}},
{ObjectCreatedAll, []Name{ObjectCreatedCompleteMultipartUpload, ObjectCreatedCopy, ObjectCreatedPost, ObjectCreatedPut, ObjectCreatedPutRetention, ObjectCreatedPutLegalHold}},
{ObjectRemovedAll, []Name{ObjectRemovedDelete}},
{ObjectAccessedHead, []Name{ObjectAccessedHead}},
}
@ -60,6 +60,11 @@ func TestNameString(t *testing.T) {
{ObjectCreatedPut, "s3:ObjectCreated:Put"},
{ObjectRemovedAll, "s3:ObjectRemoved:*"},
{ObjectRemovedDelete, "s3:ObjectRemoved:Delete"},
{ObjectCreatedPutRetention, "s3:ObjectCreated:PutRetention"},
{ObjectCreatedPutLegalHold, "s3:ObjectCreated:PutLegalHold"},
{ObjectAccessedGetRetention, "s3:ObjectAccessed:GetRetention"},
{ObjectAccessedGetLegalHold, "s3:ObjectAccessed:GetLegalHold"},
{blankName, ""},
}

4
pkg/event/rulesmap_test.go
View file

@ -153,12 +153,12 @@ func TestRulesMapMatch(t *testing.T) {
func TestNewRulesMap(t *testing.T) {
rulesMapCase1 := make(RulesMap)
rulesMapCase1.add([]Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention},
rulesMapCase1.add([]Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention, ObjectAccessedGetLegalHold},
"*", TargetID{"1", "webhook"})
rulesMapCase2 := make(RulesMap)
rulesMapCase2.add([]Name{ObjectAccessedGet, ObjectAccessedHead,
ObjectCreatedPut, ObjectAccessedGetRetention}, "*", TargetID{"1", "webhook"})
ObjectCreatedPut, ObjectAccessedGetRetention, ObjectAccessedGetLegalHold}, "*", TargetID{"1", "webhook"})
rulesMapCase3 := make(RulesMap)
rulesMapCase3.add([]Name{ObjectRemovedDelete}, "2010*.jpg", TargetID{"1", "webhook"})

12
pkg/iam/policy/action.go
View file

@ -102,6 +102,12 @@ const (
// GetObjectRetentionAction - GetObjectRetention, GetObject, HeadObject Rest API action.
GetObjectRetentionAction = "s3:GetObjectRetention"
// GetObjectLegalHoldAction - GetObjectLegalHold, GetObject Rest API action.
GetObjectLegalHoldAction = "s3:GetObjectLegalHold"
// PutObjectLegalHoldAction - PutObjectLegalHold, PutObject Rest API action.
PutObjectLegalHoldAction = "s3:PutObjectLegalHold"
// GetBucketObjectLockConfigurationAction - GetBucketObjectLockConfiguration Rest API action
GetBucketObjectLockConfigurationAction = "s3:GetBucketObjectLockConfiguration"
@ -137,6 +143,8 @@ var supportedActions = map[Action]struct{}{
PutBucketLifecycleAction: {},
PutObjectRetentionAction: {},
GetObjectRetentionAction: {},
GetObjectLegalHoldAction: {},
PutObjectLegalHoldAction: {},
PutBucketObjectLockConfigurationAction: {},
GetBucketObjectLockConfigurationAction: {},
BypassGovernanceModeAction: {},
@ -154,6 +162,8 @@ func (action Action) isObjectAction() bool {
return true
case PutObjectRetentionAction, GetObjectRetentionAction:
return true
case PutObjectLegalHoldAction, GetObjectLegalHoldAction:
return true
}
return false
@ -267,6 +277,8 @@ var actionConditionKeyMap = map[Action]condition.KeySet{
}, condition.CommonKeys...)...),
PutObjectRetentionAction: condition.NewKeySet(condition.CommonKeys...),
GetObjectRetentionAction: condition.NewKeySet(condition.CommonKeys...),
PutObjectLegalHoldAction: condition.NewKeySet(condition.CommonKeys...),
GetObjectLegalHoldAction: condition.NewKeySet(condition.CommonKeys...),
BypassGovernanceModeAction: condition.NewKeySet(condition.CommonKeys...),
BypassGovernanceRetentionAction: condition.NewKeySet(condition.CommonKeys...),
GetBucketObjectLockConfigurationAction: condition.NewKeySet(condition.CommonKeys...),

521
pkg/objectlock/objectlock.go Normal file
View file

@ -0,0 +1,521 @@
/*
* MinIO Cloud Storage, (C) 2020 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package objectlock
import (
"context"
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
"strings"
"sync"
"time"
"github.com/beevik/ntp"
xhttp "github.com/minio/minio/cmd/http"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/env"
)
// Mode - object retention mode.
type Mode string
const (
// Governance - governance mode.
Governance Mode = "GOVERNANCE"
// Compliance - compliance mode.
Compliance Mode = "COMPLIANCE"
// Invalid - invalid retention mode.
Invalid Mode = ""
)
func parseMode(modeStr string) (mode Mode) {
switch strings.ToUpper(modeStr) {
case "GOVERNANCE":
mode = Governance
case "COMPLIANCE":
mode = Compliance
default:
mode = Invalid
}
return mode
}
// LegalHoldStatus - object legal hold status.
type LegalHoldStatus string
const (
// ON -legal hold is on.
ON LegalHoldStatus = "ON"
// OFF -legal hold is off.
OFF LegalHoldStatus = "OFF"
)
func parseLegalHoldStatus(holdStr string) LegalHoldStatus {
switch strings.ToUpper(holdStr) {
case "ON":
return ON
case "OFF":
return OFF
}
return LegalHoldStatus("")
}
var (
// ErrMalformedBucketObjectConfig -indicates that the bucket object lock config is malformed
ErrMalformedBucketObjectConfig = errors.New("invalid bucket object lock config")
// ErrInvalidRetentionDate - indicates that retention date needs to be in ISO 8601 format
ErrInvalidRetentionDate = errors.New("date must be provided in ISO 8601 format")
// ErrPastObjectLockRetainDate - indicates that retention date must be in the future
ErrPastObjectLockRetainDate = errors.New("the retain until date must be in the future")
// ErrUnknownWORMModeDirective - indicates that the retention mode is invalid
ErrUnknownWORMModeDirective = errors.New("unknown WORM mode directive")
// ErrObjectLockMissingContentMD5 - indicates missing Content-MD5 header for put object requests with locking
ErrObjectLockMissingContentMD5 = errors.New("content-MD5 HTTP header is required for Put Object requests with Object Lock parameters")
// ErrObjectLockInvalidHeaders indicates that object lock headers are missing
ErrObjectLockInvalidHeaders = errors.New("x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied")
// ErrMalformedXML - generic error indicating malformed XML
ErrMalformedXML = errors.New("the XML you provided was not well-formed or did not validate against our published schema")
)
const (
ntpServerEnv = "MINIO_NTP_SERVER"
)
var (
ntpServer = env.Get(ntpServerEnv, "")
)
// UTCNowNTP - is similar in functionality to UTCNow()
// but only used when we do not wish to rely on system
// time.
func UTCNowNTP() (time.Time, error) {
// ntp server is disabled
if ntpServer == "" {
return time.Now().UTC(), nil
}
return ntp.Time(ntpServer)
}
// Retention - bucket level retention configuration.
type Retention struct {
Mode Mode
Validity time.Duration
}
// IsEmpty - returns whether retention is empty or not.
func (r Retention) IsEmpty() bool {
return r.Mode == "" || r.Validity == 0
}
// Retain - check whether given date is retainable by validity time.
func (r Retention) Retain(created time.Time) bool {
t, err := UTCNowNTP()
if err != nil {
logger.LogIf(context.Background(), err)
// Retain
return true
}
return created.Add(r.Validity).After(t)
}
// BucketObjectLockConfig - map of bucket and retention configuration.
type BucketObjectLockConfig struct {
sync.RWMutex
retentionMap map[string]Retention
}
// Set - set retention configuration.
func (config *BucketObjectLockConfig) Set(bucketName string, retention Retention) {
config.Lock()
config.retentionMap[bucketName] = retention
config.Unlock()
}
// Get - Get retention configuration.
func (config *BucketObjectLockConfig) Get(bucketName string) (r Retention, ok bool) {
config.RLock()
defer config.RUnlock()
r, ok = config.retentionMap[bucketName]
return r, ok
}
// Remove - removes retention configuration.
func (config *BucketObjectLockConfig) Remove(bucketName string) {
config.Lock()
delete(config.retentionMap, bucketName)
config.Unlock()
}
// NewBucketObjectLockConfig returns initialized BucketObjectLockConfig
func NewBucketObjectLockConfig() *BucketObjectLockConfig {
return &BucketObjectLockConfig{
retentionMap: map[string]Retention{},
}
}
// DefaultRetention - default retention configuration.
type DefaultRetention struct {
XMLName xml.Name `xml:"DefaultRetention"`
Mode Mode `xml:"Mode"`
Days *uint64 `xml:"Days"`
Years *uint64 `xml:"Years"`
}
// Maximum support retention days and years supported by AWS S3.
const (
// This tested by using `mc lock` command
maximumRetentionDays = 36500
maximumRetentionYears = 100
)
// UnmarshalXML - decodes XML data.
func (dr *DefaultRetention) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
// Make subtype to avoid recursive UnmarshalXML().
type defaultRetention DefaultRetention
retention := defaultRetention{}
if err := d.DecodeElement(&retention, &start); err != nil {
return err
}
switch string(retention.Mode) {
case "GOVERNANCE", "COMPLIANCE":
default:
return fmt.Errorf("unknown retention mode %v", retention.Mode)
}
if retention.Days == nil && retention.Years == nil {
return fmt.Errorf("either Days or Years must be specified")
}
if retention.Days != nil && retention.Years != nil {
return fmt.Errorf("either Days or Years must be specified, not both")
}
if retention.Days != nil {
if *retention.Days == 0 {
return fmt.Errorf("Default retention period must be a positive integer value for 'Days'")
}
if *retention.Days > maximumRetentionDays {
return fmt.Errorf("Default retention period too large for 'Days' %d", *retention.Days)
}
} else if *retention.Years == 0 {
return fmt.Errorf("Default retention period must be a positive integer value for 'Years'")
} else if *retention.Years > maximumRetentionYears {
return fmt.Errorf("Default retention period too large for 'Years' %d", *retention.Years)
}
*dr = DefaultRetention(retention)
return nil
}
// Config - object lock configuration specified in
// https://docs.aws.amazon.com/AmazonS3/latest/API/Type_API_ObjectLockConfiguration.html
type Config struct {
XMLNS string `xml:"xmlns,attr,omitempty"`
XMLName xml.Name `xml:"ObjectLockConfiguration"`
ObjectLockEnabled string `xml:"ObjectLockEnabled"`
Rule *struct {
DefaultRetention DefaultRetention `xml:"DefaultRetention"`
} `xml:"Rule,omitempty"`
}
// UnmarshalXML - decodes XML data.
func (config *Config) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
// Make subtype to avoid recursive UnmarshalXML().
type objectLockConfig Config
parsedConfig := objectLockConfig{}
if err := d.DecodeElement(&parsedConfig, &start); err != nil {
return err
}
if parsedConfig.ObjectLockEnabled != "Enabled" {
return fmt.Errorf("only 'Enabled' value is allowd to ObjectLockEnabled element")
}
*config = Config(parsedConfig)
return nil
}
// ToRetention - convert to Retention type.
func (config *Config) ToRetention() (r Retention) {
if config.Rule != nil {
r.Mode = config.Rule.DefaultRetention.Mode
t, err := UTCNowNTP()
if err != nil {
logger.LogIf(context.Background(), err)
// Do not change any configuration
// upon NTP failure.
return r
}
if config.Rule.DefaultRetention.Days != nil {
r.Validity = t.AddDate(0, 0, int(*config.Rule.DefaultRetention.Days)).Sub(t)
} else {
r.Validity = t.AddDate(int(*config.Rule.DefaultRetention.Years), 0, 0).Sub(t)
}
}
return r
}
// ParseObjectLockConfig parses ObjectLockConfig from xml
func ParseObjectLockConfig(reader io.Reader) (*Config, error) {
config := Config{}
if err := xml.NewDecoder(reader).Decode(&config); err != nil {
return nil, err
}
return &config, nil
}
// NewObjectLockConfig returns a initialized objectlock.Config struct
func NewObjectLockConfig() *Config {
return &Config{
ObjectLockEnabled: "Enabled",
}
}
// RetentionDate is a embedded type containing time.Time to unmarshal
// Date in Retention
type RetentionDate struct {
time.Time
}
// UnmarshalXML parses date from Retention and validates date format
func (rDate *RetentionDate) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
var dateStr string
err := d.DecodeElement(&dateStr, &startElement)
if err != nil {
return err
}
// While AWS documentation mentions that the date specified
// must be present in ISO 8601 format, in reality they allow
// users to provide RFC 3339 compliant dates.
retDate, err := time.Parse(time.RFC3339, dateStr)
if err != nil {
return ErrInvalidRetentionDate
}
*rDate = RetentionDate{retDate}
return nil
}
// MarshalXML encodes expiration date if it is non-zero and encodes
// empty string otherwise
func (rDate *RetentionDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
if *rDate == (RetentionDate{time.Time{}}) {
return nil
}
return e.EncodeElement(rDate.Format(time.RFC3339), startElement)
}
// ObjectRetention specified in
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectRetention.html
type ObjectRetention struct {
XMLNS string `xml:"xmlns,attr,omitempty"`
XMLName xml.Name `xml:"Retention"`
Mode Mode `xml:"Mode,omitempty"`
RetainUntilDate RetentionDate `xml:"RetainUntilDate,omitempty"`
}
// ParseObjectRetention constructs ObjectRetention struct from xml input
func ParseObjectRetention(reader io.Reader) (*ObjectRetention, error) {
ret := ObjectRetention{}
if err := xml.NewDecoder(reader).Decode(&ret); err != nil {
return nil, err
}
if ret.Mode != Compliance && ret.Mode != Governance {
return &ret, ErrUnknownWORMModeDirective
}
t, err := UTCNowNTP()
if err != nil {
logger.LogIf(context.Background(), err)
return &ret, ErrPastObjectLockRetainDate
}
if ret.RetainUntilDate.Before(t) {
return &ret, ErrPastObjectLockRetainDate
}
return &ret, nil
}
// IsObjectLockRetentionRequested returns true if object lock retention headers are set.
func IsObjectLockRetentionRequested(h http.Header) bool {
if _, ok := h[xhttp.AmzObjectLockMode]; ok {
return true
}
if _, ok := h[xhttp.AmzObjectLockRetainUntilDate]; ok {
return true
}
return false
}
// IsObjectLockLegalHoldRequested returns true if object lock legal hold header is set.
func IsObjectLockLegalHoldRequested(h http.Header) bool {
_, ok := h[xhttp.AmzObjectLockLegalHold]
return ok
}
// IsObjectLockGovernanceBypassSet returns true if object lock governance bypass header is set.
func IsObjectLockGovernanceBypassSet(h http.Header) bool {
return strings.ToLower(h.Get(xhttp.AmzObjectLockBypassGovernance)) == "true"
}
// IsObjectLockRequested returns true if legal hold or object lock retention headers are requested.
func IsObjectLockRequested(h http.Header) bool {
return IsObjectLockLegalHoldRequested(h) || IsObjectLockRetentionRequested(h)
}
// ParseObjectLockRetentionHeaders parses http headers to extract retention mode and retention date
func ParseObjectLockRetentionHeaders(h http.Header) (rmode Mode, r RetentionDate, err error) {
retMode := h.Get(xhttp.AmzObjectLockMode)
dateStr := h.Get(xhttp.AmzObjectLockRetainUntilDate)
if len(retMode) == 0 || len(dateStr) == 0 {
return rmode, r, ErrObjectLockInvalidHeaders
}
rmode = parseMode(retMode)
if rmode == Invalid {
return rmode, r, ErrUnknownWORMModeDirective
}
var retDate time.Time
// While AWS documentation mentions that the date specified
// must be present in ISO 8601 format, in reality they allow
// users to provide RFC 3339 compliant dates.
retDate, err = time.Parse(time.RFC3339, dateStr)
if err != nil {
return rmode, r, ErrInvalidRetentionDate
}
t, err := UTCNowNTP()
if err != nil {
logger.LogIf(context.Background(), err)
return rmode, r, ErrPastObjectLockRetainDate
}
if retDate.Before(t) {
return rmode, r, ErrPastObjectLockRetainDate
}
return rmode, RetentionDate{retDate}, nil
}
// GetObjectRetentionMeta constructs ObjectRetention from metadata
func GetObjectRetentionMeta(meta map[string]string) ObjectRetention {
var mode Mode
var retainTill RetentionDate
if modeStr, ok := meta[strings.ToLower(xhttp.AmzObjectLockMode)]; ok {
mode = parseMode(modeStr)
}
if tillStr, ok := meta[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)]; ok {
if t, e := time.Parse(time.RFC3339, tillStr); e == nil {
retainTill = RetentionDate{t.UTC()}
}
}
return ObjectRetention{Mode: mode, RetainUntilDate: retainTill}
}
// GetObjectLegalHoldMeta constructs ObjectLegalHold from metadata
func GetObjectLegalHoldMeta(meta map[string]string) ObjectLegalHold {
holdStr, ok := meta[strings.ToLower(xhttp.AmzObjectLockLegalHold)]
if ok {
return ObjectLegalHold{Status: parseLegalHoldStatus(holdStr)}
}
return ObjectLegalHold{}
}
// ParseObjectLockLegalHoldHeaders parses request headers to construct ObjectLegalHold
func ParseObjectLockLegalHoldHeaders(h http.Header) (lhold ObjectLegalHold, err error) {
holdStatus, ok := h[xhttp.AmzObjectLockLegalHold]
if ok {
lh := parseLegalHoldStatus(strings.Join(holdStatus, ""))
if lh != ON && lh != OFF {
return lhold, ErrUnknownWORMModeDirective
}
lhold = ObjectLegalHold{Status: lh}
}
return lhold, nil
}
// ObjectLegalHold specified in
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectLegalHold.html
type ObjectLegalHold struct {
XMLNS string `xml:"xmlns,attr,omitempty"`
XMLName xml.Name `xml:"LegalHold"`
Status LegalHoldStatus `xml:"Status,omitempty"`
}
// ParseObjectLegalHold decodes the XML into ObjectLegalHold
func ParseObjectLegalHold(reader io.Reader) (hold *ObjectLegalHold, err error) {
if err = xml.NewDecoder(reader).Decode(&hold); err != nil {
return
}
if hold.Status != ON && hold.Status != OFF {
return nil, ErrMalformedXML
}
return
}
// FilterObjectLockMetadata filters object lock metadata if s3:GetObjectRetention permission is denied or if isCopy flag set.
func FilterObjectLockMetadata(metadata map[string]string, filterRetention, filterLegalHold bool) map[string]string {
// Copy on write
dst := metadata
var copied bool
delKey := func(key string) {
if _, ok := metadata[key]; !ok {
return
}
if !copied {
dst = make(map[string]string, len(metadata))
for k, v := range metadata {
dst[k] = v
}
copied = true
}
delete(dst, key)
}
legalHold := GetObjectLegalHoldMeta(metadata)
if legalHold.Status == "" || filterLegalHold {
delKey(xhttp.AmzObjectLockLegalHold)
}
ret := GetObjectRetentionMeta(metadata)
if ret.Mode == Invalid || filterRetention {
delKey(xhttp.AmzObjectLockMode)
delKey(xhttp.AmzObjectLockRetainUntilDate)
return dst
}
return dst
}

567
pkg/objectlock/objectlock_test.go Normal file
View file

@ -0,0 +1,567 @@
/*
* MinIO Cloud Storage, (C) 2020 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package objectlock
import (
"encoding/xml"
"fmt"
"net/http"
"reflect"
"strings"
"testing"
"time"
xhttp "github.com/minio/minio/cmd/http"
)
func TestParseMode(t *testing.T) {
testCases := []struct {
value string
expectedMode Mode
}{
{
value: "governance",
expectedMode: Governance,
},
{
value: "complIAnce",
expectedMode: Compliance,
},
{
value: "gce",
expectedMode: Invalid,
},
}
for _, tc := range testCases {
if parseMode(tc.value) != tc.expectedMode {
t.Errorf("Expected Mode %s, got %s", tc.expectedMode, parseMode(tc.value))
}
}
}
func TestParseLegalHoldStatus(t *testing.T) {
tests := []struct {
value string
expectedStatus LegalHoldStatus
}{
{
value: "ON",
expectedStatus: ON,
},
{
value: "Off",
expectedStatus: OFF,
},
{
value: "x",
expectedStatus: "",
},
}
for _, tt := range tests {
actualStatus := parseLegalHoldStatus(tt.value)
if actualStatus != tt.expectedStatus {
t.Errorf("Expected legal hold status %s, got %s", tt.expectedStatus, actualStatus)
}
}
}
// TestUnmarshalDefaultRetention checks if default retention
// marshaling and unmarshaling work as expected
func TestUnmarshalDefaultRetention(t *testing.T) {
days := uint64(4)
years := uint64(1)
zerodays := uint64(0)
invalidDays := uint64(maximumRetentionDays + 1)
tests := []struct {
value DefaultRetention
expectedErr error
expectErr bool
}{
{
value: DefaultRetention{Mode: "retain"},
expectedErr: fmt.Errorf("unknown retention mode retain"),
expectErr: true,
},
{
value: DefaultRetention{Mode: "GOVERNANCE"},
expectedErr: fmt.Errorf("either Days or Years must be specified"),
expectErr: true,
},
{
value: DefaultRetention{Mode: "GOVERNANCE", Days: &days},
expectedErr: nil,
expectErr: false,
},
{
value: DefaultRetention{Mode: "GOVERNANCE", Years: &years},
expectedErr: nil,
expectErr: false,
},
{
value: DefaultRetention{Mode: "GOVERNANCE", Days: &days, Years: &years},
expectedErr: fmt.Errorf("either Days or Years must be specified, not both"),
expectErr: true,
},
{
value: DefaultRetention{Mode: "GOVERNANCE", Days: &zerodays},
expectedErr: fmt.Errorf("Default retention period must be a positive integer value for 'Days'"),
expectErr: true,
},
{
value: DefaultRetention{Mode: "GOVERNANCE", Days: &invalidDays},
expectedErr: fmt.Errorf("Default retention period too large for 'Days' %d", invalidDays),
expectErr: true,
},
}
for _, tt := range tests {
d, err := xml.MarshalIndent(&tt.value, "", "\t")
if err != nil {
t.Fatal(err)
}
var dr DefaultRetention
err = xml.Unmarshal(d, &dr)
if tt.expectedErr == nil {
if err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
} else if err == nil {
t.Fatalf("error: expected = %v, got = <nil>", tt.expectedErr)
} else if tt.expectedErr.Error() != err.Error() {
t.Fatalf("error: expected = %v, got = %v", tt.expectedErr, err)
}
}
}
func TestParseObjectLockConfig(t *testing.T) {
tests := []struct {
value string
expectedErr error
expectErr bool
}{
{
value: "<ObjectLockConfiguration><ObjectLockEnabled>yes</ObjectLockEnabled></ObjectLockConfiguration>",
expectedErr: fmt.Errorf("only 'Enabled' value is allowd to ObjectLockEnabled element"),
expectErr: true,
},
{
value: "<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>COMPLIANCE</Mode><Days>0</Days></DefaultRetention></Rule></ObjectLockConfiguration>",
expectedErr: fmt.Errorf("Default retention period must be a positive integer value for 'Days'"),
expectErr: true,
},
{
value: "<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>COMPLIANCE</Mode><Days>30</Days></DefaultRetention></Rule></ObjectLockConfiguration>",
expectedErr: nil,
expectErr: false,
},
}
for _, tt := range tests {
_, err := ParseObjectLockConfig(strings.NewReader(tt.value))
if tt.expectedErr == nil {
if err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
} else if err == nil {
t.Fatalf("error: expected = %v, got = <nil>", tt.expectedErr)
} else if tt.expectedErr.Error() != err.Error() {
t.Fatalf("error: expected = %v, got = %v", tt.expectedErr, err)
}
}
}
func TestParseObjectRetention(t *testing.T) {
tests := []struct {
value string
expectedErr error
expectErr bool
}{
{
value: "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Retention xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Mode>string</Mode><RetainUntilDate>2020-01-02T15:04:05Z</RetainUntilDate></Retention>",
expectedErr: ErrUnknownWORMModeDirective,
expectErr: true,
},
{
value: "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Retention xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Mode>COMPLIANCE</Mode><RetainUntilDate>2017-01-02T15:04:05Z</RetainUntilDate></Retention>",
expectedErr: ErrPastObjectLockRetainDate,
expectErr: true,
},
{
value: "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Retention xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Mode>GOVERNANCE</Mode><RetainUntilDate>2057-01-02T15:04:05Z</RetainUntilDate></Retention>",
expectedErr: nil,
expectErr: false,
},
}
for _, tt := range tests {
_, err := ParseObjectRetention(strings.NewReader(tt.value))
if tt.expectedErr == nil {
if err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
} else if err == nil {
t.Fatalf("error: expected = %v, got = <nil>", tt.expectedErr)
} else if tt.expectedErr.Error() != err.Error() {
t.Fatalf("error: expected = %v, got = %v", tt.expectedErr, err)
}
}
}
func TestIsObjectLockRequested(t *testing.T) {
tests := []struct {
header http.Header
expectedVal bool
}{
{
header: http.Header{
"Authorization": []string{"AWS4-HMAC-SHA256 <cred_string>"},
"X-Amz-Content-Sha256": []string{""},
"Content-Encoding": []string{""},
},
expectedVal: false,
},
{
header: http.Header{
xhttp.AmzObjectLockLegalHold: []string{""},
},
expectedVal: true,
},
{
header: http.Header{
xhttp.AmzObjectLockRetainUntilDate: []string{""},
xhttp.AmzObjectLockMode: []string{""},
},
expectedVal: true,
},
{
header: http.Header{
xhttp.AmzObjectLockBypassGovernance: []string{""},
},
expectedVal: false,
},
}
for _, tt := range tests {
actualVal := IsObjectLockRequested(tt.header)
if actualVal != tt.expectedVal {
t.Fatalf("error: expected %v, actual %v", tt.expectedVal, actualVal)
}
}
}
func TestIsObjectLockGovernanceBypassSet(t *testing.T) {
tests := []struct {
header http.Header
expectedVal bool
}{
{
header: http.Header{
"Authorization": []string{"AWS4-HMAC-SHA256 <cred_string>"},
"X-Amz-Content-Sha256": []string{""},
"Content-Encoding": []string{""},
},
expectedVal: false,
},
{
header: http.Header{
xhttp.AmzObjectLockLegalHold: []string{""},
},
expectedVal: false,
},
{
header: http.Header{
xhttp.AmzObjectLockRetainUntilDate: []string{""},
xhttp.AmzObjectLockMode: []string{""},
},
expectedVal: false,
},
{
header: http.Header{
xhttp.AmzObjectLockBypassGovernance: []string{""},
},
expectedVal: false,
},
{
header: http.Header{
xhttp.AmzObjectLockBypassGovernance: []string{"true"},
},
expectedVal: true,
},
}
for _, tt := range tests {
actualVal := IsObjectLockGovernanceBypassSet(tt.header)
if actualVal != tt.expectedVal {
t.Fatalf("error: expected %v, actual %v", tt.expectedVal, actualVal)
}
}
}
func TestParseObjectLockRetentionHeaders(t *testing.T) {
tests := []struct {
header http.Header
expectedErr error
}{
{
header: http.Header{
"Authorization": []string{"AWS4-HMAC-SHA256 <cred_string>"},
"X-Amz-Content-Sha256": []string{""},
"Content-Encoding": []string{""},
},
expectedErr: ErrObjectLockInvalidHeaders,
},
{
header: http.Header{
xhttp.AmzObjectLockMode: []string{"lock"},
xhttp.AmzObjectLockRetainUntilDate: []string{"2017-01-02"},
},
expectedErr: ErrUnknownWORMModeDirective,
},
{
header: http.Header{
xhttp.AmzObjectLockMode: []string{"governance"},
},
expectedErr: ErrObjectLockInvalidHeaders,
},
{
header: http.Header{
xhttp.AmzObjectLockRetainUntilDate: []string{"2017-01-02"},
xhttp.AmzObjectLockMode: []string{"governance"},
},
expectedErr: ErrInvalidRetentionDate,
},
{
header: http.Header{
xhttp.AmzObjectLockRetainUntilDate: []string{"2017-01-02T15:04:05Z"},
xhttp.AmzObjectLockMode: []string{"governance"},
},
expectedErr: ErrPastObjectLockRetainDate,
},
{
header: http.Header{
xhttp.AmzObjectLockMode: []string{"governance"},
xhttp.AmzObjectLockRetainUntilDate: []string{"2017-01-02T15:04:05Z"},
},
expectedErr: ErrPastObjectLockRetainDate,
},
{
header: http.Header{
xhttp.AmzObjectLockMode: []string{"governance"},
xhttp.AmzObjectLockRetainUntilDate: []string{"2087-01-02T15:04:05Z"},
},
expectedErr: nil,
},
}
for i, tt := range tests {
_, _, err := ParseObjectLockRetentionHeaders(tt.header)
if tt.expectedErr == nil {
if err != nil {
t.Fatalf("Case %d error: expected = <nil>, got = %v", i, err)
}
} else if err == nil {
t.Fatalf("Case %d error: expected = %v, got = <nil>", i, tt.expectedErr)
} else if tt.expectedErr.Error() != err.Error() {
t.Fatalf("Case %d error: expected = %v, got = %v", i, tt.expectedErr, err)
}
}
}
func TestGetObjectRetentionMeta(t *testing.T) {
tests := []struct {
metadata map[string]string
expected ObjectRetention
}{
{
metadata: map[string]string{
"Authorization": "AWS4-HMAC-SHA256 <cred_string>",
"X-Amz-Content-Sha256": "",
"Content-Encoding": "",
},
expected: ObjectRetention{},
},
{
metadata: map[string]string{
"x-amz-object-lock-mode": "governance",
},
expected: ObjectRetention{Mode: Governance},
},
{
metadata: map[string]string{
"x-amz-object-lock-retain-until-date": "2020-02-01",
},
expected: ObjectRetention{RetainUntilDate: RetentionDate{time.Date(2020, 2, 1, 12, 0, 0, 0, time.UTC)}},
},
}
for i, tt := range tests {
o := GetObjectRetentionMeta(tt.metadata)
if o.Mode != tt.expected.Mode {
t.Fatalf("Case %d expected %v, got %v", i, tt.expected.Mode, o.Mode)
}
}
}
func TestGetObjectLegalHoldMeta(t *testing.T) {
tests := []struct {
metadata map[string]string
expected ObjectLegalHold
}{
{
metadata: map[string]string{
"x-amz-object-lock-mode": "governance",
},
expected: ObjectLegalHold{},
},
{
metadata: map[string]string{
"x-amz-object-lock-legal-hold": "on",
},
expected: ObjectLegalHold{Status: ON},
},
{
metadata: map[string]string{
"x-amz-object-lock-legal-hold": "off",
},
expected: ObjectLegalHold{Status: OFF},
},
{
metadata: map[string]string{
"x-amz-object-lock-legal-hold": "X",
},
expected: ObjectLegalHold{Status: ""},
},
}
for i, tt := range tests {
o := GetObjectLegalHoldMeta(tt.metadata)
if o.Status != tt.expected.Status {
t.Fatalf("Case %d expected %v, got %v", i, tt.expected.Status, o.Status)
}
}
}
func TestParseObjectLegalHold(t *testing.T) {
tests := []struct {
value string
expectedErr error
expectErr bool
}{
{
value: "<?xml version=\"1.0\" encoding=\"UTF-8\"?><LegalHold xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Status>string</Status></LegalHold>",
expectedErr: ErrMalformedXML,
expectErr: true,
},
{
value: "<?xml version=\"1.0\" encoding=\"UTF-8\"?><LegalHold xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Status>ON</Status></LegalHold>",
expectedErr: nil,
expectErr: false,
},
{
value: "<?xml version=\"1.0\" encoding=\"UTF-8\"?><LegalHold xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Status>On</Status></LegalHold>",
expectedErr: ErrMalformedXML,
expectErr: true,
},
}
for i, tt := range tests {
_, err := ParseObjectLegalHold(strings.NewReader(tt.value))
if tt.expectedErr == nil {
if err != nil {
t.Fatalf("Case %d error: expected = <nil>, got = %v", i, err)
}
} else if err == nil {
t.Fatalf("Case %d error: expected = %v, got = <nil>", i, tt.expectedErr)
} else if tt.expectedErr.Error() != err.Error() {
t.Fatalf("Case %d error: expected = %v, got = %v", i, tt.expectedErr, err)
}
}
}
func TestFilterObjectLockMetadata(t *testing.T) {
tests := []struct {
metadata map[string]string
filterRetention bool
filterLegalHold bool
expected map[string]string
}{
{
metadata: map[string]string{
"Authorization": "AWS4-HMAC-SHA256 <cred_string>",
"X-Amz-Content-Sha256": "",
"Content-Encoding": "",
},
expected: map[string]string{
"Authorization": "AWS4-HMAC-SHA256 <cred_string>",
"X-Amz-Content-Sha256": "",
"Content-Encoding": "",
},
},
{
metadata: map[string]string{
"x-amz-object-lock-mode": "governance",
},
expected: map[string]string{
"x-amz-object-lock-mode": "governance",
},
filterRetention: false,
},
{
metadata: map[string]string{
"x-amz-object-lock-mode": "governance",
"x-amz-object-lock-retain-until-date": "2020-02-01",
},
expected: map[string]string{},
filterRetention: true,
},
{
metadata: map[string]string{
"x-amz-object-lock-legal-hold": "off",
},
expected: map[string]string{},
filterLegalHold: true,
},
{
metadata: map[string]string{
"x-amz-object-lock-legal-hold": "on",
},
expected: map[string]string{"x-amz-object-lock-legal-hold": "on"},
filterLegalHold: false,
},
{
metadata: map[string]string{
"x-amz-object-lock-legal-hold": "on",
"x-amz-object-lock-mode": "governance",
"x-amz-object-lock-retain-until-date": "2020-02-01",
},
expected: map[string]string{},
filterRetention: true,
filterLegalHold: true,
},
{
metadata: map[string]string{
"x-amz-object-lock-legal-hold": "on",
"x-amz-object-lock-mode": "governance",
"x-amz-object-lock-retain-until-date": "2020-02-01",
},
expected: map[string]string{"x-amz-object-lock-legal-hold": "on",
"x-amz-object-lock-mode": "governance",
"x-amz-object-lock-retain-until-date": "2020-02-01"},
},
}
for i, tt := range tests {
o := FilterObjectLockMetadata(tt.metadata, tt.filterRetention, tt.filterLegalHold)
if !reflect.DeepEqual(o, tt.metadata) {
t.Fatalf("Case %d expected %v, got %v", i, tt.metadata, o)
}
}
}

10
pkg/policy/action.go
View file

@ -98,6 +98,10 @@ const (
// GetObjectRetentionAction - GetObjectRetention, GetObject, HeadObject Rest API action.
GetObjectRetentionAction = "s3:GetObjectRetention"
// GetObjectLegalHoldAction - GetObjectLegalHold, GetObject Rest API action.
GetObjectLegalHoldAction = "s3:GetObjectLegalHold"
// PutObjectLegalHoldAction - PutObjectLegalHold, PutObject Rest API action.
PutObjectLegalHoldAction = "s3:PutObjectLegalHold"
// GetBucketObjectLockConfigurationAction - GetObjectLockConfiguration Rest API action
GetBucketObjectLockConfigurationAction = "s3:GetBucketObjectLockConfiguration"
// PutBucketObjectLockConfigurationAction - PutObjectLockConfiguration Rest API action
@ -113,6 +117,8 @@ func (action Action) isObjectAction() bool {
return true
case PutObjectRetentionAction, GetObjectRetentionAction:
return true
case PutObjectLegalHoldAction, GetObjectLegalHoldAction:
return true
case BypassGovernanceModeAction, BypassGovernanceRetentionAction:
return true
}
@ -143,6 +149,8 @@ func (action Action) IsValid() bool {
return true
case PutObjectRetentionAction, GetObjectRetentionAction:
return true
case PutObjectLegalHoldAction, GetObjectLegalHoldAction:
return true
case PutBucketObjectLockConfigurationAction, GetBucketObjectLockConfigurationAction:
return true
}
@ -231,6 +239,8 @@ var actionConditionKeyMap = map[Action]condition.KeySet{
GetObjectRetentionAction: condition.NewKeySet(condition.CommonKeys...),
BypassGovernanceModeAction: condition.NewKeySet(condition.CommonKeys...),
BypassGovernanceRetentionAction: condition.NewKeySet(condition.CommonKeys...),
PutObjectLegalHoldAction: condition.NewKeySet(condition.CommonKeys...),
GetObjectLegalHoldAction: condition.NewKeySet(condition.CommonKeys...),
GetBucketObjectLockConfigurationAction: condition.NewKeySet(condition.CommonKeys...),
PutBucketObjectLockConfigurationAction: condition.NewKeySet(condition.CommonKeys...),
}