maxmustermann
/
minio
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add immediate inline tiering support (#13298)

This commit is contained in:
Krishnan Parthasarathi 2021-10-01 11:58:17 -07:00 committed by GitHub
parent cfbaf7bf1c
commit f3aeed77e5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 223 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
cmd/bucket-lifecycle.go
View File

@ -220,16 +220,31 @@ var errInvalidStorageClass = errors.New("invalid storage class")
func validateTransitionTier(lc *lifecycle.Lifecycle) error {
for _, rule := range lc.Rules {
if rule.Transition.StorageClass == "" {
continue
if rule.Transition.StorageClass != "" {
if valid := globalTierConfigMgr.IsTierValid(rule.Transition.StorageClass); !valid {
return errInvalidStorageClass
}
}
if valid := globalTierConfigMgr.IsTierValid(rule.Transition.StorageClass); !valid {
return errInvalidStorageClass
if rule.NoncurrentVersionTransition.StorageClass != "" {
if valid := globalTierConfigMgr.IsTierValid(rule.NoncurrentVersionTransition.StorageClass); !valid {
return errInvalidStorageClass
}
}
}
return nil
}
// enqueueTransitionImmediate enqueues obj for transition if eligible.
// This is to be called after a successful upload of an object (version).
func enqueueTransitionImmediate(obj ObjectInfo) {
if lc, err := globalLifecycleSys.Get(obj.Bucket); err == nil {
switch lc.ComputeAction(obj.ToLifecycleOpts()) {
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
globalTransitionState.queueTransitionTask(obj)
}
}
}
// expireAction represents different actions to be performed on expiry of a
// restored/transitioned object
type expireAction int
@ -702,17 +717,16 @@ func isRestoredObjectOnDisk(meta map[string]string) (onDisk bool) {
// ToLifecycleOpts returns lifecycle.ObjectOpts value for oi.
func (oi ObjectInfo) ToLifecycleOpts() lifecycle.ObjectOpts {
return lifecycle.ObjectOpts{
Name: oi.Name,
UserTags: oi.UserTags,
VersionID: oi.VersionID,
ModTime: oi.ModTime,
IsLatest: oi.IsLatest,
NumVersions: oi.NumVersions,
DeleteMarker: oi.DeleteMarker,
SuccessorModTime: oi.SuccessorModTime,
RestoreOngoing: oi.RestoreOngoing,
RestoreExpires: oi.RestoreExpires,
TransitionStatus: oi.TransitionedObject.Status,
RemoteTiersImmediately: globalDebugRemoteTiersImmediately,
Name: oi.Name,
UserTags: oi.UserTags,
VersionID: oi.VersionID,
ModTime: oi.ModTime,
IsLatest: oi.IsLatest,
NumVersions: oi.NumVersions,
DeleteMarker: oi.DeleteMarker,
SuccessorModTime: oi.SuccessorModTime,
RestoreOngoing: oi.RestoreOngoing,
RestoreExpires: oi.RestoreExpires,
TransitionStatus: oi.TransitionedObject.Status,
}
}

4
cmd/common-main.go
View File

@ -598,10 +598,6 @@ func handleCommonEnvVars() {
}
GlobalKMS = KMS
}
if tiers := env.Get("_MINIO_DEBUG_REMOTE_TIERS_IMMEDIATELY", ""); tiers != "" {
globalDebugRemoteTiersImmediately = strings.Split(tiers, ",")
}
}
func logStartupMessage(msg string) {

23
cmd/data-scanner.go
View File

@ -879,18 +879,17 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
versionID := oi.VersionID
action := i.lifeCycle.ComputeAction(
lifecycle.ObjectOpts{
Name: i.objectPath(),
UserTags: oi.UserTags,
ModTime: oi.ModTime,
VersionID: oi.VersionID,
DeleteMarker: oi.DeleteMarker,
IsLatest: oi.IsLatest,
NumVersions: oi.NumVersions,
SuccessorModTime: oi.SuccessorModTime,
RestoreOngoing: oi.RestoreOngoing,
RestoreExpires: oi.RestoreExpires,
TransitionStatus: oi.TransitionedObject.Status,
RemoteTiersImmediately: globalDebugRemoteTiersImmediately,
Name: i.objectPath(),
UserTags: oi.UserTags,
ModTime: oi.ModTime,
VersionID: oi.VersionID,
DeleteMarker: oi.DeleteMarker,
IsLatest: oi.IsLatest,
NumVersions: oi.NumVersions,
SuccessorModTime: oi.SuccessorModTime,
RestoreOngoing: oi.RestoreOngoing,
RestoreExpires: oi.RestoreExpires,
TransitionStatus: oi.TransitionedObject.Status,
})
if i.debug {
if versionID != "" {

1
cmd/globals.go
View File

@ -321,7 +321,6 @@ var (
globalConsoleSrv *restapi.Server
globalDebugRemoteTiersImmediately []string
// Add new variable global values here.
)

9
cmd/object-handlers.go
View File

@ -1507,6 +1507,11 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
UserAgent: r.UserAgent(),
Host: handlers.GetSourceIP(r),
})
if !globalTierConfigMgr.Empty() {
// Schedule object for immediate transition if eligible.
enqueueTransitionImmediate(objInfo)
}
}
// PutObjectHandler - PUT Object
@ -1851,6 +1856,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
// Remove the transitioned object whose object version is being overwritten.
if !globalTierConfigMgr.Empty() {
// Schedule object for immediate transition if eligible.
enqueueTransitionImmediate(objInfo)
logger.LogIf(ctx, os.Sweep())
}
}
@ -3292,6 +3299,8 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
// Remove the transitioned object whose object version is being overwritten.
if !globalTierConfigMgr.Empty() {
// Schedule object for immediate transition if eligible.
enqueueTransitionImmediate(objInfo)
logger.LogIf(ctx, os.Sweep())
}
}

67
internal/bucket/lifecycle/lifecycle.go
View File

@ -137,7 +137,7 @@ func (lc Lifecycle) HasActiveRules(prefix string, recursive bool) bool {
if rule.NoncurrentVersionExpiration.NoncurrentDays > 0 {
return true
}
if rule.NoncurrentVersionTransition.NoncurrentDays > 0 {
if !rule.NoncurrentVersionTransition.IsNull() {
return true
}
if rule.Expiration.IsNull() && rule.Transition.IsNull() {
@ -146,12 +146,16 @@ func (lc Lifecycle) HasActiveRules(prefix string, recursive bool) bool {
if !rule.Expiration.IsDateNull() && rule.Expiration.Date.Before(time.Now()) {
return true
}
if !rule.Expiration.IsDaysNull() {
return true
}
if !rule.Transition.IsDateNull() && rule.Transition.Date.Before(time.Now()) {
return true
}
if !rule.Expiration.IsDaysNull() || !rule.Transition.IsDaysNull() {
if !rule.Transition.IsNull() { // this allows for Transition.Days to be zero.
return true
}
}
return false
}
@ -175,6 +179,7 @@ func (lc Lifecycle) Validate() error {
if len(lc.Rules) == 0 {
return errLifecycleNoRule
}
// Validate all the rules in the lifecycle config
for _, r := range lc.Rules {
if err := r.Validate(); err != nil {
@ -229,7 +234,7 @@ func (lc Lifecycle) FilterActionableRules(obj ObjectOpts) []Rule {
// The NoncurrentVersionTransition action requests MinIO to transition
// noncurrent versions of objects x days after the objects become
// noncurrent.
if !rule.NoncurrentVersionTransition.IsDaysNull() {
if !rule.NoncurrentVersionTransition.IsNull() {
rules = append(rules, rule)
continue
}
@ -247,29 +252,17 @@ func (lc Lifecycle) FilterActionableRules(obj ObjectOpts) []Rule {
// ObjectOpts provides information to deduce the lifecycle actions
// which can be triggered on the resultant object.
type ObjectOpts struct {
Name string
UserTags string
ModTime time.Time
VersionID string
IsLatest bool
DeleteMarker bool
NumVersions int
SuccessorModTime time.Time
TransitionStatus string
RestoreOngoing bool
RestoreExpires time.Time
RemoteTiersImmediately []string // strictly for debug only
}
// doesMatchDebugTiers returns true if tier matches one of the debugTiers, false
// otherwise.
func doesMatchDebugTiers(tier string, debugTiers []string) bool {
for _, t := range debugTiers {
if strings.ToUpper(tier) == strings.ToUpper(t) {
return true
}
}
return false
Name string
UserTags string
ModTime time.Time
VersionID string
IsLatest bool
DeleteMarker bool
NumVersions int
SuccessorModTime time.Time
TransitionStatus string
RestoreOngoing bool
RestoreExpires time.Time
}
// ExpiredObjectDeleteMarker returns true if an object version referred to by o
@ -316,7 +309,7 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
}
}
if !rule.NoncurrentVersionTransition.IsDaysNull() {
if !rule.NoncurrentVersionTransition.IsNull() {
if obj.VersionID != "" && !obj.IsLatest && !obj.SuccessorModTime.IsZero() && !obj.DeleteMarker && obj.TransitionStatus != TransitionComplete {
// Non current versions should be transitioned if their age exceeds non current days configuration
// https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#intro-lifecycle-rules-actions
@ -324,11 +317,6 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
return TransitionVersionAction
}
// this if condition is strictly for debug purposes to force immediate
// transition to remote tier if _MINIO_DEBUG_REMOTE_TIERS_IMMEDIATELY is set
if doesMatchDebugTiers(rule.NoncurrentVersionTransition.StorageClass, obj.RemoteTiersImmediately) {
return TransitionVersionAction
}
}
}
@ -346,21 +334,10 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
}
if obj.TransitionStatus != TransitionComplete {
switch {
case !rule.Transition.IsDateNull():
if time.Now().UTC().After(rule.Transition.Date.Time) {
if due, ok := rule.Transition.NextDue(obj); ok {
if time.Now().UTC().After(due) {
action = TransitionAction
}
case !rule.Transition.IsDaysNull():
if time.Now().UTC().After(ExpectedExpiryTime(obj.ModTime, int(rule.Transition.Days))) {
action = TransitionAction
}
}
// this if condition is strictly for debug purposes to force immediate
// transition to remote tier if _MINIO_DEBUG_REMOTE_TIERS_IMMEDIATELY is set
if !rule.Transition.IsNull() && doesMatchDebugTiers(rule.Transition.StorageClass, obj.RemoteTiersImmediately) {
action = TransitionAction
}
if !obj.RestoreExpires.IsZero() && time.Now().After(obj.RestoreExpires) {

31
internal/bucket/lifecycle/lifecycle_test.go
View File

@ -104,6 +104,12 @@ func TestParseAndValidateLifecycleConfig(t *testing.T) {
expectedParsingErr: nil,
expectedValidationErr: nil,
},
// Lifecycle with zero Transition Days
{
inputConfig: `<LifecycleConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ID>rule</ID><Filter></Filter><Status>Enabled</Status><Transition><Days>0</Days><StorageClass>S3TIER-1</StorageClass></Transition></Rule></LifecycleConfiguration>`,
expectedParsingErr: nil,
expectedValidationErr: nil,
},
}
for i, tc := range testCases {
@ -119,7 +125,7 @@ func TestParseAndValidateLifecycleConfig(t *testing.T) {
}
err = lc.Validate()
if err != tc.expectedValidationErr {
t.Fatalf("%d: Expected %v during parsing but got %v", i+1, tc.expectedValidationErr, err)
t.Fatalf("%d: Expected %v during validation but got %v", i+1, tc.expectedValidationErr, err)
}
})
}
@ -146,7 +152,7 @@ func TestMarshalLifecycleConfig(t *testing.T) {
Status: "Enabled",
Filter: Filter{Prefix: Prefix{string: "prefix-1", set: true}},
Expiration: Expiration{Date: midnightTS},
NoncurrentVersionTransition: NoncurrentVersionTransition{NoncurrentDays: 2, StorageClass: "TEST"},
NoncurrentVersionTransition: NoncurrentVersionTransition{NoncurrentDays: TransitionDays(2), StorageClass: "TEST"},
},
},
}
@ -380,6 +386,13 @@ func TestComputeActions(t *testing.T) {
isExpiredDelMarker: true,
expectedAction: DeleteVersionAction,
},
// Should not delete expired marker if its time has not come yet
{
inputConfig: `<BucketLifecycleConfiguration><Rule><Filter></Filter><Status>Enabled</Status><Transition><Days>0</Days><StorageClass>S3TIER-1</StorageClass></Transition></Rule></BucketLifecycleConfiguration>`,
objectName: "foodir/fooobject",
objectModTime: time.Now().UTC(), // Created now
expectedAction: TransitionAction,
},
}
for _, tc := range testCases {
@ -441,6 +454,16 @@ func TestHasActiveRules(t *testing.T) {
prefix: "foodir/foobject",
expectedNonRec: false, expectedRec: false,
},
{
inputConfig: `<LifecycleConfiguration><Rule><Status>Enabled</Status><Transition><StorageClass>S3TIER-1</StorageClass></Transition></Rule></LifecycleConfiguration>`,
prefix: "foodir/foobject/foo.txt",
expectedNonRec: true, expectedRec: true,
},
{
inputConfig: `<LifecycleConfiguration><Rule><Status>Enabled</Status><NoncurrentVersionTransition><StorageClass>S3TIER-1</StorageClass></NoncurrentVersionTransition></Rule></LifecycleConfiguration>`,
prefix: "foodir/foobject/foo.txt",
expectedNonRec: true, expectedRec: true,
},
}
for i, tc := range testCases {
@ -486,7 +509,7 @@ func TestSetPredictionHeaders(t *testing.T) {
ID: "rule-3",
Status: "Enabled",
NoncurrentVersionTransition: NoncurrentVersionTransition{
NoncurrentDays: ExpirationDays(5),
NoncurrentDays: TransitionDays(5),
StorageClass: "TIER-2",
set: true,
},
@ -559,7 +582,7 @@ func TestTransitionTier(t *testing.T) {
ID: "rule-2",
Status: "Enabled",
NoncurrentVersionTransition: NoncurrentVersionTransition{
NoncurrentDays: ExpirationDays(3),
NoncurrentDays: TransitionDays(3),
StorageClass: "TIER-2",
},
},

25
internal/bucket/lifecycle/noncurrentversion.go
View File

@ -70,7 +70,7 @@ func (n NoncurrentVersionExpiration) Validate() error {
// NoncurrentVersionTransition - an action for lifecycle configuration rule.
type NoncurrentVersionTransition struct {
NoncurrentDays ExpirationDays `xml:"NoncurrentDays"`
NoncurrentDays TransitionDays `xml:"NoncurrentDays"`
StorageClass string `xml:"StorageClass"`
set bool
}
@ -78,18 +78,13 @@ type NoncurrentVersionTransition struct {
// MarshalXML is extended to leave out
// <NoncurrentVersionTransition></NoncurrentVersionTransition> tags
func (n NoncurrentVersionTransition) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
if n.NoncurrentDays == ExpirationDays(0) {
if n.IsNull() {
return nil
}
type noncurrentVersionTransitionWrapper NoncurrentVersionTransition
return e.EncodeElement(noncurrentVersionTransitionWrapper(n), start)
}
// IsDaysNull returns true if days field is null
func (n NoncurrentVersionTransition) IsDaysNull() bool {
return n.NoncurrentDays == ExpirationDays(0)
}
// UnmarshalXML decodes NoncurrentVersionExpiration
func (n *NoncurrentVersionTransition) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
type noncurrentVersionTransitionWrapper NoncurrentVersionTransition
@ -103,12 +98,18 @@ func (n *NoncurrentVersionTransition) UnmarshalXML(d *xml.Decoder, startElement
return nil
}
// IsNull returns true if NoncurrentTransition doesn't refer to any storage-class.
// Note: It supports immediate transition, i.e zero noncurrent days.
func (n NoncurrentVersionTransition) IsNull() bool {
return n.StorageClass == ""
}
// Validate returns an error with wrong value
func (n NoncurrentVersionTransition) Validate() error {
if !n.set {
return nil
}
if int(n.NoncurrentDays) <= 0 || n.StorageClass == "" {
if n.StorageClass == "" {
return errXMLNotWellFormed
}
return nil
@ -117,10 +118,12 @@ func (n NoncurrentVersionTransition) Validate() error {
// NextDue returns upcoming NoncurrentVersionTransition date for obj if
// applicable, returns false otherwise.
func (n NoncurrentVersionTransition) NextDue(obj ObjectOpts) (time.Time, bool) {
switch {
case obj.IsLatest, n.IsDaysNull():
if obj.IsLatest || n.StorageClass == "" {
return time.Time{}, false
}
// Days == 0 indicates immediate tiering, i.e object is eligible for tiering since it became noncurrent.
if n.NoncurrentDays == 0 {
return obj.SuccessorModTime, true
}
return ExpectedExpiryTime(obj.SuccessorModTime, int(n.NoncurrentDays)), true
}

43
internal/bucket/lifecycle/transition.go
View File

@ -25,7 +25,7 @@ import (
var (
errTransitionInvalidDays = Errorf("Days must be 0 or greater when used with Transition")
errTransitionInvalidDate = Errorf("Date must be provided in ISO 8601 format")
errTransitionInvalid = Errorf("Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present inside Expiration.")
errTransitionInvalid = Errorf("Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present in Transition.")
errTransitionDateNotMidnight = Errorf("'Date' must be at midnight GMT")
)
@ -76,24 +76,23 @@ type TransitionDays int
// UnmarshalXML parses number of days from Transition and validates if
// >= 0
func (tDays *TransitionDays) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
var numDays int
err := d.DecodeElement(&numDays, &startElement)
var days int
err := d.DecodeElement(&days, &startElement)
if err != nil {
return err
}
if numDays < 0 {
if days < 0 {
return errTransitionInvalidDays
}
*tDays = TransitionDays(numDays)
*tDays = TransitionDays(days)
return nil
}
// MarshalXML encodes number of days to expire if it is non-zero and
// encodes empty string otherwise
func (tDays TransitionDays) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
if tDays == 0 {
return nil
}
return e.EncodeElement(int(tDays), startElement)
}
@ -135,25 +134,16 @@ func (t Transition) Validate() error {
return nil
}
if t.IsDaysNull() && t.IsDateNull() {
return errXMLNotWellFormed
}
// Both transition days and date are specified
if !t.IsDaysNull() && !t.IsDateNull() {
if !t.IsDateNull() && t.Days > 0 {
return errTransitionInvalid
}
if t.StorageClass == "" {
return errXMLNotWellFormed
}
return nil
}
// IsDaysNull returns true if days field is null
func (t Transition) IsDaysNull() bool {
return t.Days == TransitionDays(0)
}
// IsDateNull returns true if date field is null
func (t Transition) IsDateNull() bool {
return t.Date.Time.IsZero()
@ -161,22 +151,23 @@ func (t Transition) IsDateNull() bool {
// IsNull returns true if both date and days fields are null
func (t Transition) IsNull() bool {
return t.IsDaysNull() && t.IsDateNull()
return t.StorageClass == ""
}
// NextDue returns upcoming transition date for obj and true if applicable,
// returns false otherwise.
func (t Transition) NextDue(obj ObjectOpts) (time.Time, bool) {
if !obj.IsLatest {
if !obj.IsLatest || t.IsNull() {
return time.Time{}, false
}
switch {
case !t.IsDateNull():
if !t.IsDateNull() {
return t.Date.Time, true
case !t.IsDaysNull():
return ExpectedExpiryTime(obj.ModTime, int(t.Days)), true
}
return time.Time{}, false
// Days == 0 indicates immediate tiering, i.e object is eligible for tiering since its creation.
if t.Days == 0 {
return obj.ModTime, true
}
return ExpectedExpiryTime(obj.ModTime, int(t.Days)), true
}

93
internal/bucket/lifecycle/transition_test.go Normal file
View File

@ -0,0 +1,93 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"encoding/xml"
"testing"
)
func TestTransitionUnmarshalXML(t *testing.T) {
trTests := []struct {
input string
err error
}{
{
input: `<Transition>
<Days>0</Days>
<StorageClass>S3TIER-1</StorageClass>
</Transition>`,
err: nil,
},
{
input: `<Transition>
<Days>1</Days>
<Date>2021-01-01T00:00:00Z</Date>
<StorageClass>S3TIER-1</StorageClass>
</Transition>`,
err: errTransitionInvalid,
},
{
input: `<Transition>
<Days>1</Days>
</Transition>`,
err: errXMLNotWellFormed,
},
}
for i, tc := range trTests {
var tr Transition
err := xml.Unmarshal([]byte(tc.input), &tr)
if err != nil {
t.Fatalf("%d: xml unmarshal failed with %v", i+1, err)
}
if err = tr.Validate(); err != tc.err {
t.Fatalf("%d: Invalid transition %v: err %v", i+1, tr, err)
}
}
ntrTests := []struct {
input string
err error
}{
{
input: `<NoncurrentVersionTransition>
<NoncurrentDays>0</NoncurrentDays>
<StorageClass>S3TIER-1</StorageClass>
</NoncurrentVersionTransition>`,
err: nil,
},
{
input: `<NoncurrentVersionTransition>
<Days>1</Days>
</NoncurrentVersionTransition>`,
err: errXMLNotWellFormed,
},
}
for i, tc := range ntrTests {
var ntr NoncurrentVersionTransition
err := xml.Unmarshal([]byte(tc.input), &ntr)
if err != nil {
t.Fatalf("%d: xml unmarshal failed with %v", i+1, err)
}
if err = ntr.Validate(); err != tc.err {
t.Fatalf("%d: Invalid noncurrent version transition %v: err %v", i+1, ntr, err)
}
}
}