maxmustermann
/
minio
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

pkg/lifecycle: Add SetPredictionHeaders method (#12755) 

This method is used to add expected expiration and transition time 
for an object in GET/HEAD Object response headers.

Also fixed bugs in lifecycle.PredictTransitionTime and
getLifecycleTransitionTier in handling current and 
non-current versions.
This commit is contained in:
Krishnan Parthasarathi 2021-07-20 17:36:55 -07:00 committed by GitHub
parent 6ea083d197
commit d0963974a5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 204 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
cmd/api-headers.go
View File

@ -28,7 +28,6 @@ import (
"strings"
"time"
"github.com/minio/minio/internal/bucket/lifecycle"
"github.com/minio/minio/internal/crypto"
xhttp "github.com/minio/minio/internal/http"
)
@ -186,42 +185,12 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
}
if lc, err := globalLifecycleSys.Get(objInfo.Bucket); err == nil {
if opts.VersionID == "" {
if ruleID, expiryTime := lc.PredictExpiryTime(lifecycle.ObjectOpts{
Name: objInfo.Name,
UserTags: objInfo.UserTags,
VersionID: objInfo.VersionID,
ModTime: objInfo.ModTime,
IsLatest: objInfo.IsLatest,
DeleteMarker: objInfo.DeleteMarker,
SuccessorModTime: objInfo.SuccessorModTime,
}); !expiryTime.IsZero() {
w.Header()[xhttp.AmzExpiration] = []string{
fmt.Sprintf(`expiry-date="%s", rule-id="%s"`, expiryTime.Format(http.TimeFormat), ruleID),
}
}
}
if objInfo.IsRemote() {
// Check if object is being restored. For more information on x-amz-restore header see
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html#API_HeadObject_ResponseSyntax
w.Header()[xhttp.AmzStorageClass] = []string{objInfo.TransitionTier}
}
ruleID, transitionTime := lc.PredictTransitionTime(lifecycle.ObjectOpts{
Name: objInfo.Name,
UserTags: objInfo.UserTags,
VersionID: objInfo.VersionID,
ModTime: objInfo.ModTime,
IsLatest: objInfo.IsLatest,
DeleteMarker: objInfo.DeleteMarker,
TransitionStatus: objInfo.TransitionStatus,
})
if !transitionTime.IsZero() {
// This header is a MinIO centric extension to show expected transition date in a similar spirit as x-amz-expiration
w.Header()[xhttp.MinIOTransition] = []string{
fmt.Sprintf(`transition-date="%s", rule-id="%s"`, transitionTime.Format(http.TimeFormat), ruleID),
}
}
lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts())
}
return nil

23
cmd/bucket-lifecycle.go
View File

@ -309,9 +309,12 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo)
// getLifeCycleTransitionTier returns storage class for transition target
func getLifeCycleTransitionTier(ctx context.Context, lc *lifecycle.Lifecycle, bucket string, obj lifecycle.ObjectOpts) string {
for _, rule := range lc.FilterActionableRules(obj) {
if rule.Transition.StorageClass != "" {
if obj.IsLatest && rule.Transition.StorageClass != "" {
return rule.Transition.StorageClass
}
if !obj.IsLatest && rule.NoncurrentVersionTransition.StorageClass != "" {
return rule.NoncurrentVersionTransition.StorageClass
}
}
return ""
}
@ -673,3 +676,21 @@ func isRestoredObjectOnDisk(meta map[string]string) (onDisk bool) {
}
return onDisk
}
// ToLifecycleOpts returns lifecycle.ObjectOpts value for oi.
func (oi ObjectInfo) ToLifecycleOpts() lifecycle.ObjectOpts {
return lifecycle.ObjectOpts{
Name: oi.Name,
UserTags: oi.UserTags,
VersionID: oi.VersionID,
ModTime: oi.ModTime,
IsLatest: oi.IsLatest,
NumVersions: oi.NumVersions,
DeleteMarker: oi.DeleteMarker,
SuccessorModTime: oi.SuccessorModTime,
RestoreOngoing: oi.RestoreOngoing,
RestoreExpires: oi.RestoreExpires,
TransitionStatus: oi.TransitionStatus,
RemoteTiersImmediately: globalDebugRemoteTiersImmediately,
}
}

37
cmd/bucket-lifecycle_test.go
View File

@ -246,3 +246,40 @@ func TestValidateTransitionTier(t *testing.T) {
}
}
}
func TestGetLifecycleTransitionTier(t *testing.T) {
lc := lifecycle.Lifecycle{
Rules: []lifecycle.Rule{
{
ID: "rule-1",
Status: "Enabled",
Transition: lifecycle.Transition{
Days: lifecycle.TransitionDays(3),
StorageClass: "TIER-1",
},
},
{
ID: "rule-2",
Status: "Enabled",
NoncurrentVersionTransition: lifecycle.NoncurrentVersionTransition{
NoncurrentDays: lifecycle.ExpirationDays(3),
StorageClass: "TIER-2",
},
},
},
}
obj1 := lifecycle.ObjectOpts{
Name: "obj1",
IsLatest: true,
}
obj2 := lifecycle.ObjectOpts{
Name: "obj2",
}
if got := getLifeCycleTransitionTier(context.TODO(), &lc, "bucket", obj1); got != "TIER-1" {
t.Fatalf("Expected TIER-1 but got %s", got)
}
if got := getLifeCycleTransitionTier(context.TODO(), &lc, "bucket", obj2); got != "TIER-2" {
t.Fatalf("Expected TIER-2 but got %s", got)
}
}

16
cmd/object-handlers-common.go
View File

@ -19,13 +19,11 @@ package cmd
import (
"context"
"fmt"
"net/http"
"regexp"
"strconv"
"time"
"github.com/minio/minio/internal/bucket/lifecycle"
xhttp "github.com/minio/minio/internal/http"
)
@ -262,19 +260,7 @@ func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, delete bool) {
if objInfo.Bucket != "" && objInfo.Name != "" {
if lc, err := globalLifecycleSys.Get(objInfo.Bucket); err == nil && !delete {
ruleID, expiryTime := lc.PredictExpiryTime(lifecycle.ObjectOpts{
Name: objInfo.Name,
UserTags: objInfo.UserTags,
VersionID: objInfo.VersionID,
ModTime: objInfo.ModTime,
IsLatest: objInfo.IsLatest,
DeleteMarker: objInfo.DeleteMarker,
})
if !expiryTime.IsZero() {
w.Header()[xhttp.AmzExpiration] = []string{
fmt.Sprintf(`expiry-date="%s", rule-id="%s"`, expiryTime.Format(http.TimeFormat), ruleID),
}
}
lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts())
}
}
}

38
internal/bucket/lifecycle/lifecycle.go
View File

@ -21,8 +21,11 @@ import (
"encoding/xml"
"fmt"
"io"
"net/http"
"strings"
"time"
xhttp "github.com/minio/minio/internal/http"
)
var (
@ -438,25 +441,38 @@ func (lc Lifecycle) PredictTransitionTime(obj ObjectOpts) (string, time.Time) {
return "", time.Time{}
}
var finalTransitionDate time.Time
var finalTransitionRuleID string
// Iterate over all actionable rules and find the earliest
// transition date and its associated rule ID.
var finalTransitionDate time.Time
var finalTransitionRuleID string
for _, rule := range lc.FilterActionableRules(obj) {
switch {
case !rule.Transition.IsDateNull():
if finalTransitionDate.IsZero() || finalTransitionDate.After(rule.Transition.Date.Time) {
if due, ok := rule.Transition.NextDue(obj); ok {
if finalTransitionDate.IsZero() || finalTransitionDate.After(due) {
finalTransitionRuleID = rule.ID
finalTransitionDate = rule.Transition.Date.Time
finalTransitionDate = due
}
case !rule.Transition.IsDaysNull():
expectedTransition := ExpectedExpiryTime(obj.ModTime, int(rule.Expiration.Days))
if finalTransitionDate.IsZero() || finalTransitionDate.After(expectedTransition) {
}
if due, ok := rule.NoncurrentVersionTransition.NextDue(obj); ok {
if finalTransitionDate.IsZero() || finalTransitionDate.After(due) {
finalTransitionRuleID = rule.ID
finalTransitionDate = expectedTransition
finalTransitionDate = due
}
}
}
return finalTransitionRuleID, finalTransitionDate
}
// SetPredictionHeaders sets time to expiry and transition headers on w for a
// given obj.
func (lc Lifecycle) SetPredictionHeaders(w http.ResponseWriter, obj ObjectOpts) {
if ruleID, expiry := lc.PredictExpiryTime(obj); !expiry.IsZero() {
w.Header()[xhttp.AmzExpiration] = []string{
fmt.Sprintf(`expiry-date="%s", rule-id="%s"`, expiry.Format(http.TimeFormat), ruleID),
}
}
if ruleID, transition := lc.PredictTransitionTime(obj); !transition.IsZero() {
w.Header()[xhttp.MinIOTransition] = []string{
fmt.Sprintf(`transition-date="%s", rule-id="%s"`, transition.Format(http.TimeFormat), ruleID),
}
}
}

87
internal/bucket/lifecycle/lifecycle_test.go
View File

@ -21,8 +21,13 @@ import (
"bytes"
"encoding/xml"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
xhttp "github.com/minio/minio/internal/http"
)
func TestParseAndValidateLifecycleConfig(t *testing.T) {
@ -429,3 +434,85 @@ func TestHasActiveRules(t *testing.T) {
}
}
func TestSetPredictionHeaders(t *testing.T) {
lc := Lifecycle{
Rules: []Rule{
{
ID: "rule-1",
Status: "Enabled",
Expiration: Expiration{
Days: ExpirationDays(3),
set: true,
},
},
{
ID: "rule-2",
Status: "Enabled",
Transition: Transition{
Days: TransitionDays(3),
StorageClass: "TIER-1",
set: true,
},
},
{
ID: "rule-3",
Status: "Enabled",
NoncurrentVersionTransition: NoncurrentVersionTransition{
NoncurrentDays: ExpirationDays(5),
StorageClass: "TIER-2",
set: true,
},
},
},
}
// current version
obj1 := ObjectOpts{
Name: "obj1",
IsLatest: true,
}
// non-current version
obj2 := ObjectOpts{
Name: "obj2",
}
tests := []struct {
obj ObjectOpts
expRuleID int
transRuleID int
}{
{
obj: obj1,
expRuleID: 0,
transRuleID: 1,
},
{
obj: obj2,
expRuleID: 0,
transRuleID: 2,
},
}
for i, tc := range tests {
w := httptest.NewRecorder()
lc.SetPredictionHeaders(w, tc.obj)
if expHdrs, ok := w.Header()[xhttp.AmzExpiration]; ok && !strings.Contains(expHdrs[0], lc.Rules[tc.expRuleID].ID) {
t.Fatalf("Test %d: Expected %s header", i+1, xhttp.AmzExpiration)
}
if transHdrs, ok := w.Header()[xhttp.MinIOTransition]; ok {
if !strings.Contains(transHdrs[0], lc.Rules[tc.transRuleID].ID) {
t.Fatalf("Test %d: Expected %s header", i+1, xhttp.MinIOTransition)
}
if tc.obj.IsLatest {
if expectedDue, _ := lc.Rules[tc.transRuleID].Transition.NextDue(tc.obj); !strings.Contains(transHdrs[0], expectedDue.Format(http.TimeFormat)) {
t.Fatalf("Test %d: Expected transition time %s", i+1, expectedDue)
}
} else {
if expectedDue, _ := lc.Rules[tc.transRuleID].NoncurrentVersionTransition.NextDue(tc.obj); !strings.Contains(transHdrs[0], expectedDue.Format(http.TimeFormat)) {
t.Fatalf("Test %d: Expected transition time %s", i+1, expectedDue)
}
}
}
}
}

12
internal/bucket/lifecycle/noncurrentversion.go
View File

@ -19,6 +19,7 @@ package lifecycle
import (
"encoding/xml"
"time"
)
// NoncurrentVersionExpiration - an action for lifecycle configuration rule.
@ -112,3 +113,14 @@ func (n NoncurrentVersionTransition) Validate() error {
}
return nil
}
// NextDue returns upcoming NoncurrentVersionTransition date for obj if
// applicable, returns false otherwise.
func (n NoncurrentVersionTransition) NextDue(obj ObjectOpts) (time.Time, bool) {
switch {
case obj.IsLatest, n.IsDaysNull():
return time.Time{}, false
}
return ExpectedExpiryTime(obj.SuccessorModTime, int(n.NoncurrentDays)), true
}

17
internal/bucket/lifecycle/transition.go
View File

@ -163,3 +163,20 @@ func (t Transition) IsDateNull() bool {
func (t Transition) IsNull() bool {
return t.IsDaysNull() && t.IsDateNull()
}
// NextDue returns upcoming transition date for obj and true if applicable,
// returns false otherwise.
func (t Transition) NextDue(obj ObjectOpts) (time.Time, bool) {
if !obj.IsLatest {
return time.Time{}, false
}
switch {
case !t.IsDateNull():
return t.Date.Time, true
case !t.IsDaysNull():
return ExpectedExpiryTime(obj.ModTime, int(t.Days)), true
}
return time.Time{}, false
}