9b3b04ecec
We should allow quorum errors to be send upwards such that caller can retry while reading bucket encryption/policy configs when server is starting up, this allows distributed setups to load the configuration properly. Current code didn't facilitate this and would have never loaded the actual configs during rolling, server restarts.
568 lines
16 KiB
Go
568 lines
16 KiB
Go
/*
|
|
* MinIO Cloud Storage, (C) 2020 MinIO, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package lock
|
|
|
|
import (
|
|
"context"
|
|
"encoding/xml"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/beevik/ntp"
|
|
"github.com/minio/minio/cmd/logger"
|
|
"github.com/minio/minio/pkg/env"
|
|
)
|
|
|
|
// RetMode - object retention mode.
|
|
type RetMode string
|
|
|
|
const (
|
|
// RetGovernance - governance mode.
|
|
RetGovernance RetMode = "GOVERNANCE"
|
|
|
|
// RetCompliance - compliance mode.
|
|
RetCompliance RetMode = "COMPLIANCE"
|
|
)
|
|
|
|
// Valid - returns if retention mode is valid
|
|
func (r RetMode) Valid() bool {
|
|
switch r {
|
|
case RetGovernance, RetCompliance:
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func parseRetMode(modeStr string) (mode RetMode) {
|
|
switch strings.ToUpper(modeStr) {
|
|
case "GOVERNANCE":
|
|
mode = RetGovernance
|
|
case "COMPLIANCE":
|
|
mode = RetCompliance
|
|
}
|
|
return mode
|
|
}
|
|
|
|
// LegalHoldStatus - object legal hold status.
|
|
type LegalHoldStatus string
|
|
|
|
const (
|
|
// LegalHoldOn - legal hold is on.
|
|
LegalHoldOn LegalHoldStatus = "ON"
|
|
|
|
// LegalHoldOff - legal hold is off.
|
|
LegalHoldOff LegalHoldStatus = "OFF"
|
|
)
|
|
|
|
// Valid - returns true if legal hold status has valid values
|
|
func (l LegalHoldStatus) Valid() bool {
|
|
switch l {
|
|
case LegalHoldOn, LegalHoldOff:
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func parseLegalHoldStatus(holdStr string) (st LegalHoldStatus) {
|
|
switch strings.ToUpper(holdStr) {
|
|
case "ON":
|
|
st = LegalHoldOn
|
|
case "OFF":
|
|
st = LegalHoldOff
|
|
}
|
|
return st
|
|
}
|
|
|
|
// Bypass retention governance header.
|
|
const (
|
|
AmzObjectLockBypassRetGovernance = "X-Amz-Bypass-Governance-Retention"
|
|
AmzObjectLockRetainUntilDate = "X-Amz-Object-Lock-Retain-Until-Date"
|
|
AmzObjectLockMode = "X-Amz-Object-Lock-Mode"
|
|
AmzObjectLockLegalHold = "X-Amz-Object-Lock-Legal-Hold"
|
|
)
|
|
|
|
var (
|
|
// ErrMalformedBucketObjectConfig -indicates that the bucket object lock config is malformed
|
|
ErrMalformedBucketObjectConfig = errors.New("invalid bucket object lock config")
|
|
// ErrInvalidRetentionDate - indicates that retention date needs to be in ISO 8601 format
|
|
ErrInvalidRetentionDate = errors.New("date must be provided in ISO 8601 format")
|
|
// ErrPastObjectLockRetainDate - indicates that retention date must be in the future
|
|
ErrPastObjectLockRetainDate = errors.New("the retain until date must be in the future")
|
|
// ErrUnknownWORMModeDirective - indicates that the retention mode is invalid
|
|
ErrUnknownWORMModeDirective = errors.New("unknown WORM mode directive")
|
|
// ErrObjectLockMissingContentMD5 - indicates missing Content-MD5 header for put object requests with locking
|
|
ErrObjectLockMissingContentMD5 = errors.New("content-MD5 HTTP header is required for Put Object requests with Object Lock parameters")
|
|
// ErrObjectLockInvalidHeaders indicates that object lock headers are missing
|
|
ErrObjectLockInvalidHeaders = errors.New("x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied")
|
|
// ErrMalformedXML - generic error indicating malformed XML
|
|
ErrMalformedXML = errors.New("the XML you provided was not well-formed or did not validate against our published schema")
|
|
)
|
|
|
|
const (
|
|
ntpServerEnv = "MINIO_NTP_SERVER"
|
|
)
|
|
|
|
var (
|
|
ntpServer = env.Get(ntpServerEnv, "")
|
|
)
|
|
|
|
// UTCNowNTP - is similar in functionality to UTCNow()
|
|
// but only used when we do not wish to rely on system
|
|
// time.
|
|
func UTCNowNTP() (time.Time, error) {
|
|
// ntp server is disabled
|
|
if ntpServer == "" {
|
|
return time.Now().UTC(), nil
|
|
}
|
|
return ntp.Time(ntpServer)
|
|
}
|
|
|
|
// Retention - bucket level retention configuration.
|
|
type Retention struct {
|
|
Mode RetMode
|
|
Validity time.Duration
|
|
}
|
|
|
|
// IsEmpty - returns whether retention is empty or not.
|
|
func (r Retention) IsEmpty() bool {
|
|
return !r.Mode.Valid() || r.Validity == 0
|
|
}
|
|
|
|
// Retain - check whether given date is retainable by validity time.
|
|
func (r Retention) Retain(created time.Time) bool {
|
|
t, err := UTCNowNTP()
|
|
if err != nil {
|
|
logger.LogIf(context.Background(), err)
|
|
// Retain
|
|
return true
|
|
}
|
|
return created.Add(r.Validity).After(t)
|
|
}
|
|
|
|
// BucketObjectLockConfig - map of bucket and retention configuration.
|
|
type BucketObjectLockConfig struct {
|
|
sync.RWMutex
|
|
retentionMap map[string]*Retention
|
|
}
|
|
|
|
// Set - set retention configuration.
|
|
func (config *BucketObjectLockConfig) Set(bucketName string, retention *Retention) {
|
|
config.Lock()
|
|
config.retentionMap[bucketName] = retention
|
|
config.Unlock()
|
|
}
|
|
|
|
// Get - Get retention configuration.
|
|
func (config *BucketObjectLockConfig) Get(bucketName string) (r *Retention, ok bool) {
|
|
config.RLock()
|
|
defer config.RUnlock()
|
|
r, ok = config.retentionMap[bucketName]
|
|
return r, ok
|
|
}
|
|
|
|
// Remove - removes retention configuration.
|
|
func (config *BucketObjectLockConfig) Remove(bucketName string) {
|
|
config.Lock()
|
|
delete(config.retentionMap, bucketName)
|
|
config.Unlock()
|
|
}
|
|
|
|
// NewBucketObjectLockConfig returns initialized BucketObjectLockConfig
|
|
func NewBucketObjectLockConfig() *BucketObjectLockConfig {
|
|
return &BucketObjectLockConfig{
|
|
retentionMap: make(map[string]*Retention),
|
|
}
|
|
}
|
|
|
|
// DefaultRetention - default retention configuration.
|
|
type DefaultRetention struct {
|
|
XMLName xml.Name `xml:"DefaultRetention"`
|
|
Mode RetMode `xml:"Mode"`
|
|
Days *uint64 `xml:"Days"`
|
|
Years *uint64 `xml:"Years"`
|
|
}
|
|
|
|
// Maximum support retention days and years supported by AWS S3.
|
|
const (
|
|
// This tested by using `mc lock` command
|
|
maximumRetentionDays = 36500
|
|
maximumRetentionYears = 100
|
|
)
|
|
|
|
// UnmarshalXML - decodes XML data.
|
|
func (dr *DefaultRetention) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
|
// Make subtype to avoid recursive UnmarshalXML().
|
|
type defaultRetention DefaultRetention
|
|
retention := defaultRetention{}
|
|
|
|
if err := d.DecodeElement(&retention, &start); err != nil {
|
|
return err
|
|
}
|
|
|
|
switch retention.Mode {
|
|
case RetGovernance, RetCompliance:
|
|
default:
|
|
return fmt.Errorf("unknown retention mode %v", retention.Mode)
|
|
}
|
|
|
|
if retention.Days == nil && retention.Years == nil {
|
|
return fmt.Errorf("either Days or Years must be specified")
|
|
}
|
|
|
|
if retention.Days != nil && retention.Years != nil {
|
|
return fmt.Errorf("either Days or Years must be specified, not both")
|
|
}
|
|
|
|
if retention.Days != nil {
|
|
if *retention.Days == 0 {
|
|
return fmt.Errorf("Default retention period must be a positive integer value for 'Days'")
|
|
}
|
|
if *retention.Days > maximumRetentionDays {
|
|
return fmt.Errorf("Default retention period too large for 'Days' %d", *retention.Days)
|
|
}
|
|
} else if *retention.Years == 0 {
|
|
return fmt.Errorf("Default retention period must be a positive integer value for 'Years'")
|
|
} else if *retention.Years > maximumRetentionYears {
|
|
return fmt.Errorf("Default retention period too large for 'Years' %d", *retention.Years)
|
|
}
|
|
|
|
*dr = DefaultRetention(retention)
|
|
|
|
return nil
|
|
}
|
|
|
|
// Config - object lock configuration specified in
|
|
// https://docs.aws.amazon.com/AmazonS3/latest/API/Type_API_ObjectLockConfiguration.html
|
|
type Config struct {
|
|
XMLNS string `xml:"xmlns,attr,omitempty"`
|
|
XMLName xml.Name `xml:"ObjectLockConfiguration"`
|
|
ObjectLockEnabled string `xml:"ObjectLockEnabled"`
|
|
Rule *struct {
|
|
DefaultRetention DefaultRetention `xml:"DefaultRetention"`
|
|
} `xml:"Rule,omitempty"`
|
|
}
|
|
|
|
// UnmarshalXML - decodes XML data.
|
|
func (config *Config) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
|
// Make subtype to avoid recursive UnmarshalXML().
|
|
type objectLockConfig Config
|
|
parsedConfig := objectLockConfig{}
|
|
|
|
if err := d.DecodeElement(&parsedConfig, &start); err != nil {
|
|
return err
|
|
}
|
|
|
|
if parsedConfig.ObjectLockEnabled != "Enabled" {
|
|
return fmt.Errorf("only 'Enabled' value is allowd to ObjectLockEnabled element")
|
|
}
|
|
|
|
*config = Config(parsedConfig)
|
|
return nil
|
|
}
|
|
|
|
// ToRetention - convert to Retention type.
|
|
func (config *Config) ToRetention() (r *Retention) {
|
|
r = &Retention{}
|
|
if config.Rule != nil {
|
|
r.Mode = config.Rule.DefaultRetention.Mode
|
|
|
|
t, err := UTCNowNTP()
|
|
if err != nil {
|
|
logger.LogIf(context.Background(), err)
|
|
// Do not change any configuration
|
|
// upon NTP failure.
|
|
return r
|
|
}
|
|
|
|
if config.Rule.DefaultRetention.Days != nil {
|
|
r.Validity = t.AddDate(0, 0, int(*config.Rule.DefaultRetention.Days)).Sub(t)
|
|
} else {
|
|
r.Validity = t.AddDate(int(*config.Rule.DefaultRetention.Years), 0, 0).Sub(t)
|
|
}
|
|
}
|
|
|
|
return r
|
|
}
|
|
|
|
// Maximum 4KiB size per object lock config.
|
|
const maxObjectLockConfigSize = 1 << 12
|
|
|
|
// ParseObjectLockConfig parses ObjectLockConfig from xml
|
|
func ParseObjectLockConfig(reader io.Reader) (*Config, error) {
|
|
config := Config{}
|
|
if err := xml.NewDecoder(io.LimitReader(reader, maxObjectLockConfigSize)).Decode(&config); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &config, nil
|
|
}
|
|
|
|
// NewObjectLockConfig returns a initialized lock.Config struct
|
|
func NewObjectLockConfig() *Config {
|
|
return &Config{
|
|
ObjectLockEnabled: "Enabled",
|
|
}
|
|
}
|
|
|
|
// RetentionDate is a embedded type containing time.Time to unmarshal
|
|
// Date in Retention
|
|
type RetentionDate struct {
|
|
time.Time
|
|
}
|
|
|
|
// UnmarshalXML parses date from Retention and validates date format
|
|
func (rDate *RetentionDate) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
|
|
var dateStr string
|
|
err := d.DecodeElement(&dateStr, &startElement)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// While AWS documentation mentions that the date specified
|
|
// must be present in ISO 8601 format, in reality they allow
|
|
// users to provide RFC 3339 compliant dates.
|
|
retDate, err := time.Parse(time.RFC3339, dateStr)
|
|
if err != nil {
|
|
return ErrInvalidRetentionDate
|
|
}
|
|
|
|
*rDate = RetentionDate{retDate}
|
|
return nil
|
|
}
|
|
|
|
// MarshalXML encodes expiration date if it is non-zero and encodes
|
|
// empty string otherwise
|
|
func (rDate *RetentionDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
|
|
if *rDate == (RetentionDate{time.Time{}}) {
|
|
return nil
|
|
}
|
|
return e.EncodeElement(rDate.Format(time.RFC3339), startElement)
|
|
}
|
|
|
|
// ObjectRetention specified in
|
|
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectRetention.html
|
|
type ObjectRetention struct {
|
|
XMLNS string `xml:"xmlns,attr,omitempty"`
|
|
XMLName xml.Name `xml:"Retention"`
|
|
Mode RetMode `xml:"Mode,omitempty"`
|
|
RetainUntilDate RetentionDate `xml:"RetainUntilDate,omitempty"`
|
|
}
|
|
|
|
// Maximum 4KiB size per object retention config.
|
|
const maxObjectRetentionSize = 1 << 12
|
|
|
|
// ParseObjectRetention constructs ObjectRetention struct from xml input
|
|
func ParseObjectRetention(reader io.Reader) (*ObjectRetention, error) {
|
|
ret := ObjectRetention{}
|
|
if err := xml.NewDecoder(io.LimitReader(reader, maxObjectRetentionSize)).Decode(&ret); err != nil {
|
|
return nil, err
|
|
}
|
|
if !ret.Mode.Valid() {
|
|
return &ret, ErrUnknownWORMModeDirective
|
|
}
|
|
|
|
t, err := UTCNowNTP()
|
|
if err != nil {
|
|
logger.LogIf(context.Background(), err)
|
|
return &ret, ErrPastObjectLockRetainDate
|
|
}
|
|
|
|
if ret.RetainUntilDate.Before(t) {
|
|
return &ret, ErrPastObjectLockRetainDate
|
|
}
|
|
|
|
return &ret, nil
|
|
}
|
|
|
|
// IsObjectLockRetentionRequested returns true if object lock retention headers are set.
|
|
func IsObjectLockRetentionRequested(h http.Header) bool {
|
|
if _, ok := h[AmzObjectLockMode]; ok {
|
|
return true
|
|
}
|
|
if _, ok := h[AmzObjectLockRetainUntilDate]; ok {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// IsObjectLockLegalHoldRequested returns true if object lock legal hold header is set.
|
|
func IsObjectLockLegalHoldRequested(h http.Header) bool {
|
|
_, ok := h[AmzObjectLockLegalHold]
|
|
return ok
|
|
}
|
|
|
|
// IsObjectLockGovernanceBypassSet returns true if object lock governance bypass header is set.
|
|
func IsObjectLockGovernanceBypassSet(h http.Header) bool {
|
|
return strings.ToLower(h.Get(AmzObjectLockBypassRetGovernance)) == "true"
|
|
}
|
|
|
|
// IsObjectLockRequested returns true if legal hold or object lock retention headers are requested.
|
|
func IsObjectLockRequested(h http.Header) bool {
|
|
return IsObjectLockLegalHoldRequested(h) || IsObjectLockRetentionRequested(h)
|
|
}
|
|
|
|
// ParseObjectLockRetentionHeaders parses http headers to extract retention mode and retention date
|
|
func ParseObjectLockRetentionHeaders(h http.Header) (rmode RetMode, r RetentionDate, err error) {
|
|
retMode := h.Get(AmzObjectLockMode)
|
|
dateStr := h.Get(AmzObjectLockRetainUntilDate)
|
|
if len(retMode) == 0 || len(dateStr) == 0 {
|
|
return rmode, r, ErrObjectLockInvalidHeaders
|
|
}
|
|
|
|
rmode = parseRetMode(retMode)
|
|
if !rmode.Valid() {
|
|
return rmode, r, ErrUnknownWORMModeDirective
|
|
}
|
|
|
|
var retDate time.Time
|
|
// While AWS documentation mentions that the date specified
|
|
// must be present in ISO 8601 format, in reality they allow
|
|
// users to provide RFC 3339 compliant dates.
|
|
retDate, err = time.Parse(time.RFC3339, dateStr)
|
|
if err != nil {
|
|
return rmode, r, ErrInvalidRetentionDate
|
|
}
|
|
|
|
t, err := UTCNowNTP()
|
|
if err != nil {
|
|
logger.LogIf(context.Background(), err)
|
|
return rmode, r, ErrPastObjectLockRetainDate
|
|
}
|
|
|
|
if retDate.Before(t) {
|
|
return rmode, r, ErrPastObjectLockRetainDate
|
|
}
|
|
|
|
return rmode, RetentionDate{retDate}, nil
|
|
|
|
}
|
|
|
|
// GetObjectRetentionMeta constructs ObjectRetention from metadata
|
|
func GetObjectRetentionMeta(meta map[string]string) ObjectRetention {
|
|
var mode RetMode
|
|
var retainTill RetentionDate
|
|
|
|
var modeStr, tillStr string
|
|
ok := false
|
|
|
|
modeStr, ok = meta[strings.ToLower(AmzObjectLockMode)]
|
|
if !ok {
|
|
modeStr, ok = meta[AmzObjectLockMode]
|
|
}
|
|
if ok {
|
|
mode = parseRetMode(modeStr)
|
|
}
|
|
tillStr, ok = meta[strings.ToLower(AmzObjectLockRetainUntilDate)]
|
|
if !ok {
|
|
tillStr, ok = meta[AmzObjectLockRetainUntilDate]
|
|
}
|
|
if ok {
|
|
if t, e := time.Parse(time.RFC3339, tillStr); e == nil {
|
|
retainTill = RetentionDate{t.UTC()}
|
|
}
|
|
}
|
|
return ObjectRetention{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/", Mode: mode, RetainUntilDate: retainTill}
|
|
}
|
|
|
|
// GetObjectLegalHoldMeta constructs ObjectLegalHold from metadata
|
|
func GetObjectLegalHoldMeta(meta map[string]string) ObjectLegalHold {
|
|
holdStr, ok := meta[strings.ToLower(AmzObjectLockLegalHold)]
|
|
if !ok {
|
|
holdStr, ok = meta[AmzObjectLockLegalHold]
|
|
}
|
|
if ok {
|
|
return ObjectLegalHold{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/", Status: parseLegalHoldStatus(holdStr)}
|
|
}
|
|
return ObjectLegalHold{}
|
|
}
|
|
|
|
// ParseObjectLockLegalHoldHeaders parses request headers to construct ObjectLegalHold
|
|
func ParseObjectLockLegalHoldHeaders(h http.Header) (lhold ObjectLegalHold, err error) {
|
|
holdStatus, ok := h[AmzObjectLockLegalHold]
|
|
if ok {
|
|
lh := parseLegalHoldStatus(holdStatus[0])
|
|
if !lh.Valid() {
|
|
return lhold, ErrUnknownWORMModeDirective
|
|
}
|
|
lhold = ObjectLegalHold{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/", Status: lh}
|
|
}
|
|
return lhold, nil
|
|
|
|
}
|
|
|
|
// ObjectLegalHold specified in
|
|
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectLegalHold.html
|
|
type ObjectLegalHold struct {
|
|
XMLNS string `xml:"xmlns,attr,omitempty"`
|
|
XMLName xml.Name `xml:"LegalHold"`
|
|
Status LegalHoldStatus `xml:"Status,omitempty"`
|
|
}
|
|
|
|
// IsEmpty returns true if struct is empty
|
|
func (l *ObjectLegalHold) IsEmpty() bool {
|
|
return !l.Status.Valid()
|
|
}
|
|
|
|
// ParseObjectLegalHold decodes the XML into ObjectLegalHold
|
|
func ParseObjectLegalHold(reader io.Reader) (hold *ObjectLegalHold, err error) {
|
|
hold = &ObjectLegalHold{}
|
|
if err = xml.NewDecoder(reader).Decode(hold); err != nil {
|
|
return
|
|
}
|
|
|
|
if !hold.Status.Valid() {
|
|
return nil, ErrMalformedXML
|
|
}
|
|
return
|
|
}
|
|
|
|
// FilterObjectLockMetadata filters object lock metadata if s3:GetObjectRetention permission is denied or if isCopy flag set.
|
|
func FilterObjectLockMetadata(metadata map[string]string, filterRetention, filterLegalHold bool) map[string]string {
|
|
// Copy on write
|
|
dst := metadata
|
|
var copied bool
|
|
delKey := func(key string) {
|
|
if _, ok := metadata[key]; !ok {
|
|
return
|
|
}
|
|
if !copied {
|
|
dst = make(map[string]string, len(metadata))
|
|
for k, v := range metadata {
|
|
dst[k] = v
|
|
}
|
|
copied = true
|
|
}
|
|
delete(dst, key)
|
|
}
|
|
legalHold := GetObjectLegalHoldMeta(metadata)
|
|
if !legalHold.Status.Valid() || filterLegalHold {
|
|
delKey(AmzObjectLockLegalHold)
|
|
}
|
|
|
|
ret := GetObjectRetentionMeta(metadata)
|
|
if !ret.Mode.Valid() || filterRetention {
|
|
delKey(AmzObjectLockMode)
|
|
delKey(AmzObjectLockRetainUntilDate)
|
|
return dst
|
|
}
|
|
return dst
|
|
}
|