2020-05-01 00:55:54 +02:00
|
|
|
/*
|
|
|
|
* MinIO Cloud Storage, (C) 2020 MinIO, Inc.
|
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
package cmd
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"encoding/json"
|
|
|
|
"fmt"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/minio/minio/cmd/logger"
|
|
|
|
"github.com/minio/minio/pkg/event"
|
|
|
|
"github.com/minio/minio/pkg/madmin"
|
|
|
|
)
|
|
|
|
|
|
|
|
// BucketQuotaSys - map of bucket and quota configuration.
|
2020-05-27 15:45:43 +02:00
|
|
|
type BucketQuotaSys struct {
|
|
|
|
bucketStorageCache timedValue
|
|
|
|
}
|
2020-05-01 00:55:54 +02:00
|
|
|
|
|
|
|
// Get - Get quota configuration.
|
2020-05-20 19:18:15 +02:00
|
|
|
func (sys *BucketQuotaSys) Get(bucketName string) (*madmin.BucketQuota, error) {
|
2020-05-19 22:53:54 +02:00
|
|
|
if globalIsGateway {
|
|
|
|
objAPI := newObjectLayerFn()
|
|
|
|
if objAPI == nil {
|
2020-05-20 19:18:15 +02:00
|
|
|
return nil, errServerNotInitialized
|
2020-05-19 22:53:54 +02:00
|
|
|
}
|
2020-05-20 19:18:15 +02:00
|
|
|
return &madmin.BucketQuota{}, nil
|
2020-05-19 22:53:54 +02:00
|
|
|
}
|
2020-05-01 00:55:54 +02:00
|
|
|
|
2020-05-20 19:18:15 +02:00
|
|
|
return globalBucketMetadataSys.GetQuotaConfig(bucketName)
|
2020-05-01 00:55:54 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// NewBucketQuotaSys returns initialized BucketQuotaSys
|
|
|
|
func NewBucketQuotaSys() *BucketQuotaSys {
|
2020-05-20 19:18:15 +02:00
|
|
|
return &BucketQuotaSys{}
|
2020-05-01 00:55:54 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// parseBucketQuota parses BucketQuota from json
|
2020-05-20 19:18:15 +02:00
|
|
|
func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota, err error) {
|
|
|
|
quotaCfg = &madmin.BucketQuota{}
|
|
|
|
if err = json.Unmarshal(data, quotaCfg); err != nil {
|
2020-05-19 22:53:54 +02:00
|
|
|
return quotaCfg, err
|
2020-05-01 00:55:54 +02:00
|
|
|
}
|
2020-05-20 19:18:15 +02:00
|
|
|
if !quotaCfg.IsValid() {
|
|
|
|
return quotaCfg, fmt.Errorf("Invalid quota config %#v", quotaCfg)
|
2020-05-01 00:55:54 +02:00
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2020-05-27 15:45:43 +02:00
|
|
|
func (sys *BucketQuotaSys) check(ctx context.Context, bucket string, size int64) error {
|
|
|
|
objAPI := newObjectLayerWithoutSafeModeFn()
|
|
|
|
if objAPI == nil {
|
|
|
|
return errServerNotInitialized
|
|
|
|
}
|
|
|
|
|
|
|
|
q, err := sys.Get(bucket)
|
|
|
|
if err != nil {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
if q.Type == madmin.FIFOQuota {
|
|
|
|
return nil
|
|
|
|
}
|
2020-05-01 00:55:54 +02:00
|
|
|
|
2020-05-20 19:18:15 +02:00
|
|
|
if q.Quota == 0 {
|
|
|
|
// No quota set return quickly.
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-05-27 15:45:43 +02:00
|
|
|
sys.bucketStorageCache.Once.Do(func() {
|
|
|
|
sys.bucketStorageCache.TTL = 10 * time.Second
|
|
|
|
sys.bucketStorageCache.Update = func() (interface{}, error) {
|
|
|
|
return loadDataUsageFromBackend(ctx, objAPI)
|
2020-05-01 00:55:54 +02:00
|
|
|
}
|
2020-05-27 15:45:43 +02:00
|
|
|
})
|
|
|
|
|
|
|
|
v, err := sys.bucketStorageCache.Get()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
dui := v.(DataUsageInfo)
|
|
|
|
|
|
|
|
bui, ok := dui.BucketsUsage[bucket]
|
|
|
|
if !ok {
|
|
|
|
// bucket not found, cannot enforce quota
|
|
|
|
// call will fail anyways later.
|
|
|
|
return nil
|
2020-05-01 00:55:54 +02:00
|
|
|
}
|
2020-05-27 15:45:43 +02:00
|
|
|
|
|
|
|
if (bui.Size + uint64(size)) > q.Quota {
|
2020-05-01 00:55:54 +02:00
|
|
|
return BucketQuotaExceeded{Bucket: bucket}
|
|
|
|
}
|
2020-05-27 15:45:43 +02:00
|
|
|
|
2020-05-01 00:55:54 +02:00
|
|
|
return nil
|
|
|
|
}
|
2020-05-27 15:45:43 +02:00
|
|
|
|
2020-05-01 00:55:54 +02:00
|
|
|
func enforceBucketQuota(ctx context.Context, bucket string, size int64) error {
|
|
|
|
if size < 0 {
|
|
|
|
return nil
|
|
|
|
}
|
2020-05-19 22:53:54 +02:00
|
|
|
|
2020-05-27 15:45:43 +02:00
|
|
|
return globalBucketQuotaSys.check(ctx, bucket, size)
|
2020-05-01 00:55:54 +02:00
|
|
|
}
|
|
|
|
|
2020-07-15 03:59:05 +02:00
|
|
|
// enforceFIFOQuota deletes objects in FIFO order until sufficient objects
|
|
|
|
// have been deleted so as to bring bucket usage within quota.
|
|
|
|
func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket string, bui BucketUsageInfo) {
|
|
|
|
// Check if the current bucket has quota restrictions, if not skip it
|
|
|
|
cfg, err := globalBucketQuotaSys.Get(bucket)
|
|
|
|
if err != nil {
|
|
|
|
return
|
2020-05-01 00:55:54 +02:00
|
|
|
}
|
|
|
|
|
2020-07-15 03:59:05 +02:00
|
|
|
if cfg.Type != madmin.FIFOQuota {
|
|
|
|
return
|
|
|
|
}
|
2020-05-01 00:55:54 +02:00
|
|
|
|
2020-07-15 03:59:05 +02:00
|
|
|
var toFree uint64
|
|
|
|
if bui.Size > cfg.Quota && cfg.Quota > 0 {
|
|
|
|
toFree = bui.Size - cfg.Quota
|
2020-05-01 00:55:54 +02:00
|
|
|
}
|
|
|
|
|
2020-07-15 03:59:05 +02:00
|
|
|
if toFree <= 0 {
|
2020-06-13 05:04:01 +02:00
|
|
|
return
|
2020-05-01 00:55:54 +02:00
|
|
|
}
|
2020-05-27 15:45:43 +02:00
|
|
|
|
2020-07-15 03:59:05 +02:00
|
|
|
// Allocate new results channel to receive ObjectInfo.
|
|
|
|
objInfoCh := make(chan ObjectInfo)
|
|
|
|
|
|
|
|
versioned := globalBucketVersioningSys.Enabled(bucket)
|
|
|
|
|
|
|
|
// Walk through all objects
|
|
|
|
if err := objectAPI.Walk(ctx, bucket, "", objInfoCh, ObjectOptions{WalkVersions: versioned}); err != nil {
|
2020-06-13 05:04:01 +02:00
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
return
|
2020-05-20 19:18:15 +02:00
|
|
|
}
|
2020-05-27 15:45:43 +02:00
|
|
|
|
2020-07-15 03:59:05 +02:00
|
|
|
// reuse the fileScorer used by disk cache to score entries by
|
|
|
|
// ModTime to find the oldest objects in bucket to delete. In
|
|
|
|
// the context of bucket quota enforcement - number of hits are
|
|
|
|
// irrelevant.
|
|
|
|
scorer, err := newFileScorer(toFree, time.Now().Unix(), 1)
|
2020-05-27 15:45:43 +02:00
|
|
|
if err != nil {
|
2020-06-13 05:04:01 +02:00
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
return
|
2020-05-27 15:45:43 +02:00
|
|
|
}
|
|
|
|
|
2020-07-15 03:59:05 +02:00
|
|
|
rcfg, _ := globalBucketObjectLockSys.Get(bucket)
|
|
|
|
for obj := range objInfoCh {
|
|
|
|
if obj.DeleteMarker {
|
|
|
|
// Delete markers are automatically added for FIFO purge.
|
|
|
|
scorer.addFileWithObjInfo(obj, 1)
|
2020-05-01 00:55:54 +02:00
|
|
|
continue
|
|
|
|
}
|
2020-07-15 03:59:05 +02:00
|
|
|
// skip objects currently under retention
|
|
|
|
if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
|
2020-05-01 00:55:54 +02:00
|
|
|
continue
|
|
|
|
}
|
2020-07-15 03:59:05 +02:00
|
|
|
scorer.addFileWithObjInfo(obj, 1)
|
|
|
|
}
|
2020-05-19 22:53:54 +02:00
|
|
|
|
2020-07-15 03:59:05 +02:00
|
|
|
// If we saw less than quota we are good.
|
|
|
|
if scorer.seenBytes <= cfg.Quota {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
// Calculate how much we want to delete now.
|
|
|
|
toFreeNow := scorer.seenBytes - cfg.Quota
|
|
|
|
// We were less over quota than we thought. Adjust so we delete less.
|
|
|
|
// If we are more over, leave it for the next run to pick up.
|
|
|
|
if toFreeNow < toFree {
|
|
|
|
if !scorer.adjustSaveBytes(int64(toFreeNow) - int64(toFree)) {
|
|
|
|
// We got below or at quota.
|
|
|
|
return
|
2020-05-01 00:55:54 +02:00
|
|
|
}
|
2020-07-15 03:59:05 +02:00
|
|
|
}
|
2020-05-27 15:45:43 +02:00
|
|
|
|
2020-07-15 03:59:05 +02:00
|
|
|
var objects []ObjectToDelete
|
|
|
|
numKeys := len(scorer.fileObjInfos())
|
|
|
|
for i, obj := range scorer.fileObjInfos() {
|
|
|
|
objects = append(objects, ObjectToDelete{
|
|
|
|
ObjectName: obj.Name,
|
|
|
|
VersionID: obj.VersionID,
|
|
|
|
})
|
|
|
|
if len(objects) < maxDeleteList && (i < numKeys-1) {
|
|
|
|
// skip deletion until maxDeleteList or end of slice
|
2020-06-13 05:04:01 +02:00
|
|
|
continue
|
2020-05-01 00:55:54 +02:00
|
|
|
}
|
|
|
|
|
2020-07-15 03:59:05 +02:00
|
|
|
if len(objects) == 0 {
|
|
|
|
break
|
2020-05-01 00:55:54 +02:00
|
|
|
}
|
2020-06-13 05:04:01 +02:00
|
|
|
|
2020-07-15 03:59:05 +02:00
|
|
|
// Deletes a list of objects.
|
|
|
|
_, deleteErrs := objectAPI.DeleteObjects(ctx, bucket, objects, ObjectOptions{
|
|
|
|
Versioned: versioned,
|
|
|
|
})
|
|
|
|
for i := range deleteErrs {
|
|
|
|
if deleteErrs[i] != nil {
|
|
|
|
logger.LogIf(ctx, deleteErrs[i])
|
2020-05-01 00:55:54 +02:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2020-07-15 03:59:05 +02:00
|
|
|
// Notify object deleted event.
|
|
|
|
sendEvent(eventArgs{
|
|
|
|
EventName: event.ObjectRemovedDelete,
|
|
|
|
BucketName: bucket,
|
|
|
|
Object: obj,
|
|
|
|
Host: "Internal: [FIFO-QUOTA-EXPIRY]",
|
2020-06-13 05:04:01 +02:00
|
|
|
})
|
2020-05-01 00:55:54 +02:00
|
|
|
}
|
2020-07-15 03:59:05 +02:00
|
|
|
objects = nil
|
2020-05-01 00:55:54 +02:00
|
|
|
}
|
|
|
|
}
|