2019-11-13 21:17:45 +01:00
|
|
|
/*
|
|
|
|
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2018-03-28 23:14:06 +02:00
|
|
|
package cmd
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"errors"
|
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"io/ioutil"
|
|
|
|
"net/http"
|
|
|
|
"os"
|
2019-05-22 23:54:15 +02:00
|
|
|
"strings"
|
2019-08-10 02:09:08 +02:00
|
|
|
"sync"
|
2019-05-22 23:54:15 +02:00
|
|
|
"time"
|
2018-03-28 23:14:06 +02:00
|
|
|
|
|
|
|
"github.com/djherbis/atime"
|
2019-10-04 19:35:33 +02:00
|
|
|
"github.com/minio/minio/cmd/config/cache"
|
2018-04-06 00:04:40 +02:00
|
|
|
"github.com/minio/minio/cmd/logger"
|
2019-10-04 19:35:33 +02:00
|
|
|
"github.com/minio/minio/pkg/color"
|
2019-10-14 18:44:51 +02:00
|
|
|
"github.com/minio/minio/pkg/sync/errgroup"
|
2018-07-28 00:32:19 +02:00
|
|
|
"github.com/minio/minio/pkg/wildcard"
|
2018-03-28 23:14:06 +02:00
|
|
|
)
|
|
|
|
|
2019-05-22 23:54:15 +02:00
|
|
|
const (
|
2019-08-10 02:09:08 +02:00
|
|
|
cacheBlkSize = int64(1 * 1024 * 1024)
|
2019-05-22 23:54:15 +02:00
|
|
|
)
|
|
|
|
|
2018-05-30 20:30:14 +02:00
|
|
|
// CacheStorageInfo - represents total, free capacity of
|
|
|
|
// underlying cache storage.
|
|
|
|
type CacheStorageInfo struct {
|
|
|
|
Total uint64 // Total cache disk space.
|
|
|
|
Free uint64 // Free cache available space.
|
|
|
|
}
|
|
|
|
|
2018-03-28 23:14:06 +02:00
|
|
|
// CacheObjectLayer implements primitives for cache object API layer.
|
|
|
|
type CacheObjectLayer interface {
|
|
|
|
// Object operations.
|
2018-09-27 12:06:45 +02:00
|
|
|
GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
|
2018-09-10 18:42:43 +02:00
|
|
|
GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
|
2018-03-28 23:14:06 +02:00
|
|
|
DeleteObject(ctx context.Context, bucket, object string) error
|
2019-05-13 21:25:49 +02:00
|
|
|
DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error)
|
2019-09-05 16:20:16 +02:00
|
|
|
PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
|
2018-03-28 23:14:06 +02:00
|
|
|
// Storage operations.
|
2018-05-30 20:30:14 +02:00
|
|
|
StorageInfo(ctx context.Context) CacheStorageInfo
|
2019-12-06 08:16:06 +01:00
|
|
|
CacheStats() *CacheStats
|
2018-03-28 23:14:06 +02:00
|
|
|
}
|
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
// Abstracts disk caching - used by the S3 layer
|
|
|
|
type cacheObjects struct {
|
|
|
|
// slice of cache drives
|
|
|
|
cache []*diskCache
|
|
|
|
// file path patterns to exclude from cache
|
|
|
|
exclude []string
|
|
|
|
|
|
|
|
// if true migration is in progress from v1 to v2
|
|
|
|
migrating bool
|
|
|
|
// mutex to protect migration bool
|
|
|
|
migMutex sync.Mutex
|
|
|
|
|
2019-11-16 22:44:28 +01:00
|
|
|
// nsMutex namespace lock
|
|
|
|
nsMutex *nsLockMap
|
|
|
|
|
2019-12-06 08:16:06 +01:00
|
|
|
// Cache stats
|
|
|
|
cacheStats *CacheStats
|
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
// Object functions pointing to the corresponding functions of backend implementation.
|
2019-11-13 21:17:45 +01:00
|
|
|
NewNSLockFn func(ctx context.Context, bucket, object string) RWLocker
|
2019-08-10 02:09:08 +02:00
|
|
|
GetObjectNInfoFn func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
|
|
|
|
GetObjectInfoFn func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
|
|
|
|
DeleteObjectFn func(ctx context.Context, bucket, object string) error
|
|
|
|
DeleteObjectsFn func(ctx context.Context, bucket string, objects []string) ([]error, error)
|
2019-09-05 16:20:16 +02:00
|
|
|
PutObjectFn func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
|
2018-03-28 23:14:06 +02:00
|
|
|
}
|
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
func (c *cacheObjects) delete(ctx context.Context, dcache *diskCache, bucket, object string) (err error) {
|
2019-11-13 21:17:45 +01:00
|
|
|
cLock := c.NewNSLockFn(ctx, bucket, object)
|
2019-08-10 02:09:08 +02:00
|
|
|
if err := cLock.GetLock(globalObjectTimeout); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer cLock.Unlock()
|
|
|
|
return dcache.Delete(ctx, bucket, object)
|
2019-05-22 23:54:15 +02:00
|
|
|
}
|
|
|
|
|
2019-12-08 22:58:04 +01:00
|
|
|
func (c *cacheObjects) put(ctx context.Context, dcache *diskCache, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions) error {
|
2019-11-13 21:17:45 +01:00
|
|
|
cLock := c.NewNSLockFn(ctx, bucket, object)
|
2019-08-10 02:09:08 +02:00
|
|
|
if err := cLock.GetLock(globalObjectTimeout); err != nil {
|
|
|
|
return err
|
2018-03-28 23:14:06 +02:00
|
|
|
}
|
2019-08-10 02:09:08 +02:00
|
|
|
defer cLock.Unlock()
|
2019-12-08 22:58:04 +01:00
|
|
|
return dcache.Put(ctx, bucket, object, data, size, rs, opts)
|
2019-08-10 02:09:08 +02:00
|
|
|
}
|
2019-05-22 23:54:15 +02:00
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
func (c *cacheObjects) get(ctx context.Context, dcache *diskCache, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) {
|
2019-11-13 21:17:45 +01:00
|
|
|
cLock := c.NewNSLockFn(ctx, bucket, object)
|
2019-08-10 02:09:08 +02:00
|
|
|
if err := cLock.GetRLock(globalObjectTimeout); err != nil {
|
|
|
|
return nil, err
|
2018-03-28 23:14:06 +02:00
|
|
|
}
|
2019-08-10 02:09:08 +02:00
|
|
|
|
|
|
|
defer cLock.RUnlock()
|
|
|
|
return dcache.Get(ctx, bucket, object, rs, h, opts)
|
2018-03-28 23:14:06 +02:00
|
|
|
}
|
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
func (c *cacheObjects) stat(ctx context.Context, dcache *diskCache, bucket, object string) (oi ObjectInfo, err error) {
|
2019-11-13 21:17:45 +01:00
|
|
|
cLock := c.NewNSLockFn(ctx, bucket, object)
|
2019-08-10 02:09:08 +02:00
|
|
|
if err := cLock.GetRLock(globalObjectTimeout); err != nil {
|
|
|
|
return oi, err
|
2019-05-22 05:00:27 +02:00
|
|
|
}
|
2019-08-10 02:09:08 +02:00
|
|
|
|
|
|
|
defer cLock.RUnlock()
|
|
|
|
return dcache.Stat(ctx, bucket, object)
|
2019-05-22 23:54:15 +02:00
|
|
|
}
|
|
|
|
|
2019-12-08 22:58:04 +01:00
|
|
|
func (c *cacheObjects) statRange(ctx context.Context, dcache *diskCache, bucket, object string, rs *HTTPRangeSpec) (oi ObjectInfo, err error) {
|
|
|
|
cLock := c.NewNSLockFn(ctx, bucket, object)
|
|
|
|
if err := cLock.GetRLock(globalObjectTimeout); err != nil {
|
|
|
|
return oi, err
|
|
|
|
}
|
|
|
|
|
|
|
|
defer cLock.RUnlock()
|
|
|
|
oi, _, err = dcache.statRange(ctx, bucket, object, rs)
|
|
|
|
return oi, err
|
|
|
|
}
|
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
// DeleteObject clears cache entry if backend delete operation succeeds
|
|
|
|
func (c *cacheObjects) DeleteObject(ctx context.Context, bucket, object string) (err error) {
|
|
|
|
if err = c.DeleteObjectFn(ctx, bucket, object); err != nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if c.isCacheExclude(bucket, object) || c.skipCache() {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2019-10-16 03:35:41 +02:00
|
|
|
dcache, cerr := c.getCacheLoc(bucket, object)
|
2019-08-10 02:09:08 +02:00
|
|
|
if cerr != nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if dcache.Exists(ctx, bucket, object) {
|
|
|
|
c.delete(ctx, dcache, bucket, object)
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// DeleteObjects batch deletes objects in slice, and clears any cached entries
|
|
|
|
func (c *cacheObjects) DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) {
|
|
|
|
errs := make([]error, len(objects))
|
|
|
|
for idx, object := range objects {
|
|
|
|
errs[idx] = c.DeleteObject(ctx, bucket, object)
|
|
|
|
}
|
|
|
|
return errs, nil
|
2018-03-28 23:14:06 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// construct a metadata k-v map
|
2019-08-10 02:09:08 +02:00
|
|
|
func getMetadata(objInfo ObjectInfo) map[string]string {
|
2018-03-28 23:14:06 +02:00
|
|
|
metadata := make(map[string]string)
|
|
|
|
metadata["etag"] = objInfo.ETag
|
|
|
|
metadata["content-type"] = objInfo.ContentType
|
2019-08-10 02:09:08 +02:00
|
|
|
if objInfo.ContentEncoding != "" {
|
|
|
|
metadata["content-encoding"] = objInfo.ContentEncoding
|
|
|
|
}
|
|
|
|
if objInfo.Expires != timeSentinel {
|
|
|
|
metadata["expires"] = objInfo.Expires.Format(http.TimeFormat)
|
|
|
|
}
|
|
|
|
for k, v := range objInfo.UserDefined {
|
|
|
|
metadata[k] = v
|
2018-03-28 23:14:06 +02:00
|
|
|
}
|
|
|
|
return metadata
|
|
|
|
}
|
|
|
|
|
2019-12-07 18:44:33 +01:00
|
|
|
// marks cache hit
|
|
|
|
func (c *cacheObjects) incCacheStats(size int64) {
|
|
|
|
c.cacheStats.incHit()
|
|
|
|
c.cacheStats.incBytesServed(size)
|
|
|
|
}
|
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
|
|
|
|
if c.isCacheExclude(bucket, object) || c.skipCache() {
|
2019-04-09 23:49:45 +02:00
|
|
|
return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
|
2018-09-21 04:22:09 +02:00
|
|
|
}
|
2019-08-10 02:09:08 +02:00
|
|
|
var cc cacheControl
|
2019-12-07 18:44:33 +01:00
|
|
|
var cacheObjSize int64
|
2019-08-10 02:09:08 +02:00
|
|
|
// fetch diskCache if object is currently cached or nearest available cache drive
|
|
|
|
dcache, err := c.getCacheToLoc(ctx, bucket, object)
|
2018-09-21 04:22:09 +02:00
|
|
|
if err != nil {
|
2019-04-09 23:49:45 +02:00
|
|
|
return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
|
2018-09-21 04:22:09 +02:00
|
|
|
}
|
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
cacheReader, cacheErr := c.get(ctx, dcache, bucket, object, rs, h, opts)
|
|
|
|
if cacheErr == nil {
|
2019-12-07 18:44:33 +01:00
|
|
|
cacheObjSize = cacheReader.ObjInfo.Size
|
|
|
|
if rs != nil {
|
|
|
|
if _, len, err := rs.GetOffsetLength(cacheObjSize); err == nil {
|
|
|
|
cacheObjSize = len
|
|
|
|
}
|
|
|
|
}
|
2019-08-10 02:09:08 +02:00
|
|
|
cc = cacheControlOpts(cacheReader.ObjInfo)
|
2019-12-07 03:19:36 +01:00
|
|
|
if (!cc.isEmpty() && !cc.isStale(cacheReader.ObjInfo.ModTime)) ||
|
|
|
|
cc.onlyIfCached {
|
2019-12-06 08:16:06 +01:00
|
|
|
// This is a cache hit, mark it so
|
2019-12-08 22:58:04 +01:00
|
|
|
bytesServed := cacheReader.ObjInfo.Size
|
|
|
|
if rs != nil {
|
|
|
|
if _, len, err := rs.GetOffsetLength(bytesServed); err == nil {
|
|
|
|
bytesServed = len
|
|
|
|
}
|
|
|
|
}
|
|
|
|
c.cacheStats.incHit()
|
|
|
|
c.cacheStats.incBytesServed(bytesServed)
|
2019-08-10 02:09:08 +02:00
|
|
|
return cacheReader, nil
|
|
|
|
}
|
2019-12-07 03:19:36 +01:00
|
|
|
if cc.noStore {
|
|
|
|
c.cacheStats.incMiss()
|
|
|
|
return c.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
|
|
|
|
}
|
2019-08-10 02:09:08 +02:00
|
|
|
}
|
2019-05-22 05:00:27 +02:00
|
|
|
|
2019-05-22 23:54:15 +02:00
|
|
|
objInfo, err := c.GetObjectInfoFn(ctx, bucket, object, opts)
|
2018-10-17 19:57:12 +02:00
|
|
|
if backendDownError(err) && cacheErr == nil {
|
2019-12-07 18:44:33 +01:00
|
|
|
c.incCacheStats(cacheObjSize)
|
2018-10-17 19:57:12 +02:00
|
|
|
return cacheReader, nil
|
|
|
|
} else if err != nil {
|
2018-09-21 04:22:09 +02:00
|
|
|
if _, ok := err.(ObjectNotFound); ok {
|
2019-04-18 22:53:22 +02:00
|
|
|
if cacheErr == nil {
|
|
|
|
cacheReader.Close()
|
|
|
|
// Delete cached entry if backend object
|
|
|
|
// was deleted.
|
|
|
|
dcache.Delete(ctx, bucket, object)
|
|
|
|
}
|
2018-09-21 04:22:09 +02:00
|
|
|
}
|
2019-12-07 18:44:33 +01:00
|
|
|
c.cacheStats.incMiss()
|
2018-10-17 19:57:12 +02:00
|
|
|
return nil, err
|
2018-09-21 04:22:09 +02:00
|
|
|
}
|
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
if !objInfo.IsCacheable() {
|
2019-12-07 18:44:33 +01:00
|
|
|
c.cacheStats.incMiss()
|
2019-04-09 23:49:45 +02:00
|
|
|
return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
|
2018-09-21 04:22:09 +02:00
|
|
|
}
|
|
|
|
|
2018-10-17 19:57:12 +02:00
|
|
|
if cacheErr == nil {
|
2019-08-10 02:09:08 +02:00
|
|
|
// if ETag matches for stale cache entry, serve from cache
|
|
|
|
if cacheReader.ObjInfo.ETag == objInfo.ETag {
|
|
|
|
// Update metadata in case server-side copy might have changed object metadata
|
|
|
|
dcache.updateMetadataIfChanged(ctx, bucket, object, objInfo, cacheReader.ObjInfo)
|
2019-12-07 18:44:33 +01:00
|
|
|
c.incCacheStats(cacheObjSize)
|
2018-09-21 04:22:09 +02:00
|
|
|
return cacheReader, nil
|
|
|
|
}
|
2018-10-11 08:01:24 +02:00
|
|
|
cacheReader.Close()
|
2018-09-21 04:22:09 +02:00
|
|
|
// Object is stale, so delete from cache
|
2019-08-10 02:09:08 +02:00
|
|
|
c.delete(ctx, dcache, bucket, object)
|
2018-09-21 04:22:09 +02:00
|
|
|
}
|
|
|
|
|
2019-12-07 18:44:33 +01:00
|
|
|
// Reaching here implies cache miss
|
|
|
|
c.cacheStats.incMiss()
|
2018-09-21 04:22:09 +02:00
|
|
|
// Since we got here, we are serving the request from backend,
|
|
|
|
// and also adding the object to the cache.
|
2019-08-10 02:09:08 +02:00
|
|
|
if !dcache.diskUsageLow() {
|
|
|
|
select {
|
|
|
|
case dcache.purgeChan <- struct{}{}:
|
|
|
|
default:
|
|
|
|
}
|
2018-09-21 04:22:09 +02:00
|
|
|
}
|
2019-05-22 23:54:15 +02:00
|
|
|
if !dcache.diskAvailable(objInfo.Size) {
|
2019-04-09 23:49:45 +02:00
|
|
|
return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
|
2018-09-21 04:22:09 +02:00
|
|
|
}
|
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
if rs != nil {
|
|
|
|
go func() {
|
|
|
|
// fill cache in the background for range GET requests
|
2019-12-08 22:58:04 +01:00
|
|
|
bReader, bErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
|
2019-08-10 02:09:08 +02:00
|
|
|
if bErr != nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
defer bReader.Close()
|
2019-12-08 22:58:04 +01:00
|
|
|
oi, err := c.statRange(ctx, dcache, bucket, object, rs)
|
2019-08-10 02:09:08 +02:00
|
|
|
// avoid cache overwrite if another background routine filled cache
|
|
|
|
if err != nil || oi.ETag != bReader.ObjInfo.ETag {
|
2019-12-08 22:58:04 +01:00
|
|
|
c.put(ctx, dcache, bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)})
|
2019-08-10 02:09:08 +02:00
|
|
|
}
|
|
|
|
}()
|
|
|
|
return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
|
|
|
|
}
|
2019-04-09 23:49:45 +02:00
|
|
|
bkReader, bkErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
|
2018-09-21 04:22:09 +02:00
|
|
|
if bkErr != nil {
|
|
|
|
return nil, bkErr
|
|
|
|
}
|
|
|
|
// Initialize pipe.
|
|
|
|
pipeReader, pipeWriter := io.Pipe()
|
|
|
|
teeReader := io.TeeReader(bkReader, pipeWriter)
|
|
|
|
go func() {
|
2019-12-08 22:58:04 +01:00
|
|
|
putErr := c.put(ctx, dcache, bucket, object, io.LimitReader(pipeReader, bkReader.ObjInfo.Size), bkReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bkReader.ObjInfo)})
|
2018-09-21 04:22:09 +02:00
|
|
|
// close the write end of the pipe, so the error gets
|
|
|
|
// propagated to getObjReader
|
|
|
|
pipeWriter.CloseWithError(putErr)
|
|
|
|
}()
|
|
|
|
cleanupBackend := func() { bkReader.Close() }
|
2019-09-05 20:03:32 +02:00
|
|
|
cleanupPipe := func() { pipeWriter.Close() }
|
2019-03-06 21:38:41 +01:00
|
|
|
return NewGetObjectReaderFromReader(teeReader, bkReader.ObjInfo, opts.CheckCopyPrecondFn, cleanupBackend, cleanupPipe)
|
2018-09-21 04:22:09 +02:00
|
|
|
}
|
|
|
|
|
2018-03-28 23:14:06 +02:00
|
|
|
// Returns ObjectInfo from cache if available.
|
2019-08-10 02:09:08 +02:00
|
|
|
func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
|
2018-03-28 23:14:06 +02:00
|
|
|
getObjectInfoFn := c.GetObjectInfoFn
|
2019-08-10 02:09:08 +02:00
|
|
|
|
|
|
|
if c.isCacheExclude(bucket, object) || c.skipCache() {
|
2018-09-10 18:42:43 +02:00
|
|
|
return getObjectInfoFn(ctx, bucket, object, opts)
|
2018-03-28 23:14:06 +02:00
|
|
|
}
|
2019-08-10 02:09:08 +02:00
|
|
|
|
|
|
|
// fetch diskCache if object is currently cached or nearest available cache drive
|
|
|
|
dcache, err := c.getCacheToLoc(ctx, bucket, object)
|
2018-03-28 23:14:06 +02:00
|
|
|
if err != nil {
|
2018-09-10 18:42:43 +02:00
|
|
|
return getObjectInfoFn(ctx, bucket, object, opts)
|
2018-03-28 23:14:06 +02:00
|
|
|
}
|
2019-08-10 02:09:08 +02:00
|
|
|
var cc cacheControl
|
|
|
|
// if cache control setting is valid, avoid HEAD operation to backend
|
|
|
|
cachedObjInfo, cerr := c.stat(ctx, dcache, bucket, object)
|
|
|
|
if cerr == nil {
|
|
|
|
cc = cacheControlOpts(cachedObjInfo)
|
2019-12-06 08:16:06 +01:00
|
|
|
if !cc.isStale(cachedObjInfo.ModTime) {
|
|
|
|
// This is a cache hit, mark it so
|
|
|
|
c.cacheStats.incHit()
|
2019-08-10 02:09:08 +02:00
|
|
|
return cachedObjInfo, nil
|
|
|
|
}
|
|
|
|
}
|
2019-12-06 08:16:06 +01:00
|
|
|
|
2018-09-10 18:42:43 +02:00
|
|
|
objInfo, err := getObjectInfoFn(ctx, bucket, object, opts)
|
2018-03-28 23:14:06 +02:00
|
|
|
if err != nil {
|
2018-04-10 18:36:37 +02:00
|
|
|
if _, ok := err.(ObjectNotFound); ok {
|
2018-03-28 23:14:06 +02:00
|
|
|
// Delete the cached entry if backend object was deleted.
|
2019-08-10 02:09:08 +02:00
|
|
|
c.delete(ctx, dcache, bucket, object)
|
2019-12-07 18:44:33 +01:00
|
|
|
c.cacheStats.incMiss()
|
2018-03-28 23:14:06 +02:00
|
|
|
return ObjectInfo{}, err
|
|
|
|
}
|
|
|
|
if !backendDownError(err) {
|
2019-12-07 18:44:33 +01:00
|
|
|
c.cacheStats.incMiss()
|
2018-03-28 23:14:06 +02:00
|
|
|
return ObjectInfo{}, err
|
|
|
|
}
|
|
|
|
if cerr == nil {
|
2019-12-07 18:44:33 +01:00
|
|
|
// This is a cache hit, mark it so
|
|
|
|
c.cacheStats.incHit()
|
2018-03-28 23:14:06 +02:00
|
|
|
return cachedObjInfo, nil
|
|
|
|
}
|
2019-12-07 18:44:33 +01:00
|
|
|
c.cacheStats.incMiss()
|
2018-03-28 23:14:06 +02:00
|
|
|
return ObjectInfo{}, BackendDown{}
|
|
|
|
}
|
2019-12-07 18:44:33 +01:00
|
|
|
// Reaching here implies cache miss
|
|
|
|
c.cacheStats.incMiss()
|
2018-03-28 23:14:06 +02:00
|
|
|
// when backend is up, do a sanity check on cached object
|
2019-08-10 02:09:08 +02:00
|
|
|
if cerr != nil {
|
2018-03-28 23:14:06 +02:00
|
|
|
return objInfo, nil
|
|
|
|
}
|
|
|
|
if cachedObjInfo.ETag != objInfo.ETag {
|
|
|
|
// Delete the cached entry if the backend object was replaced.
|
2019-08-10 02:09:08 +02:00
|
|
|
c.delete(ctx, dcache, bucket, object)
|
2018-03-28 23:14:06 +02:00
|
|
|
}
|
|
|
|
return objInfo, nil
|
|
|
|
}
|
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
// StorageInfo - returns underlying storage statistics.
|
|
|
|
func (c *cacheObjects) StorageInfo(ctx context.Context) (cInfo CacheStorageInfo) {
|
|
|
|
var total, free uint64
|
|
|
|
for _, cache := range c.cache {
|
2018-04-26 07:09:05 +02:00
|
|
|
if cache == nil {
|
|
|
|
continue
|
|
|
|
}
|
2019-08-10 02:09:08 +02:00
|
|
|
info, err := getDiskInfo(cache.dir)
|
|
|
|
logger.GetReqInfo(ctx).AppendTags("cachePath", cache.dir)
|
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
total += info.Total
|
|
|
|
free += info.Free
|
2019-05-22 23:54:15 +02:00
|
|
|
}
|
2019-08-10 02:09:08 +02:00
|
|
|
return CacheStorageInfo{
|
|
|
|
Total: total,
|
|
|
|
Free: free,
|
2019-05-22 23:54:15 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-12-06 08:16:06 +01:00
|
|
|
// CacheStats - returns underlying storage statistics.
|
|
|
|
func (c *cacheObjects) CacheStats() (cs *CacheStats) {
|
|
|
|
return c.cacheStats
|
|
|
|
}
|
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
// skipCache() returns true if cache migration is in progress
|
|
|
|
func (c *cacheObjects) skipCache() bool {
|
|
|
|
c.migMutex.Lock()
|
|
|
|
defer c.migMutex.Unlock()
|
|
|
|
return c.migrating
|
2019-05-13 21:25:49 +02:00
|
|
|
}
|
|
|
|
|
2018-03-28 23:14:06 +02:00
|
|
|
// Returns true if object should be excluded from cache
|
2019-08-10 02:09:08 +02:00
|
|
|
func (c *cacheObjects) isCacheExclude(bucket, object string) bool {
|
|
|
|
// exclude directories from cache
|
|
|
|
if strings.HasSuffix(object, SlashSeparator) {
|
|
|
|
return true
|
|
|
|
}
|
2018-03-28 23:14:06 +02:00
|
|
|
for _, pattern := range c.exclude {
|
|
|
|
matchStr := fmt.Sprintf("%s/%s", bucket, object)
|
|
|
|
if ok := wildcard.MatchSimple(pattern, matchStr); ok {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
// choose a cache deterministically based on hash of bucket,object. The hash index is treated as
|
|
|
|
// a hint. In the event that the cache drive at hash index is offline, treat the list of cache drives
|
|
|
|
// as a circular buffer and walk through them starting at hash index until an online drive is found.
|
2019-10-16 03:35:41 +02:00
|
|
|
func (c *cacheObjects) getCacheLoc(bucket, object string) (*diskCache, error) {
|
2019-08-10 02:09:08 +02:00
|
|
|
index := c.hashIndex(bucket, object)
|
|
|
|
numDisks := len(c.cache)
|
|
|
|
for k := 0; k < numDisks; k++ {
|
|
|
|
i := (index + k) % numDisks
|
|
|
|
if c.cache[i] == nil {
|
|
|
|
continue
|
2018-03-28 23:14:06 +02:00
|
|
|
}
|
2019-08-10 02:09:08 +02:00
|
|
|
if c.cache[i].IsOnline() {
|
|
|
|
return c.cache[i], nil
|
2018-03-28 23:14:06 +02:00
|
|
|
}
|
2019-05-22 23:54:15 +02:00
|
|
|
}
|
2019-08-10 02:09:08 +02:00
|
|
|
return nil, errDiskNotFound
|
2018-03-28 23:14:06 +02:00
|
|
|
}
|
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
// get cache disk where object is currently cached for a GET operation. If object does not exist at that location,
|
|
|
|
// treat the list of cache drives as a circular buffer and walk through them starting at hash index
|
|
|
|
// until an online drive is found.If object is not found, fall back to the first online cache drive
|
|
|
|
// closest to the hash index, so that object can be re-cached.
|
|
|
|
func (c *cacheObjects) getCacheToLoc(ctx context.Context, bucket, object string) (*diskCache, error) {
|
|
|
|
index := c.hashIndex(bucket, object)
|
2019-05-22 23:54:15 +02:00
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
numDisks := len(c.cache)
|
|
|
|
// save first online cache disk closest to the hint index
|
|
|
|
var firstOnlineDisk *diskCache
|
|
|
|
for k := 0; k < numDisks; k++ {
|
|
|
|
i := (index + k) % numDisks
|
|
|
|
if c.cache[i] == nil {
|
|
|
|
continue
|
2019-05-22 23:54:15 +02:00
|
|
|
}
|
2019-08-10 02:09:08 +02:00
|
|
|
if c.cache[i].IsOnline() {
|
|
|
|
if firstOnlineDisk == nil {
|
|
|
|
firstOnlineDisk = c.cache[i]
|
|
|
|
}
|
|
|
|
if c.cache[i].Exists(ctx, bucket, object) {
|
|
|
|
return c.cache[i], nil
|
|
|
|
}
|
2019-05-22 23:54:15 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
if firstOnlineDisk != nil {
|
|
|
|
return firstOnlineDisk, nil
|
2019-05-22 23:54:15 +02:00
|
|
|
}
|
2019-08-10 02:09:08 +02:00
|
|
|
return nil, errDiskNotFound
|
2019-05-22 23:54:15 +02:00
|
|
|
}
|
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
// Compute a unique hash sum for bucket and object
|
|
|
|
func (c *cacheObjects) hashIndex(bucket, object string) int {
|
|
|
|
return crcHashMod(pathJoin(bucket, object), len(c.cache))
|
2018-03-28 23:14:06 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// newCache initializes the cacheFSObjects for the "drives" specified in config.json
|
|
|
|
// or the global env overrides.
|
2019-10-04 19:35:33 +02:00
|
|
|
func newCache(config cache.Config) ([]*diskCache, bool, error) {
|
2019-08-10 02:09:08 +02:00
|
|
|
var caches []*diskCache
|
2018-04-06 00:04:40 +02:00
|
|
|
ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{})
|
2019-08-10 02:09:08 +02:00
|
|
|
formats, migrating, err := loadAndValidateCacheFormat(ctx, config.Drives)
|
2018-03-28 23:14:06 +02:00
|
|
|
if err != nil {
|
2019-08-10 02:09:08 +02:00
|
|
|
return nil, false, err
|
2018-03-28 23:14:06 +02:00
|
|
|
}
|
2018-03-29 23:38:26 +02:00
|
|
|
for i, dir := range config.Drives {
|
2019-08-10 02:09:08 +02:00
|
|
|
// skip diskCache creation for cache drives missing a format.json
|
2018-03-28 23:14:06 +02:00
|
|
|
if formats[i] == nil {
|
2019-08-10 02:09:08 +02:00
|
|
|
caches = append(caches, nil)
|
2018-03-28 23:14:06 +02:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
if err := checkAtimeSupport(dir); err != nil {
|
2019-08-10 02:09:08 +02:00
|
|
|
return nil, false, errors.New("Atime support required for disk caching")
|
2018-03-28 23:14:06 +02:00
|
|
|
}
|
2019-08-10 02:09:08 +02:00
|
|
|
|
2019-10-23 07:59:13 +02:00
|
|
|
quota := config.MaxUse
|
|
|
|
if quota == 0 {
|
|
|
|
quota = config.Quota
|
|
|
|
}
|
|
|
|
|
|
|
|
cache, err := newDiskCache(dir, config.Expiry, quota)
|
2018-03-29 23:38:26 +02:00
|
|
|
if err != nil {
|
2019-08-10 02:09:08 +02:00
|
|
|
return nil, false, err
|
|
|
|
}
|
|
|
|
// Start the purging go-routine for entries that have expired if no migration in progress
|
|
|
|
if !migrating {
|
|
|
|
go cache.purge()
|
2018-03-29 23:38:26 +02:00
|
|
|
}
|
2019-05-22 23:54:15 +02:00
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
caches = append(caches, cache)
|
2018-03-28 23:14:06 +02:00
|
|
|
}
|
2019-08-10 02:09:08 +02:00
|
|
|
return caches, migrating, nil
|
2018-03-28 23:14:06 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Return error if Atime is disabled on the O/S
|
|
|
|
func checkAtimeSupport(dir string) (err error) {
|
|
|
|
file, err := ioutil.TempFile(dir, "prefix")
|
|
|
|
if err != nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
defer os.Remove(file.Name())
|
|
|
|
finfo1, err := os.Stat(file.Name())
|
|
|
|
if err != nil {
|
|
|
|
return
|
|
|
|
}
|
2019-07-11 00:41:11 +02:00
|
|
|
// add a sleep to ensure atime change is detected
|
|
|
|
time.Sleep(10 * time.Millisecond)
|
|
|
|
|
|
|
|
if _, err = io.Copy(ioutil.Discard, file); err != nil {
|
2018-03-28 23:14:06 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
finfo2, err := os.Stat(file.Name())
|
|
|
|
|
|
|
|
if atime.Get(finfo2).Equal(atime.Get(finfo1)) {
|
|
|
|
return errors.New("Atime not supported")
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
2019-08-10 02:09:08 +02:00
|
|
|
func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
|
2019-10-04 19:35:33 +02:00
|
|
|
logStartupMessage(color.Blue("Cache migration initiated ...."))
|
2019-08-23 19:13:22 +02:00
|
|
|
|
2019-10-14 18:44:51 +02:00
|
|
|
g := errgroup.WithNErrs(len(c.cache))
|
|
|
|
for index, dc := range c.cache {
|
2019-08-10 02:09:08 +02:00
|
|
|
if dc == nil {
|
|
|
|
continue
|
|
|
|
}
|
2019-10-14 18:44:51 +02:00
|
|
|
index := index
|
|
|
|
g.Go(func() error {
|
|
|
|
// start migration from V1 to V2
|
|
|
|
return migrateOldCache(ctx, c.cache[index])
|
|
|
|
}, index)
|
2019-08-10 02:09:08 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
errCnt := 0
|
2019-10-14 18:44:51 +02:00
|
|
|
for index, err := range g.Wait() {
|
2019-08-10 02:09:08 +02:00
|
|
|
if err != nil {
|
|
|
|
errCnt++
|
2019-10-14 18:44:51 +02:00
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
continue
|
2019-08-10 02:09:08 +02:00
|
|
|
}
|
2019-10-14 18:44:51 +02:00
|
|
|
go c.cache[index].purge()
|
2019-08-10 02:09:08 +02:00
|
|
|
}
|
2019-10-14 18:44:51 +02:00
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
if errCnt > 0 {
|
|
|
|
return
|
|
|
|
}
|
2019-10-14 18:44:51 +02:00
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
// update migration status
|
|
|
|
c.migMutex.Lock()
|
|
|
|
defer c.migMutex.Unlock()
|
|
|
|
c.migrating = false
|
2019-10-04 19:35:33 +02:00
|
|
|
logStartupMessage(color.Blue("Cache migration completed successfully."))
|
2019-08-10 02:09:08 +02:00
|
|
|
}
|
2018-03-28 23:14:06 +02:00
|
|
|
|
2019-09-05 16:20:16 +02:00
|
|
|
// PutObject - caches the uploaded object for single Put operations
|
|
|
|
func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
|
|
|
|
putObjectFn := c.PutObjectFn
|
|
|
|
dcache, err := c.getCacheToLoc(ctx, bucket, object)
|
|
|
|
if err != nil {
|
|
|
|
// disk cache could not be located,execute backend call.
|
|
|
|
return putObjectFn(ctx, bucket, object, r, opts)
|
|
|
|
}
|
|
|
|
size := r.Size()
|
|
|
|
if c.skipCache() {
|
|
|
|
return putObjectFn(ctx, bucket, object, r, opts)
|
|
|
|
}
|
|
|
|
|
|
|
|
// fetch from backend if there is no space on cache drive
|
|
|
|
if !dcache.diskAvailable(size) {
|
|
|
|
return putObjectFn(ctx, bucket, object, r, opts)
|
|
|
|
}
|
|
|
|
if opts.ServerSideEncryption != nil {
|
2019-12-13 00:11:27 +01:00
|
|
|
dcache.Delete(ctx, bucket, object)
|
2019-09-05 16:20:16 +02:00
|
|
|
return putObjectFn(ctx, bucket, object, r, opts)
|
|
|
|
}
|
2019-11-20 22:18:09 +01:00
|
|
|
|
|
|
|
// skip cache for objects with locks
|
|
|
|
objRetention := getObjectRetentionMeta(opts.UserDefined)
|
|
|
|
if objRetention.Mode == Governance || objRetention.Mode == Compliance {
|
2019-12-13 00:11:27 +01:00
|
|
|
dcache.Delete(ctx, bucket, object)
|
2019-11-20 22:18:09 +01:00
|
|
|
return putObjectFn(ctx, bucket, object, r, opts)
|
|
|
|
}
|
|
|
|
|
2019-09-05 16:20:16 +02:00
|
|
|
// fetch from backend if cache exclude pattern or cache-control
|
|
|
|
// directive set to exclude
|
|
|
|
if c.isCacheExclude(bucket, object) {
|
|
|
|
dcache.Delete(ctx, bucket, object)
|
|
|
|
return putObjectFn(ctx, bucket, object, r, opts)
|
|
|
|
}
|
|
|
|
|
2019-09-16 23:24:04 +02:00
|
|
|
objInfo, err = putObjectFn(ctx, bucket, object, r, opts)
|
2019-09-05 16:20:16 +02:00
|
|
|
|
2019-09-16 23:24:04 +02:00
|
|
|
if err == nil {
|
|
|
|
go func() {
|
|
|
|
// fill cache in the background
|
|
|
|
bReader, bErr := c.GetObjectNInfoFn(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{})
|
|
|
|
if bErr != nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
defer bReader.Close()
|
|
|
|
oi, err := c.stat(ctx, dcache, bucket, object)
|
|
|
|
// avoid cache overwrite if another background routine filled cache
|
|
|
|
if err != nil || oi.ETag != bReader.ObjInfo.ETag {
|
2019-12-08 22:58:04 +01:00
|
|
|
c.put(ctx, dcache, bucket, object, bReader, bReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)})
|
2019-09-16 23:24:04 +02:00
|
|
|
}
|
|
|
|
}()
|
2019-09-05 16:20:16 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
return objInfo, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2018-03-28 23:14:06 +02:00
|
|
|
// Returns cacheObjects for use by Server.
|
2019-10-04 19:35:33 +02:00
|
|
|
func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjectLayer, error) {
|
2018-03-28 23:14:06 +02:00
|
|
|
// list of disk caches for cache "drives" specified in config.json or MINIO_CACHE_DRIVES env var.
|
2019-08-10 02:09:08 +02:00
|
|
|
cache, migrateSw, err := newCache(config)
|
2018-03-28 23:14:06 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2019-05-22 23:54:15 +02:00
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
c := &cacheObjects{
|
2019-12-06 08:16:06 +01:00
|
|
|
cache: cache,
|
|
|
|
exclude: config.Exclude,
|
|
|
|
migrating: migrateSw,
|
|
|
|
migMutex: sync.Mutex{},
|
|
|
|
nsMutex: newNSLock(false),
|
|
|
|
cacheStats: newCacheStats(),
|
2018-09-10 18:42:43 +02:00
|
|
|
GetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
|
2019-11-09 18:27:23 +01:00
|
|
|
return newObjectLayerFn().GetObjectInfo(ctx, bucket, object, opts)
|
2018-03-28 23:14:06 +02:00
|
|
|
},
|
2018-10-11 08:01:24 +02:00
|
|
|
GetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
|
2019-11-09 18:27:23 +01:00
|
|
|
return newObjectLayerFn().GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
|
2018-10-11 08:01:24 +02:00
|
|
|
},
|
2018-03-28 23:14:06 +02:00
|
|
|
DeleteObjectFn: func(ctx context.Context, bucket, object string) error {
|
2019-11-09 18:27:23 +01:00
|
|
|
return newObjectLayerFn().DeleteObject(ctx, bucket, object)
|
2018-03-28 23:14:06 +02:00
|
|
|
},
|
2019-05-13 21:25:49 +02:00
|
|
|
DeleteObjectsFn: func(ctx context.Context, bucket string, objects []string) ([]error, error) {
|
|
|
|
errs := make([]error, len(objects))
|
|
|
|
for idx, object := range objects {
|
2019-11-09 18:27:23 +01:00
|
|
|
errs[idx] = newObjectLayerFn().DeleteObject(ctx, bucket, object)
|
2019-05-13 21:25:49 +02:00
|
|
|
}
|
|
|
|
return errs, nil
|
|
|
|
},
|
2019-09-05 16:20:16 +02:00
|
|
|
PutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
|
2019-11-09 18:27:23 +01:00
|
|
|
return newObjectLayerFn().PutObject(ctx, bucket, object, data, opts)
|
2019-09-05 16:20:16 +02:00
|
|
|
},
|
2019-05-22 23:54:15 +02:00
|
|
|
}
|
2019-11-16 22:44:28 +01:00
|
|
|
c.NewNSLockFn = func(ctx context.Context, bucket, object string) RWLocker {
|
|
|
|
return c.nsMutex.NewNSLock(ctx, nil, bucket, object)
|
|
|
|
}
|
|
|
|
|
2019-08-10 02:09:08 +02:00
|
|
|
if migrateSw {
|
|
|
|
go c.migrateCacheFromV1toV2(ctx)
|
2019-05-22 23:54:15 +02:00
|
|
|
}
|
|
|
|
return c, nil
|
|
|
|
}
|