2016-05-21 05:48:47 +02:00
|
|
|
/*
|
|
|
|
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2016-08-19 01:23:42 +02:00
|
|
|
package cmd
|
2016-05-21 05:48:47 +02:00
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
2016-11-12 01:36:07 +01:00
|
|
|
"os"
|
2016-12-09 05:35:07 +01:00
|
|
|
"runtime/debug"
|
2016-05-26 23:13:10 +02:00
|
|
|
"sort"
|
2016-11-12 01:36:07 +01:00
|
|
|
"strings"
|
2016-11-17 01:42:23 +01:00
|
|
|
"sync"
|
2016-05-26 23:13:10 +02:00
|
|
|
|
2016-11-23 03:18:22 +01:00
|
|
|
humanize "github.com/dustin/go-humanize"
|
2016-05-26 23:13:10 +02:00
|
|
|
"github.com/minio/minio/pkg/disk"
|
2016-07-04 01:58:21 +02:00
|
|
|
"github.com/minio/minio/pkg/objcache"
|
2016-05-21 05:48:47 +02:00
|
|
|
)
|
|
|
|
|
2016-05-31 01:51:59 +02:00
|
|
|
// XL constants.
|
2016-05-21 05:48:47 +02:00
|
|
|
const (
|
2016-05-31 01:51:59 +02:00
|
|
|
// Format config file carries backend format specific details.
|
2016-05-21 05:48:47 +02:00
|
|
|
formatConfigFile = "format.json"
|
2016-06-23 02:18:31 +02:00
|
|
|
|
|
|
|
// Format config tmp file carries backend format.
|
|
|
|
formatConfigFileTmp = "format.json.tmp"
|
|
|
|
|
2016-05-31 01:51:59 +02:00
|
|
|
// XL metadata file carries per object metadata.
|
|
|
|
xlMetaJSONFile = "xl.json"
|
2016-06-23 02:18:31 +02:00
|
|
|
|
2016-05-31 01:51:59 +02:00
|
|
|
// Uploads metadata file carries per multipart object metadata.
|
|
|
|
uploadsJSONFile = "uploads.json"
|
2016-07-04 01:58:21 +02:00
|
|
|
|
2016-12-09 05:35:07 +01:00
|
|
|
// Represents the minimum required RAM size before
|
|
|
|
// we enable caching.
|
|
|
|
minRAMSize = 8 * humanize.GiByte
|
2016-07-04 01:58:21 +02:00
|
|
|
|
|
|
|
// Maximum erasure blocks.
|
|
|
|
maxErasureBlocks = 16
|
|
|
|
|
|
|
|
// Minimum erasure blocks.
|
2016-07-20 04:24:32 +02:00
|
|
|
minErasureBlocks = 4
|
2016-05-21 05:48:47 +02:00
|
|
|
)
|
|
|
|
|
2016-05-31 01:51:59 +02:00
|
|
|
// xlObjects - Implements XL object layer.
|
2016-05-21 05:48:47 +02:00
|
|
|
type xlObjects struct {
|
2016-11-17 01:42:23 +01:00
|
|
|
mutex *sync.Mutex
|
2016-08-26 02:16:34 +02:00
|
|
|
storageDisks []StorageAPI // Collection of initialized backend disks.
|
|
|
|
dataBlocks int // dataBlocks count caculated for erasure.
|
|
|
|
parityBlocks int // parityBlocks count calculated for erasure.
|
|
|
|
readQuorum int // readQuorum minimum required disks to read data.
|
|
|
|
writeQuorum int // writeQuorum minimum required disks to write data.
|
2016-05-31 01:51:59 +02:00
|
|
|
|
2016-07-04 01:58:21 +02:00
|
|
|
// ListObjects pool management.
|
2016-06-05 20:55:45 +02:00
|
|
|
listPool *treeWalkPool
|
2016-05-21 05:48:47 +02:00
|
|
|
|
2016-07-04 01:58:21 +02:00
|
|
|
// Object cache for caching objects.
|
|
|
|
objCache *objcache.Cache
|
2016-05-21 05:48:47 +02:00
|
|
|
|
2016-07-04 01:58:21 +02:00
|
|
|
// Object cache enabled.
|
|
|
|
objCacheEnabled bool
|
|
|
|
}
|
2016-05-21 05:48:47 +02:00
|
|
|
|
2016-09-15 22:43:40 +02:00
|
|
|
// list of all errors that can be ignored in tree walk operation in XL
|
2016-11-24 05:05:04 +01:00
|
|
|
var xlTreeWalkIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied, errVolumeNotFound, errFileNotFound)
|
2016-09-15 22:43:40 +02:00
|
|
|
|
2017-01-17 02:05:00 +01:00
|
|
|
// newXLObjectLayer - initialize any object layer depending on the number of disks.
|
|
|
|
func newXLObjectLayer(storageDisks []StorageAPI) (ObjectLayer, error) {
|
|
|
|
// Initialize XL object layer.
|
|
|
|
objAPI, err := newXLObjects(storageDisks)
|
|
|
|
fatalIf(err, "Unable to initialize XL object layer.")
|
|
|
|
|
|
|
|
// Initialize and load bucket policies.
|
|
|
|
err = initBucketPolicies(objAPI)
|
|
|
|
fatalIf(err, "Unable to load all bucket policies.")
|
|
|
|
|
|
|
|
// Initialize a new event notifier.
|
|
|
|
err = initEventNotifier(objAPI)
|
|
|
|
fatalIf(err, "Unable to initialize event notification.")
|
|
|
|
|
|
|
|
// Success.
|
|
|
|
return objAPI, nil
|
|
|
|
}
|
|
|
|
|
2016-08-29 04:31:59 +02:00
|
|
|
// newXLObjects - initialize new xl object layer.
|
2016-10-05 21:48:07 +02:00
|
|
|
func newXLObjects(storageDisks []StorageAPI) (ObjectLayer, error) {
|
|
|
|
if storageDisks == nil {
|
2016-08-29 04:31:59 +02:00
|
|
|
return nil, errInvalidArgument
|
2016-06-03 01:34:15 +02:00
|
|
|
}
|
|
|
|
|
2016-10-15 04:57:40 +02:00
|
|
|
readQuorum := len(storageDisks) / 2
|
|
|
|
writeQuorum := len(storageDisks)/2 + 1
|
|
|
|
|
2016-05-21 05:48:47 +02:00
|
|
|
// Load saved XL format.json and validate.
|
2016-10-15 04:57:40 +02:00
|
|
|
newStorageDisks, err := loadFormatXL(storageDisks, readQuorum)
|
2016-05-21 05:48:47 +02:00
|
|
|
if err != nil {
|
2016-06-03 01:34:15 +02:00
|
|
|
return nil, fmt.Errorf("Unable to recognize backend format, %s", err)
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Calculate data and parity blocks.
|
2016-10-15 04:57:40 +02:00
|
|
|
dataBlocks, parityBlocks := len(newStorageDisks)/2, len(newStorageDisks)/2
|
2016-05-21 05:48:47 +02:00
|
|
|
|
2016-07-09 05:34:27 +02:00
|
|
|
// Initialize list pool.
|
|
|
|
listPool := newTreeWalkPool(globalLookupTimeout)
|
|
|
|
|
2016-11-12 01:36:07 +01:00
|
|
|
// Check if object cache is disabled.
|
|
|
|
objCacheDisabled := strings.EqualFold(os.Getenv("_MINIO_CACHE"), "off")
|
|
|
|
|
2016-06-02 01:43:31 +02:00
|
|
|
// Initialize xl objects.
|
2016-11-17 01:42:23 +01:00
|
|
|
xl := &xlObjects{
|
2016-12-09 05:35:07 +01:00
|
|
|
mutex: &sync.Mutex{},
|
|
|
|
storageDisks: newStorageDisks,
|
|
|
|
dataBlocks: dataBlocks,
|
|
|
|
parityBlocks: parityBlocks,
|
|
|
|
listPool: listPool,
|
|
|
|
}
|
|
|
|
|
|
|
|
// Object cache is enabled when _MINIO_CACHE env is missing.
|
|
|
|
// and cache size is > 0.
|
|
|
|
xl.objCacheEnabled = !objCacheDisabled && globalMaxCacheSize > 0
|
|
|
|
|
|
|
|
// Check if object cache is enabled.
|
|
|
|
if xl.objCacheEnabled {
|
|
|
|
// Initialize object cache.
|
|
|
|
objCache := objcache.New(globalMaxCacheSize, globalCacheExpiry)
|
|
|
|
objCache.OnEviction = func(key string) {
|
|
|
|
debug.FreeOSMemory()
|
|
|
|
}
|
|
|
|
xl.objCache = objCache
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
|
|
|
|
2016-11-26 03:11:50 +01:00
|
|
|
// Initialize meta volume, if volume already exists ignores it.
|
|
|
|
if err = initMetaVolume(storageDisks); err != nil {
|
|
|
|
return nil, fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err)
|
|
|
|
}
|
|
|
|
|
2016-05-21 05:48:47 +02:00
|
|
|
// Figure out read and write quorum based on number of storage disks.
|
2016-07-20 04:24:32 +02:00
|
|
|
// READ and WRITE quorum is always set to (N/2) number of disks.
|
2016-10-15 04:57:40 +02:00
|
|
|
xl.readQuorum = readQuorum
|
|
|
|
xl.writeQuorum = writeQuorum
|
2016-05-21 05:48:47 +02:00
|
|
|
|
2016-11-17 01:42:23 +01:00
|
|
|
// Do a quick heal on the buckets themselves for any discrepancies.
|
2016-11-21 10:47:26 +01:00
|
|
|
if err := quickHeal(xl.storageDisks, xl.writeQuorum, xl.readQuorum); err != nil {
|
2016-11-17 01:42:23 +01:00
|
|
|
return xl, err
|
|
|
|
}
|
|
|
|
|
2016-05-21 05:48:47 +02:00
|
|
|
// Return successfully initialized object layer.
|
|
|
|
return xl, nil
|
|
|
|
}
|
2016-05-26 23:13:10 +02:00
|
|
|
|
2016-08-15 08:55:48 +02:00
|
|
|
// Shutdown function for object storage interface.
|
|
|
|
func (xl xlObjects) Shutdown() error {
|
|
|
|
// Add any object layer shutdown activities here.
|
2017-01-23 09:32:55 +01:00
|
|
|
for _, disk := range xl.storageDisks {
|
|
|
|
// This closes storage rpc client connections if any.
|
|
|
|
// Otherwise this is a no-op.
|
|
|
|
if disk == nil {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
disk.Close()
|
|
|
|
}
|
2016-08-15 08:55:48 +02:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-05-26 23:13:10 +02:00
|
|
|
// byDiskTotal is a collection satisfying sort.Interface.
|
|
|
|
type byDiskTotal []disk.Info
|
|
|
|
|
|
|
|
func (d byDiskTotal) Len() int { return len(d) }
|
|
|
|
func (d byDiskTotal) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
|
|
|
|
func (d byDiskTotal) Less(i, j int) bool {
|
|
|
|
return d[i].Total < d[j].Total
|
|
|
|
}
|
|
|
|
|
2016-10-05 21:48:07 +02:00
|
|
|
// getDisksInfo - fetch disks info across all other storage API.
|
|
|
|
func getDisksInfo(disks []StorageAPI) (disksInfo []disk.Info, onlineDisks int, offlineDisks int) {
|
2016-10-17 23:31:33 +02:00
|
|
|
disksInfo = make([]disk.Info, len(disks))
|
|
|
|
for i, storageDisk := range disks {
|
2016-09-15 08:53:42 +02:00
|
|
|
if storageDisk == nil {
|
|
|
|
// Storage disk is empty, perhaps ignored disk or not available.
|
2016-10-05 21:48:07 +02:00
|
|
|
offlineDisks++
|
2016-09-15 08:53:42 +02:00
|
|
|
continue
|
|
|
|
}
|
2016-08-26 02:16:34 +02:00
|
|
|
info, err := storageDisk.DiskInfo()
|
2016-05-26 23:13:10 +02:00
|
|
|
if err != nil {
|
2016-08-26 02:16:34 +02:00
|
|
|
errorIf(err, "Unable to fetch disk info for %#v", storageDisk)
|
2017-01-02 19:43:56 +01:00
|
|
|
if isErr(err, baseErrs...) {
|
2016-10-05 21:48:07 +02:00
|
|
|
offlineDisks++
|
2017-01-02 19:43:56 +01:00
|
|
|
continue
|
2016-10-05 21:48:07 +02:00
|
|
|
}
|
2016-05-26 23:13:10 +02:00
|
|
|
}
|
2016-10-05 21:48:07 +02:00
|
|
|
onlineDisks++
|
2016-10-17 23:31:33 +02:00
|
|
|
disksInfo[i] = info
|
2016-05-26 23:13:10 +02:00
|
|
|
}
|
|
|
|
|
2016-10-05 21:48:07 +02:00
|
|
|
// Success.
|
|
|
|
return disksInfo, onlineDisks, offlineDisks
|
|
|
|
}
|
|
|
|
|
2016-10-17 23:31:33 +02:00
|
|
|
// returns sorted disksInfo slice which has only valid entries.
|
|
|
|
// i.e the entries where the total size of the disk is not stated
|
|
|
|
// as 0Bytes, this means that the disk is not online or ignored.
|
|
|
|
func sortValidDisksInfo(disksInfo []disk.Info) []disk.Info {
|
|
|
|
var validDisksInfo []disk.Info
|
|
|
|
for _, diskInfo := range disksInfo {
|
|
|
|
if diskInfo.Total == 0 {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
validDisksInfo = append(validDisksInfo, diskInfo)
|
|
|
|
}
|
|
|
|
sort.Sort(byDiskTotal(validDisksInfo))
|
|
|
|
return validDisksInfo
|
|
|
|
}
|
|
|
|
|
2016-10-05 21:48:07 +02:00
|
|
|
// Get an aggregated storage info across all disks.
|
|
|
|
func getStorageInfo(disks []StorageAPI) StorageInfo {
|
|
|
|
disksInfo, onlineDisks, offlineDisks := getDisksInfo(disks)
|
2016-10-17 23:31:33 +02:00
|
|
|
|
|
|
|
// Sort so that the first element is the smallest.
|
|
|
|
validDisksInfo := sortValidDisksInfo(disksInfo)
|
|
|
|
if len(validDisksInfo) == 0 {
|
2016-09-23 01:35:12 +02:00
|
|
|
return StorageInfo{
|
|
|
|
Total: -1,
|
|
|
|
Free: -1,
|
|
|
|
}
|
|
|
|
}
|
2016-10-17 23:31:33 +02:00
|
|
|
|
2016-05-26 23:13:10 +02:00
|
|
|
// Return calculated storage info, choose the lowest Total and
|
|
|
|
// Free as the total aggregated values. Total capacity is always
|
|
|
|
// the multiple of smallest disk among the disk list.
|
2016-10-05 21:48:07 +02:00
|
|
|
storageInfo := StorageInfo{
|
2016-10-17 23:31:33 +02:00
|
|
|
Total: validDisksInfo[0].Total * int64(onlineDisks) / 2,
|
|
|
|
Free: validDisksInfo[0].Free * int64(onlineDisks) / 2,
|
2016-05-26 23:13:10 +02:00
|
|
|
}
|
2016-10-17 23:31:33 +02:00
|
|
|
|
2016-10-05 21:48:07 +02:00
|
|
|
storageInfo.Backend.Type = XL
|
|
|
|
storageInfo.Backend.OnlineDisks = onlineDisks
|
|
|
|
storageInfo.Backend.OfflineDisks = offlineDisks
|
|
|
|
return storageInfo
|
|
|
|
}
|
|
|
|
|
|
|
|
// StorageInfo - returns underlying storage statistics.
|
|
|
|
func (xl xlObjects) StorageInfo() StorageInfo {
|
2016-10-10 08:03:10 +02:00
|
|
|
storageInfo := getStorageInfo(xl.storageDisks)
|
|
|
|
storageInfo.Backend.ReadQuorum = xl.readQuorum
|
|
|
|
storageInfo.Backend.WriteQuorum = xl.writeQuorum
|
|
|
|
return storageInfo
|
2016-05-26 23:13:10 +02:00
|
|
|
}
|