2016-04-30 02:52:17 +02:00
|
|
|
/*
|
|
|
|
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2016-08-19 01:23:42 +02:00
|
|
|
package cmd
|
2016-04-30 02:52:17 +02:00
|
|
|
|
2016-05-25 10:33:39 +02:00
|
|
|
import (
|
2016-07-31 23:11:14 +02:00
|
|
|
"net"
|
2016-05-25 10:33:39 +02:00
|
|
|
"strings"
|
|
|
|
"sync"
|
|
|
|
)
|
2016-04-30 02:52:17 +02:00
|
|
|
|
2016-05-30 00:38:14 +02:00
|
|
|
const (
|
|
|
|
// Block size used for all internal operations version 1.
|
|
|
|
blockSizeV1 = 10 * 1024 * 1024 // 10MiB.
|
2016-06-25 12:03:27 +02:00
|
|
|
|
|
|
|
// Staging buffer read size for all internal operations version 1.
|
|
|
|
readSizeV1 = 128 * 1024 // 128KiB.
|
2016-07-22 02:31:14 +02:00
|
|
|
|
|
|
|
// Buckets meta prefix.
|
|
|
|
bucketMetaPrefix = "buckets"
|
2016-05-30 00:38:14 +02:00
|
|
|
)
|
|
|
|
|
2016-07-08 07:10:27 +02:00
|
|
|
// isErrIgnored should we ignore this error?, takes a list of errors which can be ignored.
|
|
|
|
func isErrIgnored(err error, ignoredErrs []error) bool {
|
2016-08-25 18:39:01 +02:00
|
|
|
err = errorCause(err)
|
2016-07-08 07:10:27 +02:00
|
|
|
for _, ignoredErr := range ignoredErrs {
|
|
|
|
if ignoredErr == err {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2016-05-31 01:51:59 +02:00
|
|
|
// House keeping code needed for FS.
|
|
|
|
func fsHouseKeeping(storageDisk StorageAPI) error {
|
|
|
|
// Cleanup all temp entries upon start.
|
2016-07-02 10:59:28 +02:00
|
|
|
err := cleanupDir(storageDisk, minioMetaBucket, tmpMetaPrefix)
|
2016-05-31 01:51:59 +02:00
|
|
|
if err != nil {
|
2016-07-02 10:59:28 +02:00
|
|
|
return toObjectErr(err, minioMetaBucket, tmpMetaPrefix)
|
2016-05-31 01:51:59 +02:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-07-31 23:11:14 +02:00
|
|
|
// Check if a network path is local to this node.
|
|
|
|
func isLocalStorage(networkPath string) bool {
|
|
|
|
if idx := strings.LastIndex(networkPath, ":"); idx != -1 {
|
2016-08-15 23:33:48 +02:00
|
|
|
// e.g 10.0.0.1:/mnt/networkPath
|
|
|
|
netAddr, _, err := splitNetPath(networkPath)
|
2016-07-31 23:11:14 +02:00
|
|
|
if err != nil {
|
2016-08-15 23:33:48 +02:00
|
|
|
errorIf(err, "Splitting into ip and path failed")
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
// netAddr will only be set if this is not a local path.
|
|
|
|
if netAddr == "" {
|
|
|
|
return true
|
2016-07-31 23:11:14 +02:00
|
|
|
}
|
|
|
|
// Resolve host to address to check if the IP is loopback.
|
|
|
|
// If address resolution fails, assume it's a non-local host.
|
2016-08-15 23:33:48 +02:00
|
|
|
addrs, err := net.LookupHost(netAddr)
|
2016-07-31 23:11:14 +02:00
|
|
|
if err != nil {
|
2016-08-15 23:33:48 +02:00
|
|
|
errorIf(err, "Failed to lookup host")
|
2016-07-31 23:11:14 +02:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
for _, addr := range addrs {
|
|
|
|
if ip := net.ParseIP(addr); ip.IsLoopback() {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
iaddrs, err := net.InterfaceAddrs()
|
|
|
|
if err != nil {
|
2016-08-15 23:33:48 +02:00
|
|
|
errorIf(err, "Unable to list interface addresses")
|
2016-07-31 23:11:14 +02:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
for _, addr := range addrs {
|
|
|
|
for _, iaddr := range iaddrs {
|
|
|
|
ip, _, err := net.ParseCIDR(iaddr.String())
|
|
|
|
if err != nil {
|
2016-08-15 23:33:48 +02:00
|
|
|
errorIf(err, "Unable to parse CIDR")
|
2016-07-31 23:11:14 +02:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
if ip.String() == addr {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2016-06-02 10:49:46 +02:00
|
|
|
// Depending on the disk type network or local, initialize storage API.
|
|
|
|
func newStorageAPI(disk string) (storage StorageAPI, err error) {
|
2016-07-31 23:11:14 +02:00
|
|
|
if isLocalStorage(disk) {
|
2016-06-02 10:49:46 +02:00
|
|
|
return newPosix(disk)
|
|
|
|
}
|
|
|
|
return newRPCClient(disk)
|
|
|
|
}
|
|
|
|
|
2016-07-02 10:59:28 +02:00
|
|
|
// Initializes meta volume on all input storage disks.
|
|
|
|
func initMetaVolume(storageDisks []StorageAPI) error {
|
2016-05-06 00:00:03 +02:00
|
|
|
// This happens for the first time, but keep this here since this
|
|
|
|
// is the only place where it can be made expensive optimizing all
|
|
|
|
// other calls. Create minio meta volume, if it doesn't exist yet.
|
2016-05-25 10:33:39 +02:00
|
|
|
var wg = &sync.WaitGroup{}
|
|
|
|
|
|
|
|
// Initialize errs to collect errors inside go-routine.
|
|
|
|
var errs = make([]error, len(storageDisks))
|
|
|
|
|
|
|
|
// Initialize all disks in parallel.
|
|
|
|
for index, disk := range storageDisks {
|
2016-06-03 01:34:15 +02:00
|
|
|
if disk == nil {
|
2016-07-10 23:38:15 +02:00
|
|
|
// Ignore create meta volume on disks which are not found.
|
2016-06-03 01:34:15 +02:00
|
|
|
continue
|
|
|
|
}
|
2016-05-25 10:33:39 +02:00
|
|
|
wg.Add(1)
|
|
|
|
go func(index int, disk StorageAPI) {
|
|
|
|
// Indicate this wait group is done.
|
|
|
|
defer wg.Done()
|
|
|
|
|
2016-09-07 05:31:50 +02:00
|
|
|
// Attempt to create `.minio.sys`.
|
2016-05-25 10:33:39 +02:00
|
|
|
err := disk.MakeVol(minioMetaBucket)
|
2016-07-02 10:59:28 +02:00
|
|
|
if err != nil {
|
|
|
|
switch err {
|
|
|
|
// Ignored errors.
|
|
|
|
case errVolumeExists, errDiskNotFound, errFaultyDisk:
|
|
|
|
default:
|
|
|
|
errs[index] = err
|
|
|
|
}
|
2016-05-20 11:22:22 +02:00
|
|
|
}
|
2016-07-02 10:59:28 +02:00
|
|
|
}(index, disk)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Wait for all cleanup to finish.
|
|
|
|
wg.Wait()
|
|
|
|
|
|
|
|
// Return upon first error.
|
|
|
|
for _, err := range errs {
|
|
|
|
if err == nil {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
return toObjectErr(err, minioMetaBucket)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Return success here.
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// House keeping code needed for XL.
|
|
|
|
func xlHouseKeeping(storageDisks []StorageAPI) error {
|
|
|
|
// This happens for the first time, but keep this here since this
|
|
|
|
// is the only place where it can be made expensive optimizing all
|
|
|
|
// other calls. Create metavolume.
|
|
|
|
var wg = &sync.WaitGroup{}
|
|
|
|
|
|
|
|
// Initialize errs to collect errors inside go-routine.
|
|
|
|
var errs = make([]error, len(storageDisks))
|
|
|
|
|
|
|
|
// Initialize all disks in parallel.
|
|
|
|
for index, disk := range storageDisks {
|
|
|
|
if disk == nil {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
wg.Add(1)
|
|
|
|
go func(index int, disk StorageAPI) {
|
|
|
|
// Indicate this wait group is done.
|
|
|
|
defer wg.Done()
|
|
|
|
|
2016-05-25 10:33:39 +02:00
|
|
|
// Cleanup all temp entries upon start.
|
2016-07-02 10:59:28 +02:00
|
|
|
err := cleanupDir(disk, minioMetaBucket, tmpMetaPrefix)
|
2016-09-07 05:31:50 +02:00
|
|
|
if err != nil {
|
|
|
|
switch errorCause(err) {
|
|
|
|
case errDiskNotFound, errVolumeNotFound:
|
|
|
|
default:
|
|
|
|
errs[index] = err
|
|
|
|
}
|
2016-05-25 10:33:39 +02:00
|
|
|
}
|
|
|
|
}(index, disk)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Wait for all cleanup to finish.
|
|
|
|
wg.Wait()
|
|
|
|
|
|
|
|
// Return upon first error.
|
|
|
|
for _, err := range errs {
|
|
|
|
if err == nil {
|
|
|
|
continue
|
2016-05-20 11:22:22 +02:00
|
|
|
}
|
2016-05-25 10:33:39 +02:00
|
|
|
return toObjectErr(err, minioMetaBucket, tmpMetaPrefix)
|
2016-05-06 00:00:03 +02:00
|
|
|
}
|
2016-05-25 10:33:39 +02:00
|
|
|
|
|
|
|
// Return success here.
|
2016-05-06 00:00:03 +02:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-05-08 19:15:34 +02:00
|
|
|
// Cleanup a directory recursively.
|
|
|
|
func cleanupDir(storage StorageAPI, volume, dirPath string) error {
|
|
|
|
var delFunc func(string) error
|
|
|
|
// Function to delete entries recursively.
|
|
|
|
delFunc = func(entryPath string) error {
|
|
|
|
if !strings.HasSuffix(entryPath, slashSeparator) {
|
2016-07-12 10:01:47 +02:00
|
|
|
// Delete the file entry.
|
2016-09-15 01:41:39 +02:00
|
|
|
err := storage.DeleteFile(volume, entryPath)
|
|
|
|
return traceError(err)
|
2016-05-08 19:15:34 +02:00
|
|
|
}
|
2016-07-12 10:01:47 +02:00
|
|
|
|
2016-05-08 19:15:34 +02:00
|
|
|
// If it's a directory, list and call delFunc() for each entry.
|
|
|
|
entries, err := storage.ListDir(volume, entryPath)
|
2016-07-12 10:01:47 +02:00
|
|
|
// If entryPath prefix never existed, safe to ignore.
|
|
|
|
if err == errFileNotFound {
|
|
|
|
return nil
|
|
|
|
} else if err != nil { // For any other errors fail.
|
2016-08-25 18:39:01 +02:00
|
|
|
return traceError(err)
|
2016-07-12 10:01:47 +02:00
|
|
|
} // else on success..
|
2016-06-14 10:39:40 +02:00
|
|
|
|
2016-07-12 10:01:47 +02:00
|
|
|
// Recurse and delete all other entries.
|
|
|
|
for _, entry := range entries {
|
|
|
|
if err = delFunc(pathJoin(entryPath, entry)); err != nil {
|
2016-05-08 19:15:34 +02:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
2016-06-03 01:34:15 +02:00
|
|
|
err := delFunc(retainSlash(pathJoin(dirPath)))
|
|
|
|
return err
|
2016-05-08 19:15:34 +02:00
|
|
|
}
|
2016-09-22 01:38:50 +02:00
|
|
|
|
|
|
|
// Checks whether bucket exists.
|
|
|
|
func isBucketExist(bucket string, obj ObjectLayer) error {
|
|
|
|
if !IsValidBucketName(bucket) {
|
|
|
|
return BucketNameInvalid{Bucket: bucket}
|
|
|
|
}
|
|
|
|
_, err := obj.GetBucketInfo(bucket)
|
|
|
|
if err != nil {
|
|
|
|
return BucketNotFound{Bucket: bucket}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|