2016-11-24 00:48:10 +01:00
|
|
|
/*
|
2017-01-18 21:24:34 +01:00
|
|
|
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
|
2016-11-24 00:48:10 +01:00
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
package cmd
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
2017-05-16 23:21:52 +02:00
|
|
|
"crypto/sha256"
|
|
|
|
"encoding/hex"
|
2017-08-11 20:38:46 +02:00
|
|
|
"errors"
|
2017-08-13 04:25:43 +02:00
|
|
|
"os"
|
2016-11-24 00:48:10 +01:00
|
|
|
"reflect"
|
|
|
|
"testing"
|
2016-12-31 02:08:02 +01:00
|
|
|
"time"
|
2016-11-24 00:48:10 +01:00
|
|
|
)
|
|
|
|
|
|
|
|
// Tests retry storage.
|
|
|
|
func TestRetryStorage(t *testing.T) {
|
2017-01-18 21:24:34 +01:00
|
|
|
root, err := newTestConfig(globalMinioDefaultRegion)
|
2016-11-24 00:48:10 +01:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2017-08-13 04:25:43 +02:00
|
|
|
defer os.RemoveAll(root)
|
2016-11-24 00:48:10 +01:00
|
|
|
|
|
|
|
originalStorageDisks, disks := prepareXLStorageDisks(t)
|
|
|
|
defer removeRoots(disks)
|
|
|
|
|
|
|
|
var storageDisks = make([]StorageAPI, len(originalStorageDisks))
|
|
|
|
for i := range originalStorageDisks {
|
|
|
|
retryDisk, ok := originalStorageDisks[i].(*retryStorage)
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("storage disk is not *retryStorage type")
|
|
|
|
}
|
2016-12-31 02:08:02 +01:00
|
|
|
storageDisks[i] = &retryStorage{
|
|
|
|
remoteStorage: newNaughtyDisk(retryDisk, map[int]error{
|
|
|
|
1: errDiskNotFound,
|
|
|
|
}, nil),
|
|
|
|
maxRetryAttempts: 1,
|
|
|
|
retryUnit: time.Millisecond,
|
|
|
|
retryCap: time.Millisecond * 10,
|
|
|
|
}
|
2016-11-24 00:48:10 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// Validate all the conditions for retrying calls.
|
|
|
|
|
|
|
|
storageDisks = make([]StorageAPI, len(originalStorageDisks))
|
|
|
|
for i := range originalStorageDisks {
|
|
|
|
retryDisk, ok := originalStorageDisks[i].(*retryStorage)
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("storage disk is not *retryStorage type")
|
|
|
|
}
|
2016-12-31 02:08:02 +01:00
|
|
|
storageDisks[i] = &retryStorage{
|
|
|
|
remoteStorage: newNaughtyDisk(retryDisk, map[int]error{
|
|
|
|
1: errDiskNotFound,
|
|
|
|
}, nil),
|
|
|
|
maxRetryAttempts: 1,
|
|
|
|
retryUnit: time.Millisecond,
|
|
|
|
retryCap: time.Millisecond * 10,
|
|
|
|
}
|
2016-11-24 00:48:10 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, disk := range storageDisks {
|
|
|
|
err = disk.Init()
|
2016-12-29 12:13:51 +01:00
|
|
|
if err != errDiskNotFound {
|
|
|
|
t.Fatal("Expected errDiskNotFound, got", err)
|
2016-11-24 00:48:10 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, disk := range storageDisks {
|
|
|
|
_, err = disk.DiskInfo()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
storageDisks = make([]StorageAPI, len(originalStorageDisks))
|
|
|
|
for i := range originalStorageDisks {
|
|
|
|
retryDisk, ok := originalStorageDisks[i].(*retryStorage)
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("storage disk is not *retryStorage type")
|
|
|
|
}
|
2016-12-31 02:08:02 +01:00
|
|
|
storageDisks[i] = &retryStorage{
|
|
|
|
remoteStorage: newNaughtyDisk(retryDisk, map[int]error{
|
|
|
|
1: errDiskNotFound,
|
|
|
|
}, nil),
|
|
|
|
maxRetryAttempts: 1,
|
|
|
|
retryUnit: time.Millisecond,
|
|
|
|
retryCap: time.Millisecond * 10,
|
|
|
|
}
|
2016-11-24 00:48:10 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, disk := range storageDisks {
|
|
|
|
if err = disk.MakeVol("existent"); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
if _, err = disk.StatVol("existent"); err == errVolumeNotFound {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
storageDisks = make([]StorageAPI, len(originalStorageDisks))
|
|
|
|
for i := range originalStorageDisks {
|
|
|
|
retryDisk, ok := originalStorageDisks[i].(*retryStorage)
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("storage disk is not *retryStorage type")
|
|
|
|
}
|
2016-12-31 02:08:02 +01:00
|
|
|
storageDisks[i] = &retryStorage{
|
|
|
|
remoteStorage: newNaughtyDisk(retryDisk, map[int]error{
|
|
|
|
1: errDiskNotFound,
|
|
|
|
}, nil),
|
|
|
|
maxRetryAttempts: 1,
|
|
|
|
retryUnit: time.Millisecond,
|
|
|
|
retryCap: time.Millisecond * 10,
|
|
|
|
}
|
2016-11-24 00:48:10 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, disk := range storageDisks {
|
|
|
|
if _, err = disk.StatVol("existent"); err == errVolumeNotFound {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
storageDisks = make([]StorageAPI, len(originalStorageDisks))
|
|
|
|
for i := range originalStorageDisks {
|
|
|
|
retryDisk, ok := originalStorageDisks[i].(*retryStorage)
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("storage disk is not *retryStorage type")
|
|
|
|
}
|
2016-12-31 02:08:02 +01:00
|
|
|
storageDisks[i] = &retryStorage{
|
|
|
|
remoteStorage: newNaughtyDisk(retryDisk, map[int]error{
|
|
|
|
1: errDiskNotFound,
|
|
|
|
}, nil),
|
|
|
|
maxRetryAttempts: 1,
|
|
|
|
retryUnit: time.Millisecond,
|
|
|
|
retryCap: time.Millisecond * 10,
|
|
|
|
}
|
2016-11-24 00:48:10 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, disk := range storageDisks {
|
|
|
|
if _, err = disk.ListVols(); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
storageDisks = make([]StorageAPI, len(originalStorageDisks))
|
|
|
|
for i := range originalStorageDisks {
|
|
|
|
retryDisk, ok := originalStorageDisks[i].(*retryStorage)
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("storage disk is not *retryStorage type")
|
|
|
|
}
|
2016-12-31 02:08:02 +01:00
|
|
|
storageDisks[i] = &retryStorage{
|
|
|
|
remoteStorage: newNaughtyDisk(retryDisk, map[int]error{
|
|
|
|
1: errDiskNotFound,
|
|
|
|
}, nil),
|
|
|
|
maxRetryAttempts: 1,
|
|
|
|
retryUnit: time.Millisecond,
|
|
|
|
retryCap: time.Millisecond * 10,
|
|
|
|
}
|
2016-11-24 00:48:10 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, disk := range storageDisks {
|
|
|
|
if err = disk.DeleteVol("existent"); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
if str := disk.String(); str == "" {
|
|
|
|
t.Fatal("String method for disk cannot be empty.")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
storageDisks = make([]StorageAPI, len(originalStorageDisks))
|
|
|
|
for i := range originalStorageDisks {
|
|
|
|
retryDisk, ok := originalStorageDisks[i].(*retryStorage)
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("storage disk is not *retryStorage type")
|
|
|
|
}
|
2016-12-31 02:08:02 +01:00
|
|
|
storageDisks[i] = &retryStorage{
|
|
|
|
remoteStorage: newNaughtyDisk(retryDisk, map[int]error{
|
|
|
|
1: errDiskNotFound,
|
|
|
|
}, nil),
|
|
|
|
maxRetryAttempts: 1,
|
|
|
|
retryUnit: time.Millisecond,
|
|
|
|
retryCap: time.Millisecond * 10,
|
|
|
|
}
|
2016-11-24 00:48:10 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, disk := range storageDisks {
|
|
|
|
if err = disk.MakeVol("existent"); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
storageDisks = make([]StorageAPI, len(originalStorageDisks))
|
|
|
|
for i := range originalStorageDisks {
|
|
|
|
retryDisk, ok := originalStorageDisks[i].(*retryStorage)
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("storage disk is not *retryStorage type")
|
|
|
|
}
|
2016-12-31 02:08:02 +01:00
|
|
|
storageDisks[i] = &retryStorage{
|
|
|
|
remoteStorage: newNaughtyDisk(retryDisk, map[int]error{
|
|
|
|
1: errDiskNotFound,
|
|
|
|
}, nil),
|
|
|
|
maxRetryAttempts: 1,
|
|
|
|
retryUnit: time.Millisecond,
|
|
|
|
retryCap: time.Millisecond * 10,
|
|
|
|
}
|
2016-11-24 00:48:10 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, disk := range storageDisks {
|
|
|
|
if err = disk.PrepareFile("existent", "path", 10); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
storageDisks = make([]StorageAPI, len(originalStorageDisks))
|
|
|
|
for i := range originalStorageDisks {
|
|
|
|
retryDisk, ok := originalStorageDisks[i].(*retryStorage)
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("storage disk is not *retryStorage type")
|
|
|
|
}
|
2016-12-31 02:08:02 +01:00
|
|
|
storageDisks[i] = &retryStorage{
|
|
|
|
remoteStorage: newNaughtyDisk(retryDisk, map[int]error{
|
|
|
|
1: errDiskNotFound,
|
|
|
|
}, nil),
|
|
|
|
maxRetryAttempts: 1,
|
|
|
|
retryUnit: time.Millisecond,
|
|
|
|
retryCap: time.Millisecond * 10,
|
|
|
|
}
|
2016-11-24 00:48:10 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, disk := range storageDisks {
|
|
|
|
if err = disk.AppendFile("existent", "path", []byte("Hello, World")); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
storageDisks = make([]StorageAPI, len(originalStorageDisks))
|
|
|
|
for i := range originalStorageDisks {
|
|
|
|
retryDisk, ok := originalStorageDisks[i].(*retryStorage)
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("storage disk is not *retryStorage type")
|
|
|
|
}
|
2016-12-31 02:08:02 +01:00
|
|
|
storageDisks[i] = &retryStorage{
|
|
|
|
remoteStorage: newNaughtyDisk(retryDisk, map[int]error{
|
|
|
|
1: errDiskNotFound,
|
|
|
|
}, nil),
|
|
|
|
maxRetryAttempts: 1,
|
|
|
|
retryUnit: time.Millisecond,
|
|
|
|
retryCap: time.Millisecond * 10,
|
|
|
|
}
|
2016-11-24 00:48:10 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, disk := range storageDisks {
|
|
|
|
var buf1 []byte
|
|
|
|
if buf1, err = disk.ReadAll("existent", "path"); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
if !bytes.Equal(buf1, []byte("Hello, World")) {
|
|
|
|
t.Fatalf("Expected `Hello, World`, got %s", string(buf1))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
storageDisks = make([]StorageAPI, len(originalStorageDisks))
|
|
|
|
for i := range originalStorageDisks {
|
|
|
|
retryDisk, ok := originalStorageDisks[i].(*retryStorage)
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("storage disk is not *retryStorage type")
|
|
|
|
}
|
2016-12-31 02:08:02 +01:00
|
|
|
storageDisks[i] = &retryStorage{
|
|
|
|
remoteStorage: newNaughtyDisk(retryDisk, map[int]error{
|
|
|
|
1: errDiskNotFound,
|
|
|
|
}, nil),
|
|
|
|
maxRetryAttempts: 1,
|
|
|
|
retryUnit: time.Millisecond,
|
|
|
|
retryCap: time.Millisecond * 10,
|
|
|
|
}
|
2016-11-24 00:48:10 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, disk := range storageDisks {
|
|
|
|
var buf2 = make([]byte, 5)
|
|
|
|
var n int64
|
|
|
|
if n, err = disk.ReadFile("existent", "path", 7, buf2); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2017-05-16 23:21:52 +02:00
|
|
|
if err != nil {
|
|
|
|
t.Error("Error in ReadFile", err)
|
|
|
|
}
|
|
|
|
if n != 5 {
|
|
|
|
t.Fatalf("Expected 5, got %d", n)
|
|
|
|
}
|
|
|
|
if !bytes.Equal(buf2, []byte("World")) {
|
|
|
|
t.Fatalf("Expected `World`, got %s", string(buf2))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
sha256Hash := func(s string) string {
|
|
|
|
k := sha256.Sum256([]byte(s))
|
|
|
|
return hex.EncodeToString(k[:])
|
|
|
|
}
|
|
|
|
for _, disk := range storageDisks {
|
|
|
|
var buf2 = make([]byte, 5)
|
|
|
|
var n int64
|
|
|
|
if n, err = disk.ReadFileWithVerify("existent", "path", 7, buf2,
|
|
|
|
HashSha256, sha256Hash("Hello, World")); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
t.Error("Error in ReadFileWithVerify", err)
|
|
|
|
}
|
2016-11-24 00:48:10 +01:00
|
|
|
if n != 5 {
|
|
|
|
t.Fatalf("Expected 5, got %d", n)
|
|
|
|
}
|
|
|
|
if !bytes.Equal(buf2, []byte("World")) {
|
|
|
|
t.Fatalf("Expected `World`, got %s", string(buf2))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
storageDisks = make([]StorageAPI, len(originalStorageDisks))
|
|
|
|
for i := range originalStorageDisks {
|
|
|
|
retryDisk, ok := originalStorageDisks[i].(*retryStorage)
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("storage disk is not *retryStorage type")
|
|
|
|
}
|
2016-12-31 02:08:02 +01:00
|
|
|
storageDisks[i] = &retryStorage{
|
|
|
|
remoteStorage: newNaughtyDisk(retryDisk, map[int]error{
|
|
|
|
1: errDiskNotFound,
|
|
|
|
}, nil),
|
|
|
|
maxRetryAttempts: 1,
|
|
|
|
retryUnit: time.Millisecond,
|
|
|
|
retryCap: time.Millisecond * 10,
|
|
|
|
}
|
2016-11-24 00:48:10 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, disk := range storageDisks {
|
|
|
|
if err = disk.RenameFile("existent", "path", "existent", "new-path"); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
if _, err = disk.StatFile("existent", "new-path"); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
storageDisks = make([]StorageAPI, len(originalStorageDisks))
|
|
|
|
for i := range originalStorageDisks {
|
|
|
|
retryDisk, ok := originalStorageDisks[i].(*retryStorage)
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("storage disk is not *retryStorage type")
|
|
|
|
}
|
2016-12-31 02:08:02 +01:00
|
|
|
storageDisks[i] = &retryStorage{
|
|
|
|
remoteStorage: newNaughtyDisk(retryDisk, map[int]error{
|
|
|
|
1: errDiskNotFound,
|
|
|
|
}, nil),
|
|
|
|
maxRetryAttempts: 1,
|
|
|
|
retryUnit: time.Millisecond,
|
|
|
|
retryCap: time.Millisecond * 10,
|
|
|
|
}
|
2016-11-24 00:48:10 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, disk := range storageDisks {
|
|
|
|
if _, err = disk.StatFile("existent", "new-path"); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
storageDisks = make([]StorageAPI, len(originalStorageDisks))
|
|
|
|
for i := range originalStorageDisks {
|
|
|
|
retryDisk, ok := originalStorageDisks[i].(*retryStorage)
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("storage disk is not *retryStorage type")
|
|
|
|
}
|
2016-12-31 02:08:02 +01:00
|
|
|
storageDisks[i] = &retryStorage{
|
|
|
|
remoteStorage: newNaughtyDisk(retryDisk, map[int]error{
|
|
|
|
1: errDiskNotFound,
|
|
|
|
}, nil),
|
|
|
|
maxRetryAttempts: 1,
|
|
|
|
retryUnit: time.Millisecond,
|
|
|
|
retryCap: time.Millisecond * 10,
|
|
|
|
}
|
2016-11-24 00:48:10 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, disk := range storageDisks {
|
|
|
|
var entries []string
|
|
|
|
if entries, err = disk.ListDir("existent", ""); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
if !reflect.DeepEqual(entries, []string{"new-path"}) {
|
|
|
|
t.Fatalf("Expected []string{\"new-path\"}, got %s", entries)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
storageDisks = make([]StorageAPI, len(originalStorageDisks))
|
|
|
|
for i := range originalStorageDisks {
|
|
|
|
retryDisk, ok := originalStorageDisks[i].(*retryStorage)
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("storage disk is not *retryStorage type")
|
|
|
|
}
|
2016-12-31 02:08:02 +01:00
|
|
|
storageDisks[i] = &retryStorage{
|
|
|
|
remoteStorage: newNaughtyDisk(retryDisk, map[int]error{
|
|
|
|
1: errDiskNotFound,
|
|
|
|
}, nil),
|
|
|
|
maxRetryAttempts: 1,
|
|
|
|
retryUnit: time.Millisecond,
|
|
|
|
retryCap: time.Millisecond * 10,
|
|
|
|
}
|
2016-11-24 00:48:10 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, disk := range storageDisks {
|
|
|
|
if err = disk.DeleteFile("existent", "new-path"); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
if err = disk.DeleteVol("existent"); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2017-08-11 20:38:46 +02:00
|
|
|
|
|
|
|
// Tests reply storage error transformation.
|
|
|
|
func TestReplyStorageErr(t *testing.T) {
|
|
|
|
unknownErr := errors.New("Unknown error")
|
|
|
|
testCases := []struct {
|
|
|
|
expectedErr error
|
|
|
|
err error
|
|
|
|
}{
|
|
|
|
{
|
|
|
|
expectedErr: errDiskNotFound,
|
|
|
|
err: errDiskNotFoundFromNetError,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
expectedErr: errDiskNotFound,
|
|
|
|
err: errDiskNotFoundFromRPCShutdown,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
expectedErr: unknownErr,
|
|
|
|
err: unknownErr,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
for i, testCase := range testCases {
|
|
|
|
resultErr := retryToStorageErr(testCase.err)
|
|
|
|
if testCase.expectedErr != resultErr {
|
|
|
|
t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.expectedErr, resultErr)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|