414 lines
12 KiB
Go
414 lines
12 KiB
Go
/*
|
|
* MinIO Cloud Storage, (C) 2015, 2016, 2017 MinIO, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package cmd
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"os"
|
|
"path"
|
|
"strconv"
|
|
"testing"
|
|
"time"
|
|
|
|
humanize "github.com/dustin/go-humanize"
|
|
)
|
|
|
|
const ActualSize = 1000
|
|
|
|
// Tests for reading XL object info.
|
|
func TestXLReadStat(t *testing.T) {
|
|
ExecObjectLayerDiskAlteredTest(t, testXLReadStat)
|
|
}
|
|
|
|
func testXLReadStat(obj ObjectLayer, instanceType string, disks []string, t *testing.T) {
|
|
// Setup for the tests.
|
|
bucketName := getRandomBucketName()
|
|
objectName := "test-object"
|
|
// create bucket.
|
|
err := obj.MakeBucketWithLocation(context.Background(), bucketName, "")
|
|
// Stop the test if creation of the bucket fails.
|
|
if err != nil {
|
|
t.Fatalf("%s : %s", instanceType, err.Error())
|
|
}
|
|
|
|
// set of byte data for PutObject.
|
|
// object has to be created before running tests for GetObject.
|
|
// this is required even to assert the GetObject data,
|
|
// since dataInserted === dataFetched back is a primary criteria for any object storage this assertion is critical.
|
|
bytesData := []struct {
|
|
byteData []byte
|
|
}{
|
|
{generateBytesData(6 * humanize.MiByte)},
|
|
}
|
|
// set of inputs for uploading the objects before tests for downloading is done.
|
|
putObjectInputs := []struct {
|
|
bucketName string
|
|
objectName string
|
|
contentLength int64
|
|
textData []byte
|
|
metaData map[string]string
|
|
}{
|
|
// case - 1.
|
|
{bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)},
|
|
}
|
|
// iterate through the above set of inputs and upkoad the object.
|
|
for i, input := range putObjectInputs {
|
|
// uploading the object.
|
|
_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), ObjectOptions{UserDefined: input.metaData})
|
|
// if object upload fails stop the test.
|
|
if err != nil {
|
|
t.Fatalf("Put Object case %d: Error uploading object: <ERROR> %v", i+1, err)
|
|
}
|
|
}
|
|
|
|
_, _, err = obj.(*xlObjects).readXLMetaStat(context.Background(), bucketName, objectName)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Remove one disk.
|
|
removeDiskN(disks, 7)
|
|
|
|
// Removing disk shouldn't affect reading object info.
|
|
_, _, err = obj.(*xlObjects).readXLMetaStat(context.Background(), bucketName, objectName)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
for _, disk := range disks {
|
|
os.RemoveAll(path.Join(disk, bucketName))
|
|
}
|
|
|
|
_, _, err = obj.(*xlObjects).readXLMetaStat(context.Background(), bucketName, objectName)
|
|
if err != errVolumeNotFound {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// Tests for reading XL meta parts.
|
|
func TestXLReadMetaParts(t *testing.T) {
|
|
ExecObjectLayerDiskAlteredTest(t, testXLReadMetaParts)
|
|
}
|
|
|
|
// testListObjectParts - Tests validate listing of object parts when disks go offline.
|
|
func testXLReadMetaParts(obj ObjectLayer, instanceType string, disks []string, t *testing.T) {
|
|
bucketNames := []string{"minio-bucket", "minio-2-bucket"}
|
|
objectNames := []string{"minio-object-1.txt"}
|
|
uploadIDs := []string{}
|
|
var opts ObjectOptions
|
|
|
|
// bucketnames[0].
|
|
// objectNames[0].
|
|
// uploadIds [0].
|
|
// Create bucket before intiating NewMultipartUpload.
|
|
err := obj.MakeBucketWithLocation(context.Background(), bucketNames[0], "")
|
|
if err != nil {
|
|
// Failed to create newbucket, abort.
|
|
t.Fatalf("%s : %s", instanceType, err.Error())
|
|
}
|
|
// Initiate Multipart Upload on the above created bucket.
|
|
uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], opts)
|
|
if err != nil {
|
|
// Failed to create NewMultipartUpload, abort.
|
|
t.Fatalf("%s : %s", instanceType, err.Error())
|
|
}
|
|
|
|
uploadIDs = append(uploadIDs, uploadID)
|
|
|
|
// Create multipart parts.
|
|
// Need parts to be uploaded before MultipartLists can be called and tested.
|
|
createPartCases := []struct {
|
|
bucketName string
|
|
objName string
|
|
uploadID string
|
|
PartID int
|
|
inputReaderData string
|
|
inputMd5 string
|
|
intputDataSize int64
|
|
expectedMd5 string
|
|
}{
|
|
// Case 1-4.
|
|
// Creating sequence of parts for same uploadID.
|
|
// Used to ensure that the ListMultipartResult produces one output for the four parts uploaded below for the given upload ID.
|
|
{bucketNames[0], objectNames[0], uploadIDs[0], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
|
|
{bucketNames[0], objectNames[0], uploadIDs[0], 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh")), "1f7690ebdd9b4caf8fab49ca1757bf27"},
|
|
{bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd")), "09a0877d04abf8759f99adec02baf579"},
|
|
{bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd")), "e132e96a5ddad6da8b07bba6f6131fef"},
|
|
}
|
|
sha256sum := ""
|
|
// Iterating over creatPartCases to generate multipart chunks.
|
|
for _, testCase := range createPartCases {
|
|
_, perr := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), opts)
|
|
if perr != nil {
|
|
t.Fatalf("%s : %s", instanceType, perr)
|
|
}
|
|
}
|
|
|
|
uploadIDPath := obj.(*xlObjects).getUploadIDDir(bucketNames[0], objectNames[0], uploadIDs[0])
|
|
|
|
_, _, err = obj.(*xlObjects).readXLMetaParts(context.Background(), minioMetaMultipartBucket, uploadIDPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Remove one disk.
|
|
removeDiskN(disks, 7)
|
|
|
|
// Removing disk shouldn't affect reading object parts info.
|
|
_, _, err = obj.(*xlObjects).readXLMetaParts(context.Background(), minioMetaMultipartBucket, uploadIDPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
for _, disk := range disks {
|
|
os.RemoveAll(path.Join(disk, bucketNames[0]))
|
|
os.RemoveAll(path.Join(disk, minioMetaMultipartBucket, obj.(*xlObjects).getMultipartSHADir(bucketNames[0], objectNames[0])))
|
|
}
|
|
|
|
_, _, err = obj.(*xlObjects).readXLMetaParts(context.Background(), minioMetaMultipartBucket, uploadIDPath)
|
|
if err != errFileNotFound {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// Test xlMetaV1.AddObjectPart()
|
|
func TestAddObjectPart(t *testing.T) {
|
|
testCases := []struct {
|
|
partNum int
|
|
expectedIndex int
|
|
}{
|
|
{1, 0},
|
|
{2, 1},
|
|
{4, 2},
|
|
{5, 3},
|
|
{7, 4},
|
|
// Insert part.
|
|
{3, 2},
|
|
// Replace existing part.
|
|
{4, 3},
|
|
// Missing part.
|
|
{6, -1},
|
|
}
|
|
|
|
// Setup.
|
|
xlMeta := newXLMetaV1("test-object", 8, 8)
|
|
if !xlMeta.IsValid() {
|
|
t.Fatalf("unable to get xl meta")
|
|
}
|
|
|
|
// Test them.
|
|
for _, testCase := range testCases {
|
|
if testCase.expectedIndex > -1 {
|
|
partNumString := strconv.Itoa(testCase.partNum)
|
|
xlMeta.AddObjectPart(testCase.partNum, "part."+partNumString, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize)
|
|
}
|
|
|
|
if index := objectPartIndex(xlMeta.Parts, testCase.partNum); index != testCase.expectedIndex {
|
|
t.Fatalf("%+v: expected = %d, got: %d", testCase, testCase.expectedIndex, index)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Test objectPartIndex().
|
|
// generates a sample xlMeta data and asserts the output of objectPartIndex() with the expected value.
|
|
func TestObjectPartIndex(t *testing.T) {
|
|
testCases := []struct {
|
|
partNum int
|
|
expectedIndex int
|
|
}{
|
|
{2, 1},
|
|
{1, 0},
|
|
{5, 3},
|
|
{4, 2},
|
|
{7, 4},
|
|
}
|
|
|
|
// Setup.
|
|
xlMeta := newXLMetaV1("test-object", 8, 8)
|
|
if !xlMeta.IsValid() {
|
|
t.Fatalf("unable to get xl meta")
|
|
}
|
|
|
|
// Add some parts for testing.
|
|
for _, testCase := range testCases {
|
|
partNumString := strconv.Itoa(testCase.partNum)
|
|
xlMeta.AddObjectPart(testCase.partNum, "part."+partNumString, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize)
|
|
}
|
|
|
|
// Add failure test case.
|
|
testCases = append(testCases, struct {
|
|
partNum int
|
|
expectedIndex int
|
|
}{6, -1})
|
|
|
|
// Test them.
|
|
for _, testCase := range testCases {
|
|
if index := objectPartIndex(xlMeta.Parts, testCase.partNum); index != testCase.expectedIndex {
|
|
t.Fatalf("%+v: expected = %d, got: %d", testCase, testCase.expectedIndex, index)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Test xlMetaV1.ObjectToPartOffset().
|
|
func TestObjectToPartOffset(t *testing.T) {
|
|
// Setup.
|
|
xlMeta := newXLMetaV1("test-object", 8, 8)
|
|
if !xlMeta.IsValid() {
|
|
t.Fatalf("unable to get xl meta")
|
|
}
|
|
|
|
// Add some parts for testing.
|
|
// Total size of all parts is 5,242,899 bytes.
|
|
for _, partNum := range []int{1, 2, 4, 5, 7} {
|
|
partNumString := strconv.Itoa(partNum)
|
|
xlMeta.AddObjectPart(partNum, "part."+partNumString, "etag."+partNumString, int64(partNum+humanize.MiByte), ActualSize)
|
|
}
|
|
|
|
testCases := []struct {
|
|
offset int64
|
|
expectedIndex int
|
|
expectedOffset int64
|
|
expectedErr error
|
|
}{
|
|
{0, 0, 0, nil},
|
|
{1 * humanize.MiByte, 0, 1 * humanize.MiByte, nil},
|
|
{1 + humanize.MiByte, 1, 0, nil},
|
|
{2 + humanize.MiByte, 1, 1, nil},
|
|
// Its valid for zero sized object.
|
|
{-1, 0, -1, nil},
|
|
// Max fffset is always (size - 1).
|
|
{(1 + 2 + 4 + 5 + 7) + (5 * humanize.MiByte) - 1, 4, 1048582, nil},
|
|
// Error if offset is size.
|
|
{(1 + 2 + 4 + 5 + 7) + (5 * humanize.MiByte), 0, 0, InvalidRange{}},
|
|
}
|
|
|
|
// Test them.
|
|
for _, testCase := range testCases {
|
|
index, offset, err := xlMeta.ObjectToPartOffset(context.Background(), testCase.offset)
|
|
if err != testCase.expectedErr {
|
|
t.Fatalf("%+v: expected = %s, got: %s", testCase, testCase.expectedErr, err)
|
|
}
|
|
if index != testCase.expectedIndex {
|
|
t.Fatalf("%+v: index: expected = %d, got: %d", testCase, testCase.expectedIndex, index)
|
|
}
|
|
if offset != testCase.expectedOffset {
|
|
t.Fatalf("%+v: offset: expected = %d, got: %d", testCase, testCase.expectedOffset, offset)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Helper function to check if two xlMetaV1 values are similar.
|
|
func isXLMetaSimilar(m, n xlMetaV1) bool {
|
|
if m.Version != n.Version {
|
|
return false
|
|
}
|
|
if m.Format != n.Format {
|
|
return false
|
|
}
|
|
if len(m.Parts) != len(n.Parts) {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func TestPickValidXLMeta(t *testing.T) {
|
|
obj := "object"
|
|
x1 := newXLMetaV1(obj, 4, 4)
|
|
now := UTCNow()
|
|
x1.Stat.ModTime = now
|
|
invalidX1 := x1
|
|
invalidX1.Version = "invalid-version"
|
|
xs := []xlMetaV1{x1, x1, x1, x1}
|
|
invalidXS := []xlMetaV1{invalidX1, invalidX1, invalidX1, invalidX1}
|
|
testCases := []struct {
|
|
metaArr []xlMetaV1
|
|
modTime time.Time
|
|
xlMeta xlMetaV1
|
|
expectedErr error
|
|
}{
|
|
{
|
|
metaArr: xs,
|
|
modTime: now,
|
|
xlMeta: x1,
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
metaArr: invalidXS,
|
|
modTime: now,
|
|
xlMeta: invalidX1,
|
|
expectedErr: errXLReadQuorum,
|
|
},
|
|
}
|
|
for i, test := range testCases {
|
|
xlMeta, err := pickValidXLMeta(context.Background(), test.metaArr, test.modTime, len(test.metaArr)/2)
|
|
if test.expectedErr != nil {
|
|
if err.Error() != test.expectedErr.Error() {
|
|
t.Errorf("Test %d: Expected to fail with %v but received %v",
|
|
i+1, test.expectedErr, err)
|
|
}
|
|
} else {
|
|
if !isXLMetaSimilar(xlMeta, test.xlMeta) {
|
|
t.Errorf("Test %d: Expected %v but received %v",
|
|
i+1, test.xlMeta, xlMeta)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestIsXLMetaFormatValid(t *testing.T) {
|
|
tests := []struct {
|
|
name int
|
|
version string
|
|
format string
|
|
want bool
|
|
}{
|
|
{1, "123", "fs", false},
|
|
{2, "123", xlMetaFormat, false},
|
|
{3, xlMetaVersion, "test", false},
|
|
{4, xlMetaVersion100, "hello", false},
|
|
{5, xlMetaVersion, xlMetaFormat, true},
|
|
{6, xlMetaVersion100, xlMetaFormat, true},
|
|
}
|
|
for _, tt := range tests {
|
|
if got := isXLMetaFormatValid(tt.version, tt.format); got != tt.want {
|
|
t.Errorf("Test %d: Expected %v but received %v", tt.name, got, tt.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestIsXLMetaErasureInfoValid(t *testing.T) {
|
|
tests := []struct {
|
|
name int
|
|
data int
|
|
parity int
|
|
want bool
|
|
}{
|
|
{1, 5, 6, false},
|
|
{2, 5, 5, true},
|
|
{3, 0, 5, false},
|
|
{4, 5, 0, false},
|
|
{5, 5, 0, false},
|
|
{6, 5, 4, true},
|
|
}
|
|
for _, tt := range tests {
|
|
if got := isXLMetaErasureInfoValid(tt.data, tt.parity); got != tt.want {
|
|
t.Errorf("Test %d: Expected %v but received %v", tt.name, got, tt.want)
|
|
}
|
|
}
|
|
}
|