maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Adding XL Object layer validation for existing unit tests of single node (#1499)

Browse source 
object layer.

Adding isBucketExist check for GetObjectInfo in the XL layer.
This commit is contained in:
karthic rao 2016-05-07 00:27:04 +05:30 committed by Harshavardhana
parent 48d3be36da
commit 0b4bbe6d9e
5 changed files with 167 additions and 106 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
object-api-getobjectinfo_test.go
View file

@ -27,28 +27,21 @@ import (
"testing"
)
// Testing GetObjectInfo().
// Wrapper for calling GetObjectInfo tests for both XL muliple disks and single node setup.
func TestGetObjectInfo(t *testing.T) {
directory, err := ioutil.TempDir("", "minio-get-objinfo-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(directory)
// Create the obj.
obj, err := newFSObjects(directory)
if err != nil {
t.Fatal(err)
}
ExecObjectLayerTest(t, testGetObjectInfo)
}
// Testing GetObjectInfo().
func testGetObjectInfo(obj ObjectLayer, instanceType string, t *testing.T) {
// This bucket is used for testing getObjectInfo operations.
err = obj.MakeBucket("test-getobjectinfo")
err := obj.MakeBucket("test-getobjectinfo")
if err != nil {
t.Fatal(err)
t.Fatalf("%s : %s", instanceType, err.Error())
}
_, err = obj.PutObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil)
if err != nil {
t.Fatal(err)
t.Fatalf("%s : %s", instanceType, err.Error())
}
resultCases := []ObjectInfo{
// ObjectInfo -1.
@ -88,31 +81,31 @@ func TestGetObjectInfo(t *testing.T) {
for i, testCase := range testCases {
result, err := obj.GetObjectInfo(testCase.bucketName, testCase.objectName)
if err != nil && testCase.shouldPass {
t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
}
if err == nil && !testCase.shouldPass {
t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, instanceType, testCase.err.Error())
}
// Failed as expected, but does it fail for the expected reason.
if err != nil && !testCase.shouldPass {
if testCase.err.Error() != err.Error() {
t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error())
t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.err.Error(), err.Error())
}
}
// Test passes as expected, but the output values are verified for correctness here.
if err == nil && testCase.shouldPass {
if testCase.result.Bucket != result.Bucket {
t.Fatalf("Test %d: Expected Bucket name to be '%s', but found '%s' instead", i+1, testCase.result.Bucket, result.Bucket)
t.Fatalf("Test %d: %s: Expected Bucket name to be '%s', but found '%s' instead", i+1, instanceType, testCase.result.Bucket, result.Bucket)
}
if testCase.result.Name != result.Name {
t.Errorf("Test %d: Expected Object name to be %s, but instead found it to be %s", i+1, testCase.result.Name, result.Name)
t.Errorf("Test %d: %s: Expected Object name to be %s, but instead found it to be %s", i+1, instanceType, testCase.result.Name, result.Name)
}
if testCase.result.ContentType != result.ContentType {
t.Errorf("Test %d: Expected Content Type of the object to be %v, but instead found it to be %v", i+1, testCase.result.ContentType, result.ContentType)
t.Errorf("Test %d: %s: Expected Content Type of the object to be %v, but instead found it to be %v", i+1, instanceType, testCase.result.ContentType, result.ContentType)
}
if testCase.result.IsDir != result.IsDir {
t.Errorf("Test %d: Expected IsDir flag of the object to be %v, but instead found it to be %v", i+1, testCase.result.IsDir, result.IsDir)
t.Errorf("Test %d: %s: Expected IsDir flag of the object to be %v, but instead found it to be %v", i+1, instanceType,testCase.result.IsDir, result.IsDir)
}
}
}

49
object-api-listobjects_test.go
View file

@ -26,70 +26,64 @@ import (
"testing"
)
// Wrapper for calling ListObjects tests for both XL muliple disks and single node setup.
func TestListObjects(t *testing.T) {
// Make a temporary directory to use as the obj.
directory, err := ioutil.TempDir("", "minio-list-object-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(directory)
ExecObjectLayerTest(t, testListObjects)
}
// Create the obj.
obj, err := newFSObjects(directory)
if err != nil {
t.Fatal(err)
}
// Unit test for ListObjects in general.
func testListObjects(obj ObjectLayer, instanceType string, t *testing.T) {
// This bucket is used for testing ListObject operations.
err = obj.MakeBucket("test-bucket-list-object")
err := obj.MakeBucket("test-bucket-list-object")
if err != nil {
t.Fatal(err)
t.Fatalf("%s : %s", instanceType, err.Error())
}
// Will not store any objects in this bucket,
// Its to test ListObjects on an empty bucket.
err = obj.MakeBucket("empty-bucket")
if err != nil {
t.Fatal(err)
t.Fatalf("%s : %s", instanceType, err.Error())
}
tmpfile, err := ioutil.TempFile("", "simple-file.txt")
if err != nil {
t.Fatal(err)
t.Fatalf("%s : %s", instanceType, err.Error())
}
defer os.Remove(tmpfile.Name()) // clean up
_, err = obj.PutObject("test-bucket-list-object", "Asia-maps", int64(len("asia-maps")), bytes.NewBufferString("asia-maps"), nil)
if err != nil {
t.Fatal(err)
t.Fatalf("%s : %s", instanceType, err.Error())
}
_, err = obj.PutObject("test-bucket-list-object", "Asia/India/India-summer-photos-1", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil)
if err != nil {
t.Fatal(err)
t.Fatalf("%s : %s", instanceType, err.Error())
}
_, err = obj.PutObject("test-bucket-list-object", "Asia/India/Karnataka/Bangalore/Koramangala/pics", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil)
if err != nil {
t.Fatal(err)
t.Fatalf("%s : %s", instanceType, err.Error())
}
for i := 0; i < 2; i++ {
key := "newPrefix" + strconv.Itoa(i)
_, err = obj.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil)
if err != nil {
t.Fatal(err)
t.Fatalf("%s : %s", instanceType, err.Error())
}
}
_, err = obj.PutObject("test-bucket-list-object", "newzen/zen/recurse/again/again/again/pics", int64(len("recurse")), bytes.NewBufferString("recurse"), nil)
if err != nil {
t.Fatal(err)
t.Fatalf("%s : %s", instanceType, err.Error())
}
for i := 0; i < 3; i++ {
key := "obj" + strconv.Itoa(i)
_, err = obj.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil)
if err != nil {
t.Fatal(err)
t.Fatalf("%s : %s", instanceType, err.Error())
}
}
@ -532,16 +526,15 @@ func TestListObjects(t *testing.T) {
for i, testCase := range testCases {
result, err := obj.ListObjects(testCase.bucketName, testCase.prefix, testCase.marker, testCase.delimeter, testCase.maxKeys)
if err != nil && testCase.shouldPass {
t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
}
if err == nil && !testCase.shouldPass {
t.Log(result)
t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, instanceType, testCase.err.Error())
}
// Failed as expected, but does it fail for the expected reason.
if err != nil && !testCase.shouldPass {
if !strings.Contains(err.Error(), testCase.err.Error()) {
t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error())
t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.err.Error(), err.Error())
}
}
// Since there are cases for which ListObjects fails, this is
@ -554,15 +547,15 @@ func TestListObjects(t *testing.T) {
// otherwise it may lead to index out of range error in
// assertion following this.
if len(testCase.result.Objects) != len(result.Objects) {
t.Fatalf("Test %d: Expected number of object in the result to be '%d', but found '%d' objects instead", i+1, len(testCase.result.Objects), len(result.Objects))
t.Fatalf("Test %d: %s: Expected number of object in the result to be '%d', but found '%d' objects instead", i+1, instanceType, len(testCase.result.Objects), len(result.Objects))
}
for j := 0; j < len(testCase.result.Objects); j++ {
if testCase.result.Objects[j].Name != result.Objects[j].Name {
t.Errorf("Test %d: Expected object name to be \"%s\", but found \"%s\" instead", i+1, testCase.result.Objects[j].Name, result.Objects[j].Name)
t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Objects[j].Name, result.Objects[j].Name)
}
}
if testCase.result.IsTruncated != result.IsTruncated {
t.Errorf("Test %d: Expected IsTruncated flag to be %v, but instead found it to be %v", i+1, testCase.result.IsTruncated, result.IsTruncated)
t.Errorf("Test %d: %s: Expected IsTruncated flag to be %v, but instead found it to be %v", i+1, instanceType, testCase.result.IsTruncated, result.IsTruncated)
}
}

90
object-api-multipart_test.go
View file

@ -19,24 +19,16 @@ package main
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"testing"
)
// Tests validate creation of new multipart upload instance.
// Wrapper for calling NewMultipartUpload tests for both XL muliple disks and single node setup.
func TestObjectNewMultipartUpload(t *testing.T) {
directory, err := ioutil.TempDir("", "minio-multipart-1-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(directory)
ExecObjectLayerTest(t, testObjectNewMultipartUpload)
}
// Initialize fs object layer.
obj, err := newFSObjects(directory)
if err != nil {
t.Fatal(err)
}
// Tests validate creation of new multipart upload instance.
func testObjectNewMultipartUpload(obj ObjectLayer, instanceType string, t *testing.T) {
bucket := "minio-bucket"
object := "minio-object"
@ -45,107 +37,93 @@ func TestObjectNewMultipartUpload(t *testing.T) {
// opearation expected to fail since the bucket on which NewMultipartUpload is being initiated doesn't exist.
uploadID, err := obj.NewMultipartUpload(bucket, object)
if err == nil {
t.Fatalf("Expected to fail since the NewMultipartUpload is intialized on a non-existant bucket.")
t.Fatalf("%s: Expected to fail since the NewMultipartUpload is intialized on a non-existant bucket.", instanceType)
}
if errMsg != err.Error() {
t.Errorf("Expected to fail with Error \"%s\", but instead found \"%s\".", errMsg, err.Error())
t.Errorf("%s, Expected to fail with Error \"%s\", but instead found \"%s\".", instanceType, errMsg, err.Error())
}
// Create bucket before intiating NewMultipartUpload.
err = obj.MakeBucket(bucket)
if err != nil {
// failed to create newbucket, abort.
t.Fatal(err)
t.Fatalf("%s : %s", instanceType, err.Error())
}
uploadID, err = obj.NewMultipartUpload(bucket, object)
if err != nil {
t.Fatal(err)
t.Fatalf("%s : %s", instanceType, err.Error())
}
err = obj.AbortMultipartUpload(bucket, object, uploadID)
if err != nil {
switch err.(type) {
case InvalidUploadID:
t.Fatalf("New Multipart upload failed to create uuid file.")
t.Fatalf("%s: New Multipart upload failed to create uuid file.", instanceType)
default:
t.Fatalf(err.Error())
}
}
}
// Tests validates the validator for existence of uploadID.
// Wrapper for calling isUploadIDExists tests for both XL muliple disks and single node setup.
func TestObjectAPIIsUploadIDExists(t *testing.T) {
directory, err := ioutil.TempDir("", "minio-multipart-2-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(directory)
// Initialize fs object layer.
obj, err := newFSObjects(directory)
if err != nil {
t.Fatal(err)
}
ExecObjectLayerTest(t, testObjectAPIIsUploadIDExists)
}
// Tests validates the validator for existence of uploadID.
func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t *testing.T) {
bucket := "minio-bucket"
object := "minio-object"
// Create bucket before intiating NewMultipartUpload.
err = obj.MakeBucket(bucket)
err := obj.MakeBucket(bucket)
if err != nil {
// Failed to create newbucket, abort.
t.Fatal(err)
t.Fatalf("%s : %s", instanceType, err.Error())
}
_, err = obj.NewMultipartUpload(bucket, object)
if err != nil {
t.Fatal(err)
t.Fatalf("%s : %s", instanceType, err.Error())
}
err = obj.AbortMultipartUpload(bucket, object, "abc")
switch err.(type) {
case InvalidUploadID:
default:
t.Fatal("Expected uploadIDPath to exist.")
t.Fatalf("%s: Expected uploadIDPath to exist.", instanceType)
}
}
// Tests validate correctness of PutObjectPart.
// Wrapper for calling PutObjectPart tests for both XL muliple disks and single node setup.
func TestObjectAPIPutObjectPart(t *testing.T) {
ExecObjectLayerTest(t, testObjectAPIPutObjectPart)
}
// Tests validate correctness of PutObjectPart.
func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t *testing.T) {
// Generating cases for which the PutObjectPart fails.
directory, err := ioutil.TempDir("", "minio-multipart-3-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(directory)
// Initializing fs object layer.
obj, err := newFSObjects(directory)
if err != nil {
t.Fatal(err)
}
bucket := "minio-bucket"
object := "minio-object"
// Create bucket before intiating NewMultipartUpload.
err = obj.MakeBucket(bucket)
err := obj.MakeBucket(bucket)
if err != nil {
// Failed to create newbucket, abort.
t.Fatal(err)
t.Fatalf("%s : %s", instanceType, err.Error())
}
// Initiate Multipart Upload on the above created bucket.
uploadID, err := obj.NewMultipartUpload(bucket, object)
if err != nil {
// Failed to create NewMultipartUpload, abort.
t.Fatal(err)
t.Fatalf("%s : %s", instanceType, err.Error())
}
// Creating a dummy bucket for tests.
err = obj.MakeBucket("unused-bucket")
if err != nil {
// Failed to create newbucket, abort.
t.Fatal(err)
t.Fatalf("%s : %s", instanceType, err.Error())
}
failCases := []struct {
@ -220,23 +198,23 @@ func TestObjectAPIPutObjectPart(t *testing.T) {
// All are test cases above are expected to fail.
if actualErr != nil && testCase.shouldPass {
t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s.", i+1, actualErr.Error())
t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", i+1, instanceType, actualErr.Error())
}
if actualErr == nil && !testCase.shouldPass {
t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead.", i+1, testCase.expectedError.Error())
t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead.", i+1, instanceType, testCase.expectedError.Error())
}
// Failed as expected, but does it fail for the expected reason.
if actualErr != nil && !testCase.shouldPass {
if testCase.expectedError.Error() != actualErr.Error() {
t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1,
testCase.expectedError.Error(), actualErr.Error())
t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1,
instanceType, testCase.expectedError.Error(), actualErr.Error())
}
}
// Test passes as expected, but the output values are verified for correctness here.
if actualErr == nil && testCase.shouldPass {
// Asserting whether the md5 output is correct.
if testCase.inputMd5 != actualMd5Hex {
t.Errorf("Test %d: Calculated Md5 different from the actual one %s.", i+1, actualMd5Hex)
t.Errorf("Test %d: %s: Calculated Md5 different from the actual one %s.", i+1, instanceType, actualMd5Hex)
}
}
}

91
test-utils.go Normal file
View file

@ -0,0 +1,91 @@
/*
* Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"io/ioutil"
"os"
"testing"
)
const (
// singleNodeTestStr is the string which is used as notation for Single node ObjectLayer in the unit tests.
singleNodeTestStr string = "SingleNode"
// xLTestStr is the string which is used as notation for XL ObjectLayer in the unit tests.
xLTestStr string = "XL"
)
// ExecObjectLayerTest - executes object layer tests.
// Creates single node and XL ObjectLayer instance and runs test for both the layers.
func ExecObjectLayerTest(t *testing.T, objTest func(obj ObjectLayer, instanceType string, t *testing.T)) {
// getXLObjectLayer - Instantiates XL object layer and returns it.
getXLObjectLayer := func() (ObjectLayer, []string, error) {
var nDisks = 16 // Maximum disks.
var erasureDisks []string
for i := 0; i < nDisks; i++ {
path, err := ioutil.TempDir(os.TempDir(), "minio-")
if err != nil {
return nil, nil, err
}
erasureDisks = append(erasureDisks, path)
}
objLayer, err := newXLObjects(erasureDisks...)
if err != nil {
return nil, nil, err
}
return objLayer, erasureDisks, nil
}
// getSingleNodeObjectLayer - Instantiates single node object layer and retuns it.
getSingleNodeObjectLayer := func() (ObjectLayer, string, error) {
// Make a temporary directory to use as the obj.
fsDir, err := ioutil.TempDir("", "minio-")
if err != nil {
return nil, "", err
}
// Create the obj.
objLayer, err := newFSObjects(fsDir)
if err != nil {
return nil, "", err
}
return objLayer, fsDir, nil
}
// removeRoots - Cleans up initialized directories during tests.
removeRoots := func(roots []string) {
for _, root := range roots {
os.RemoveAll(root)
}
}
objLayer, fsDir, err := getSingleNodeObjectLayer()
if err != nil {
t.Fatalf("Initialization of object layer failed for single node setup: %s", err.Error())
}
// Executing the object layer tests for single node setup.
objTest(objLayer, singleNodeTestStr, t)
initNSLock()
objLayer, fsDirs, err := getXLObjectLayer()
if err != nil {
t.Fatalf("Initialization of object layer failed for XL setup: %s", err.Error())
}
// Executing the object layer tests for XL.
objTest(objLayer, xLTestStr, t)
defer removeRoots(append(fsDirs, fsDir))
}

6
xl-objects.go
View file

@ -159,6 +159,12 @@ func (xl xlObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
if !IsValidObjectName(object) {
return ObjectInfo{}, ObjectNameInvalid{Bucket: bucket, Object: object}
}
// Check whether the bucket exists.
if isExist, err := isBucketExist(xl.storage, bucket); err != nil {
return ObjectInfo{}, err
} else if !isExist {
return ObjectInfo{}, BucketNotFound{Bucket: bucket}
}
fi, err := xl.storage.StatFile(bucket, object)
if err != nil {
if err != errFileNotFound {