2016-05-06 20:57:04 +02:00
/ *
* Minio Cloud Storage , ( C ) 2015 , 2016 Minio , Inc .
*
* Licensed under the Apache License , Version 2.0 ( the "License" ) ;
* you may not use this file except in compliance with the License .
* You may obtain a copy of the License at
*
* http : //www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing , software
* distributed under the License is distributed on an "AS IS" BASIS ,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
* See the License for the specific language governing permissions and
* limitations under the License .
* /
2016-08-19 01:23:42 +02:00
package cmd
2016-05-06 20:57:04 +02:00
import (
2016-06-21 21:10:18 +02:00
"bytes"
"encoding/base64"
"encoding/hex"
2016-08-16 01:13:03 +02:00
"encoding/json"
"errors"
2016-06-21 21:10:18 +02:00
"fmt"
"io"
2016-05-06 20:57:04 +02:00
"io/ioutil"
2016-06-26 04:07:44 +02:00
"math/rand"
2016-06-21 21:10:18 +02:00
"net/http"
"net/http/httptest"
2016-06-26 04:07:44 +02:00
"net/url"
2016-05-06 20:57:04 +02:00
"os"
2016-09-16 22:06:49 +02:00
"reflect"
2016-06-26 04:07:44 +02:00
"regexp"
2016-06-21 21:10:18 +02:00
"sort"
2016-06-29 12:13:44 +02:00
"strconv"
2016-06-21 21:10:18 +02:00
"strings"
2016-06-29 12:13:44 +02:00
"sync"
2016-05-06 20:57:04 +02:00
"testing"
2016-06-21 21:10:18 +02:00
"time"
2016-06-26 04:07:44 +02:00
"unicode/utf8"
2016-07-03 04:05:16 +02:00
router "github.com/gorilla/mux"
2016-05-06 20:57:04 +02:00
)
2016-07-08 04:50:44 +02:00
// Tests should initNSLock only once.
func init ( ) {
// Initialize name space lock.
2016-08-10 22:08:11 +02:00
isDist := false
initNSLock ( isDist )
2016-07-08 04:50:44 +02:00
}
2016-08-31 04:22:27 +02:00
func prepareFS ( ) ( ObjectLayer , string , error ) {
fsDirs , err := getRandomDisks ( 1 )
if err != nil {
return nil , "" , err
}
obj , err := getSingleNodeObjectLayer ( fsDirs [ 0 ] )
if err != nil {
removeRoots ( fsDirs )
return nil , "" , err
}
return obj , fsDirs [ 0 ] , nil
}
func prepareXL ( ) ( ObjectLayer , [ ] string , error ) {
nDisks := 16
fsDirs , err := getRandomDisks ( nDisks )
if err != nil {
return nil , nil , err
}
2016-09-15 08:53:42 +02:00
obj , err := getXLObjectLayer ( fsDirs , nil )
2016-08-31 04:22:27 +02:00
if err != nil {
removeRoots ( fsDirs )
return nil , nil , err
}
return obj , fsDirs , nil
}
2016-07-26 05:39:14 +02:00
// TestErrHandler - Golang Testing.T and Testing.B, and gocheck.C satisfy this interface.
2016-06-21 21:10:18 +02:00
// This makes it easy to run the TestServer from any of the tests.
2016-07-26 05:39:14 +02:00
// Using this interface, functionalities to be used in tests can be made generalized, and can be integrated in benchmarks/unit tests/go check suite tests.
2016-06-21 21:10:18 +02:00
type TestErrHandler interface {
Error ( args ... interface { } )
Errorf ( format string , args ... interface { } )
Failed ( ) bool
Fatal ( args ... interface { } )
Fatalf ( format string , args ... interface { } )
}
2016-05-06 20:57:04 +02:00
const (
// singleNodeTestStr is the string which is used as notation for Single node ObjectLayer in the unit tests.
2016-07-09 03:26:04 +02:00
singleNodeTestStr string = "FS"
2016-05-06 20:57:04 +02:00
// xLTestStr is the string which is used as notation for XL ObjectLayer in the unit tests.
xLTestStr string = "XL"
)
2016-06-26 04:07:44 +02:00
const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569"
const (
letterIdxBits = 6 // 6 bits to represent a letter index
letterIdxMask = 1 << letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits
)
2016-06-29 12:13:44 +02:00
// Random number state.
// We generate random temporary file names so that there's a good
// chance the file doesn't exist yet.
var randN uint32
var randmu sync . Mutex
2016-07-02 10:59:28 +02:00
// reseed - returns a new seed every time the function is called.
2016-06-29 12:13:44 +02:00
func reseed ( ) uint32 {
return uint32 ( time . Now ( ) . UnixNano ( ) + int64 ( os . Getpid ( ) ) )
}
2016-07-02 10:59:28 +02:00
// nextSuffix - provides a new unique suffix every time the function is called.
2016-06-29 12:13:44 +02:00
func nextSuffix ( ) string {
randmu . Lock ( )
r := randN
// Initial seed required, generate one.
if r == 0 {
r = reseed ( )
}
// constants from Numerical Recipes
r = r * 1664525 + 1013904223
randN = r
randmu . Unlock ( )
return strconv . Itoa ( int ( 1e9 + r % 1e9 ) ) [ 1 : ]
}
2016-09-16 22:06:49 +02:00
// isSameType - compares two object types via reflect.TypeOf
func isSameType ( obj1 , obj2 interface { } ) bool {
return reflect . TypeOf ( obj1 ) == reflect . TypeOf ( obj2 )
}
2016-06-21 21:10:18 +02:00
// TestServer encapsulates an instantiation of a Minio instance with a temporary backend.
// Example usage:
// s := StartTestServer(t,"XL")
// defer s.Stop()
type TestServer struct {
Root string
Disks [ ] string
AccessKey string
SecretKey string
Server * httptest . Server
2016-08-31 04:22:27 +02:00
Obj ObjectLayer
2016-06-21 21:10:18 +02:00
}
// Starts the test server and returns the TestServer instance.
func StartTestServer ( t TestErrHandler , instanceType string ) TestServer {
// create an instance of TestServer.
testServer := TestServer { }
// create temporary backend for the test server.
2016-08-31 04:22:27 +02:00
nDisks := 16
disks , err := getRandomDisks ( nDisks )
2016-06-21 21:10:18 +02:00
if err != nil {
2016-08-31 04:22:27 +02:00
t . Fatal ( "Failed to create disks for the backend" )
2016-06-21 21:10:18 +02:00
}
2016-07-03 04:05:16 +02:00
2016-07-26 09:01:35 +02:00
root , err := newTestConfig ( "us-east-1" )
2016-06-21 21:10:18 +02:00
if err != nil {
2016-07-03 04:05:16 +02:00
t . Fatalf ( "%s" , err )
2016-06-21 21:10:18 +02:00
}
2016-07-26 09:01:35 +02:00
2016-08-31 04:22:27 +02:00
// Test Server needs to start before formatting of disks.
2016-07-26 09:01:35 +02:00
// Get credential.
credentials := serverConfig . GetCredential ( )
2016-06-21 21:10:18 +02:00
testServer . Root = root
2016-08-31 04:22:27 +02:00
testServer . Disks = disks
2016-07-03 04:05:16 +02:00
testServer . AccessKey = credentials . AccessKeyID
testServer . SecretKey = credentials . SecretAccessKey
// Run TestServer.
2016-08-31 04:22:27 +02:00
testServer . Server = httptest . NewServer ( configureServerHandler ( serverCmdConfig { disks : disks } ) )
2016-07-03 04:05:16 +02:00
2016-08-31 04:22:27 +02:00
objLayer , err := makeTestBackend ( disks , instanceType )
if err != nil {
t . Fatalf ( "Failed obtaining Temp Backend: <ERROR> %s" , err )
}
testServer . Obj = objLayer
objLayerMutex . Lock ( )
globalObjectAPI = objLayer
objLayerMutex . Unlock ( )
2016-07-03 04:05:16 +02:00
return testServer
}
2016-08-31 20:39:08 +02:00
// Initializes control RPC end points.
// The object Layer will be a temp back used for testing purpose.
func initTestControlRPCEndPoint ( objectLayer ObjectLayer ) http . Handler {
// Initialize Web.
controllerHandlers := & controllerAPIHandlers {
ObjectAPI : func ( ) ObjectLayer { return objectLayer } ,
}
// Initialize router.
muxRouter := router . NewRouter ( )
registerControllerRPCRouter ( muxRouter , controllerHandlers )
return muxRouter
}
// StartTestRPCServer - Creates a temp XL/FS backend and initializes control RPC end points,
// then starts a test server with those control RPC end points registered.
func StartTestRPCServer ( t TestErrHandler , instanceType string ) TestServer {
// create temporary backend for the test server.
nDisks := 16
disks , err := getRandomDisks ( nDisks )
if err != nil {
t . Fatal ( "Failed to create disks for the backend" )
}
// create an instance of TestServer.
testRPCServer := TestServer { }
// create temporary backend for the test server.
objLayer , err := makeTestBackend ( disks , instanceType )
if err != nil {
t . Fatalf ( "Failed obtaining Temp Backend: <ERROR> %s" , err )
}
root , err := newTestConfig ( "us-east-1" )
if err != nil {
t . Fatalf ( "%s" , err )
}
// Get credential.
credentials := serverConfig . GetCredential ( )
testRPCServer . Root = root
testRPCServer . Disks = disks
testRPCServer . AccessKey = credentials . AccessKeyID
testRPCServer . SecretKey = credentials . SecretAccessKey
testRPCServer . Obj = objLayer
// Run TestServer.
testRPCServer . Server = httptest . NewServer ( initTestControlRPCEndPoint ( objLayer ) )
return testRPCServer
}
2016-07-03 04:05:16 +02:00
// Configure the server for the test run.
2016-07-26 09:01:35 +02:00
func newTestConfig ( bucketLocation string ) ( rootPath string , err error ) {
// Get test root.
rootPath , err = getTestRoot ( )
2016-07-03 04:05:16 +02:00
if err != nil {
2016-07-26 09:01:35 +02:00
return "" , err
2016-07-03 04:05:16 +02:00
}
2016-07-26 09:01:35 +02:00
2016-06-21 21:10:18 +02:00
// Do this only once here.
2016-07-03 04:05:16 +02:00
setGlobalConfigPath ( rootPath )
2016-06-21 21:10:18 +02:00
2016-07-26 09:01:35 +02:00
// Initialize server config.
if err = initConfig ( ) ; err != nil {
return "" , err
2016-06-21 21:10:18 +02:00
}
2016-07-26 09:01:35 +02:00
// Set a default region.
serverConfig . SetRegion ( bucketLocation )
// Save config.
if err = serverConfig . Save ( ) ; err != nil {
return "" , err
}
// Return root path.
return rootPath , nil
2016-06-21 21:10:18 +02:00
}
// Deleting the temporary backend and stopping the server.
func ( testServer TestServer ) Stop ( ) {
removeAll ( testServer . Root )
for _ , disk := range testServer . Disks {
removeAll ( disk )
}
testServer . Server . Close ( )
}
2016-09-04 22:37:14 +02:00
// Sign given request using Signature V4.
func signStreamingRequest ( req * http . Request , accessKey , secretKey string ) ( string , error ) {
// Get hashed payload.
hashedPayload := req . Header . Get ( "x-amz-content-sha256" )
if hashedPayload == "" {
return "" , fmt . Errorf ( "Invalid hashed payload." )
}
currTime := time . Now ( ) . UTC ( )
// Set x-amz-date.
req . Header . Set ( "x-amz-date" , currTime . Format ( iso8601Format ) )
// Get header map.
headerMap := make ( map [ string ] [ ] string )
for k , vv := range req . Header {
// If request header key is not in ignored headers, then add it.
if _ , ok := ignoredStreamingHeaders [ http . CanonicalHeaderKey ( k ) ] ; ! ok {
headerMap [ strings . ToLower ( k ) ] = vv
}
}
// Get header keys.
headers := [ ] string { "host" }
for k := range headerMap {
headers = append ( headers , k )
}
sort . Strings ( headers )
// Get canonical headers.
var buf bytes . Buffer
for _ , k := range headers {
buf . WriteString ( k )
buf . WriteByte ( ':' )
switch {
case k == "host" :
buf . WriteString ( req . URL . Host )
fallthrough
default :
for idx , v := range headerMap [ k ] {
if idx > 0 {
buf . WriteByte ( ',' )
}
buf . WriteString ( v )
}
buf . WriteByte ( '\n' )
}
}
canonicalHeaders := buf . String ( )
// Get signed headers.
signedHeaders := strings . Join ( headers , ";" )
// Get canonical query string.
req . URL . RawQuery = strings . Replace ( req . URL . Query ( ) . Encode ( ) , "+" , "%20" , - 1 )
// Get canonical URI.
canonicalURI := getURLEncodedName ( req . URL . Path )
// Get canonical request.
// canonicalRequest =
// <HTTPMethod>\n
// <CanonicalURI>\n
// <CanonicalQueryString>\n
// <CanonicalHeaders>\n
// <SignedHeaders>\n
// <HashedPayload>
//
canonicalRequest := strings . Join ( [ ] string {
req . Method ,
canonicalURI ,
req . URL . RawQuery ,
canonicalHeaders ,
signedHeaders ,
hashedPayload ,
} , "\n" )
// Get scope.
scope := strings . Join ( [ ] string {
currTime . Format ( yyyymmdd ) ,
"us-east-1" ,
"s3" ,
"aws4_request" ,
} , "/" )
stringToSign := "AWS4-HMAC-SHA256" + "\n" + currTime . Format ( iso8601Format ) + "\n"
stringToSign = stringToSign + scope + "\n"
stringToSign = stringToSign + hex . EncodeToString ( sum256 ( [ ] byte ( canonicalRequest ) ) )
date := sumHMAC ( [ ] byte ( "AWS4" + secretKey ) , [ ] byte ( currTime . Format ( yyyymmdd ) ) )
region := sumHMAC ( date , [ ] byte ( "us-east-1" ) )
service := sumHMAC ( region , [ ] byte ( "s3" ) )
signingKey := sumHMAC ( service , [ ] byte ( "aws4_request" ) )
signature := hex . EncodeToString ( sumHMAC ( signingKey , [ ] byte ( stringToSign ) ) )
// final Authorization header
parts := [ ] string {
"AWS4-HMAC-SHA256" + " Credential=" + accessKey + "/" + scope ,
"SignedHeaders=" + signedHeaders ,
"Signature=" + signature ,
}
auth := strings . Join ( parts , ", " )
req . Header . Set ( "Authorization" , auth )
return signature , nil
}
// Returns new HTTP request object.
func newTestStreamingRequest ( method , urlStr string , dataLength , chunkSize int64 , body io . ReadSeeker ) ( * http . Request , error ) {
if method == "" {
method = "POST"
}
req , err := http . NewRequest ( method , urlStr , nil )
if err != nil {
return nil , err
}
if body == nil {
// this is added to avoid panic during ioutil.ReadAll(req.Body).
// th stack trace can be found here https://github.com/minio/minio/pull/2074 .
// This is very similar to https://github.com/golang/go/issues/7527.
req . Body = ioutil . NopCloser ( bytes . NewReader ( [ ] byte ( "" ) ) )
}
contentLength := calculateStreamContentLength ( dataLength , chunkSize )
req . Header . Set ( "x-amz-content-sha256" , "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" )
req . Header . Set ( "content-encoding" , "aws-chunked" )
req . Header . Set ( "x-amz-decoded-content-length" , strconv . FormatInt ( dataLength , 10 ) )
req . Header . Set ( "content-length" , strconv . FormatInt ( contentLength , 10 ) )
// Seek back to beginning.
body . Seek ( 0 , 0 )
2016-09-16 11:45:42 +02:00
2016-09-04 22:37:14 +02:00
// Add body
req . Body = ioutil . NopCloser ( body )
req . ContentLength = contentLength
return req , nil
}
// Returns new HTTP request object signed with streaming signature v4.
func newTestStreamingSignedRequest ( method , urlStr string , contentLength , chunkSize int64 , body io . ReadSeeker , accessKey , secretKey string ) ( * http . Request , error ) {
req , err := newTestStreamingRequest ( method , urlStr , contentLength , chunkSize , body )
if err != nil {
return nil , err
}
signature , err := signStreamingRequest ( req , accessKey , secretKey )
if err != nil {
return nil , err
}
2016-09-16 11:45:42 +02:00
regionStr := serverConfig . GetRegion ( )
2016-09-04 22:37:14 +02:00
var stream [ ] byte
var buffer [ ] byte
body . Seek ( 0 , 0 )
for {
buffer = make ( [ ] byte , chunkSize )
n , err := body . Read ( buffer )
if err != nil && err != io . EOF {
return nil , err
}
currTime := time . Now ( ) . UTC ( )
// Get scope.
scope := strings . Join ( [ ] string {
currTime . Format ( yyyymmdd ) ,
2016-09-16 11:45:42 +02:00
regionStr ,
2016-09-04 22:37:14 +02:00
"s3" ,
"aws4_request" ,
} , "/" )
stringToSign := "AWS4-HMAC-SHA256-PAYLOAD" + "\n"
stringToSign = stringToSign + currTime . Format ( iso8601Format ) + "\n"
stringToSign = stringToSign + scope + "\n"
stringToSign = stringToSign + signature + "\n"
stringToSign = stringToSign + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + "\n" // hex(sum256(""))
stringToSign = stringToSign + hex . EncodeToString ( sum256 ( buffer [ : n ] ) )
date := sumHMAC ( [ ] byte ( "AWS4" + secretKey ) , [ ] byte ( currTime . Format ( yyyymmdd ) ) )
2016-09-16 11:45:42 +02:00
region := sumHMAC ( date , [ ] byte ( regionStr ) )
2016-09-04 22:37:14 +02:00
service := sumHMAC ( region , [ ] byte ( "s3" ) )
signingKey := sumHMAC ( service , [ ] byte ( "aws4_request" ) )
signature = hex . EncodeToString ( sumHMAC ( signingKey , [ ] byte ( stringToSign ) ) )
stream = append ( stream , [ ] byte ( fmt . Sprintf ( "%x" , n ) + ";chunk-signature=" + signature + "\r\n" ) ... )
stream = append ( stream , buffer [ : n ] ... )
stream = append ( stream , [ ] byte ( "\r\n" ) ... )
if n <= 0 {
break
}
}
req . Body = ioutil . NopCloser ( bytes . NewReader ( stream ) )
return req , nil
}
2016-07-10 20:10:59 +02:00
// Sign given request using Signature V4.
func signRequest ( req * http . Request , accessKey , secretKey string ) error {
// Get hashed payload.
hashedPayload := req . Header . Get ( "x-amz-content-sha256" )
if hashedPayload == "" {
return fmt . Errorf ( "Invalid hashed payload." )
2016-06-21 21:10:18 +02:00
}
2016-07-10 20:10:59 +02:00
currTime := time . Now ( ) . UTC ( )
2016-06-21 21:10:18 +02:00
2016-07-10 20:10:59 +02:00
// Set x-amz-date.
req . Header . Set ( "x-amz-date" , currTime . Format ( iso8601Format ) )
2016-06-21 21:10:18 +02:00
2016-07-10 20:10:59 +02:00
// Get header map.
headerMap := make ( map [ string ] [ ] string )
for k , vv := range req . Header {
// If request header key is not in ignored headers, then add it.
if _ , ok := ignoredHeaders [ http . CanonicalHeaderKey ( k ) ] ; ! ok {
headerMap [ strings . ToLower ( k ) ] = vv
2016-06-21 21:10:18 +02:00
}
}
2016-07-10 20:10:59 +02:00
// Get header keys.
headers := [ ] string { "host" }
for k := range headerMap {
headers = append ( headers , k )
2016-06-21 21:10:18 +02:00
}
sort . Strings ( headers )
2016-09-15 08:53:42 +02:00
region := serverConfig . GetRegion ( )
2016-07-10 20:10:59 +02:00
// Get canonical headers.
var buf bytes . Buffer
2016-06-21 21:10:18 +02:00
for _ , k := range headers {
2016-07-10 20:10:59 +02:00
buf . WriteString ( k )
buf . WriteByte ( ':' )
2016-06-21 21:10:18 +02:00
switch {
case k == "host" :
2016-07-10 20:10:59 +02:00
buf . WriteString ( req . URL . Host )
2016-06-21 21:10:18 +02:00
fallthrough
default :
2016-07-10 20:10:59 +02:00
for idx , v := range headerMap [ k ] {
2016-06-21 21:10:18 +02:00
if idx > 0 {
2016-07-10 20:10:59 +02:00
buf . WriteByte ( ',' )
2016-06-21 21:10:18 +02:00
}
2016-07-10 20:10:59 +02:00
buf . WriteString ( v )
2016-06-21 21:10:18 +02:00
}
2016-07-10 20:10:59 +02:00
buf . WriteByte ( '\n' )
2016-06-21 21:10:18 +02:00
}
}
2016-07-10 20:10:59 +02:00
canonicalHeaders := buf . String ( )
2016-06-21 21:10:18 +02:00
2016-07-10 20:10:59 +02:00
// Get signed headers.
2016-06-21 21:10:18 +02:00
signedHeaders := strings . Join ( headers , ";" )
2016-07-10 20:10:59 +02:00
// Get canonical query string.
2016-06-21 21:10:18 +02:00
req . URL . RawQuery = strings . Replace ( req . URL . Query ( ) . Encode ( ) , "+" , "%20" , - 1 )
2016-07-10 20:10:59 +02:00
// Get canonical URI.
canonicalURI := getURLEncodedName ( req . URL . Path )
// Get canonical request.
2016-06-21 21:10:18 +02:00
// canonicalRequest =
// <HTTPMethod>\n
// <CanonicalURI>\n
// <CanonicalQueryString>\n
// <CanonicalHeaders>\n
// <SignedHeaders>\n
// <HashedPayload>
//
canonicalRequest := strings . Join ( [ ] string {
req . Method ,
2016-07-10 20:10:59 +02:00
canonicalURI ,
2016-06-21 21:10:18 +02:00
req . URL . RawQuery ,
2016-07-10 20:10:59 +02:00
canonicalHeaders ,
2016-06-21 21:10:18 +02:00
signedHeaders ,
hashedPayload ,
} , "\n" )
2016-07-10 20:10:59 +02:00
// Get scope.
2016-06-21 21:10:18 +02:00
scope := strings . Join ( [ ] string {
2016-07-10 20:10:59 +02:00
currTime . Format ( yyyymmdd ) ,
2016-09-15 08:53:42 +02:00
region ,
2016-06-21 21:10:18 +02:00
"s3" ,
"aws4_request" ,
} , "/" )
2016-07-10 20:10:59 +02:00
stringToSign := "AWS4-HMAC-SHA256" + "\n" + currTime . Format ( iso8601Format ) + "\n"
2016-06-21 21:10:18 +02:00
stringToSign = stringToSign + scope + "\n"
stringToSign = stringToSign + hex . EncodeToString ( sum256 ( [ ] byte ( canonicalRequest ) ) )
2016-07-10 20:10:59 +02:00
date := sumHMAC ( [ ] byte ( "AWS4" + secretKey ) , [ ] byte ( currTime . Format ( yyyymmdd ) ) )
2016-09-15 08:53:42 +02:00
regionHMAC := sumHMAC ( date , [ ] byte ( region ) )
service := sumHMAC ( regionHMAC , [ ] byte ( "s3" ) )
2016-06-21 21:10:18 +02:00
signingKey := sumHMAC ( service , [ ] byte ( "aws4_request" ) )
signature := hex . EncodeToString ( sumHMAC ( signingKey , [ ] byte ( stringToSign ) ) )
// final Authorization header
parts := [ ] string {
"AWS4-HMAC-SHA256" + " Credential=" + accessKey + "/" + scope ,
"SignedHeaders=" + signedHeaders ,
"Signature=" + signature ,
}
auth := strings . Join ( parts , ", " )
req . Header . Set ( "Authorization" , auth )
2016-07-10 20:10:59 +02:00
return nil
}
2016-09-14 04:00:01 +02:00
// getCredential generate a credential string.
func getCredential ( accessKeyID , location string , t time . Time ) string {
return accessKeyID + "/" + getScope ( t , location )
}
2016-07-10 20:10:59 +02:00
// Returns new HTTP request object.
func newTestRequest ( method , urlStr string , contentLength int64 , body io . ReadSeeker ) ( * http . Request , error ) {
if method == "" {
method = "POST"
}
req , err := http . NewRequest ( method , urlStr , nil )
if err != nil {
return nil , err
}
// Add Content-Length
req . ContentLength = contentLength
// Save for subsequent use
var hashedPayload string
switch {
case body == nil :
hashedPayload = hex . EncodeToString ( sum256 ( [ ] byte { } ) )
default :
payloadBytes , e := ioutil . ReadAll ( body )
if e != nil {
return nil , e
}
hashedPayload = hex . EncodeToString ( sum256 ( payloadBytes ) )
md5Base64 := base64 . StdEncoding . EncodeToString ( sumMD5 ( payloadBytes ) )
req . Header . Set ( "Content-Md5" , md5Base64 )
}
req . Header . Set ( "x-amz-content-sha256" , hashedPayload )
// Seek back to beginning.
if body != nil {
body . Seek ( 0 , 0 )
// Add body
req . Body = ioutil . NopCloser ( body )
} else {
// this is added to avoid panic during ioutil.ReadAll(req.Body).
// th stack trace can be found here https://github.com/minio/minio/pull/2074 .
// This is very similar to https://github.com/golang/go/issues/7527.
req . Body = ioutil . NopCloser ( bytes . NewReader ( [ ] byte ( "" ) ) )
}
return req , nil
}
// Returns new HTTP request object signed with signature v4.
func newTestSignedRequest ( method , urlStr string , contentLength int64 , body io . ReadSeeker , accessKey , secretKey string ) ( * http . Request , error ) {
req , err := newTestRequest ( method , urlStr , contentLength , body )
if err != nil {
return nil , err
}
2016-09-09 19:18:38 +02:00
// Anonymous request return quickly.
if accessKey == "" || secretKey == "" {
return req , nil
}
2016-07-10 20:10:59 +02:00
err = signRequest ( req , accessKey , secretKey )
if err != nil {
return nil , err
}
2016-06-21 21:10:18 +02:00
return req , nil
}
2016-08-16 01:13:03 +02:00
// Return new WebRPC request object.
func newWebRPCRequest ( methodRPC , authorization string , body io . ReadSeeker ) ( * http . Request , error ) {
req , err := http . NewRequest ( "POST" , "/minio/webrpc" , nil )
if err != nil {
return nil , err
}
req . Header . Set ( "Content-Type" , "application/json" )
if authorization != "" {
req . Header . Set ( "Authorization" , "Bearer " + authorization )
}
// Seek back to beginning.
if body != nil {
body . Seek ( 0 , 0 )
// Add body
req . Body = ioutil . NopCloser ( body )
} else {
// this is added to avoid panic during ioutil.ReadAll(req.Body).
// th stack trace can be found here https://github.com/minio/minio/pull/2074 .
// This is very similar to https://github.com/golang/go/issues/7527.
req . Body = ioutil . NopCloser ( bytes . NewReader ( [ ] byte ( "" ) ) )
}
return req , nil
}
// Marshal request and return a new HTTP request object to call the webrpc
func newTestWebRPCRequest ( rpcMethod string , authorization string , data interface { } ) ( * http . Request , error ) {
type genericJSON struct {
JSONRPC string ` json:"jsonrpc" `
ID string ` json:"id" `
Method string ` json:"method" `
Params interface { } ` json:"params" `
}
encapsulatedData := genericJSON { JSONRPC : "2.0" , ID : "1" , Method : rpcMethod , Params : data }
jsonData , err := json . Marshal ( encapsulatedData )
2016-09-02 08:10:50 +02:00
if err != nil {
return nil , err
}
2016-08-16 01:13:03 +02:00
req , err := newWebRPCRequest ( rpcMethod , authorization , bytes . NewReader ( jsonData ) )
if err != nil {
return nil , err
}
return req , nil
}
type ErrWebRPC struct {
Code int ` json:"code" `
Message string ` json:"message" `
Data interface { } ` json:"data" `
}
// Unmarshal response and return the webrpc response
func getTestWebRPCResponse ( resp * httptest . ResponseRecorder , data interface { } ) error {
type rpcReply struct {
ID string ` json:"id" `
JSONRPC string ` json:"jsonrpc" `
Result interface { } ` json:"result" `
Error * ErrWebRPC ` json:"error" `
}
reply := & rpcReply { Result : & data }
err := json . NewDecoder ( resp . Body ) . Decode ( reply )
if err != nil {
return err
}
// For the moment, web handlers errors code are not meaningful
// Return only the error message
if reply . Error != nil {
return errors . New ( reply . Error . Message )
}
return nil
}
2016-06-21 21:10:18 +02:00
// creates the temp backend setup.
// if the option is
// FS: Returns a temp single disk setup initializes FS Backend.
// XL: Returns a 16 temp single disk setup and initializse XL Backend.
2016-08-31 04:22:27 +02:00
func makeTestBackend ( disks [ ] string , instanceType string ) ( ObjectLayer , error ) {
2016-06-21 21:10:18 +02:00
switch instanceType {
case "FS" :
2016-08-31 04:22:27 +02:00
objLayer , err := getSingleNodeObjectLayer ( disks [ 0 ] )
2016-06-21 21:10:18 +02:00
if err != nil {
2016-08-31 04:22:27 +02:00
return nil , err
2016-06-21 21:10:18 +02:00
}
2016-08-31 04:22:27 +02:00
return objLayer , err
2016-06-21 21:10:18 +02:00
case "XL" :
2016-09-15 08:53:42 +02:00
objectLayer , err := getXLObjectLayer ( disks , nil )
2016-06-21 21:10:18 +02:00
if err != nil {
2016-08-31 04:22:27 +02:00
return nil , err
2016-06-21 21:10:18 +02:00
}
2016-08-31 04:22:27 +02:00
return objectLayer , err
2016-06-21 21:10:18 +02:00
default :
errMsg := "Invalid instance type, Only FS and XL are valid options"
2016-08-31 04:22:27 +02:00
return nil , fmt . Errorf ( "Failed obtaining Temp XL layer: <ERROR> %s" , errMsg )
2016-06-21 21:10:18 +02:00
}
}
2016-06-26 04:07:44 +02:00
var src = rand . NewSource ( time . Now ( ) . UTC ( ) . UnixNano ( ) )
// Function to generate random string for bucket/object names.
func randString ( n int ) string {
b := make ( [ ] byte , n )
// A rand.Int63() generates 63 random bits, enough for letterIdxMax letters!
for i , cache , remain := n - 1 , src . Int63 ( ) , letterIdxMax ; i >= 0 ; {
if remain == 0 {
cache , remain = src . Int63 ( ) , letterIdxMax
}
if idx := int ( cache & letterIdxMask ) ; idx < len ( letterBytes ) {
b [ i ] = letterBytes [ idx ]
i --
}
cache >>= letterIdxBits
remain --
}
return string ( b )
}
2016-07-08 23:28:06 +02:00
// generate random object name.
func getRandomObjectName ( ) string {
return randString ( 16 )
}
2016-06-26 04:07:44 +02:00
// generate random bucket name.
func getRandomBucketName ( ) string {
return randString ( 60 )
}
2016-07-30 22:36:43 +02:00
// TruncateWriter - Writes `n` bytes, then returns with number of bytes written.
// differs from iotest.TruncateWriter, the difference is commented in the Write method.
func TruncateWriter ( w io . Writer , n int64 ) io . Writer {
return & truncateWriter { w , n }
}
type truncateWriter struct {
w io . Writer
n int64
}
func ( t * truncateWriter ) Write ( p [ ] byte ) ( n int , err error ) {
if t . n <= 0 {
return len ( p ) , nil
}
// real write
n = len ( p )
if int64 ( n ) > t . n {
n = int ( t . n )
}
n , err = t . w . Write ( p [ 0 : n ] )
t . n -= int64 ( n )
// Removed from iotest.TruncateWriter.
// Need the Write method to return truncated number of bytes written, not the size of the buffer requested to be written.
// if err == nil {
// n = len(p)
// }
return
}
2016-07-09 03:26:04 +02:00
// NewEOFWriter returns a Writer that writes to w,
// but returns EOF error after writing n bytes.
func NewEOFWriter ( w io . Writer , n int64 ) io . Writer {
return & EOFWriter { w , n }
}
type EOFWriter struct {
w io . Writer
n int64
}
2016-07-10 02:11:08 +02:00
// io.Writer implementation designed to error out with io.EOF after reading `n` bytes.
2016-07-09 03:26:04 +02:00
func ( t * EOFWriter ) Write ( p [ ] byte ) ( n int , err error ) {
if t . n <= 0 {
return - 1 , io . EOF
}
// real write
n = len ( p )
if int64 ( n ) > t . n {
n = int ( t . n )
}
n , err = t . w . Write ( p [ 0 : n ] )
t . n -= int64 ( n )
if err == nil {
n = len ( p )
}
return
}
2016-06-26 04:07:44 +02:00
// queryEncode - encodes query values in their URL encoded form.
func queryEncode ( v url . Values ) string {
if v == nil {
return ""
}
var buf bytes . Buffer
keys := make ( [ ] string , 0 , len ( v ) )
for k := range v {
keys = append ( keys , k )
}
sort . Strings ( keys )
for _ , k := range keys {
vs := v [ k ]
prefix := urlEncodePath ( k ) + "="
for _ , v := range vs {
if buf . Len ( ) > 0 {
buf . WriteByte ( '&' )
}
buf . WriteString ( prefix )
buf . WriteString ( urlEncodePath ( v ) )
}
}
return buf . String ( )
}
// urlEncodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences
//
// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8
// non english characters cannot be parsed due to the nature in which url.Encode() is written
//
// This function on the other hand is a direct replacement for url.Encode() technique to support
// pretty much every UTF-8 character.
func urlEncodePath ( pathName string ) string {
// if object matches reserved string, no need to encode them
reservedNames := regexp . MustCompile ( "^[a-zA-Z0-9-_.~/]+$" )
if reservedNames . MatchString ( pathName ) {
return pathName
}
var encodedPathname string
for _ , s := range pathName {
if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark)
encodedPathname = encodedPathname + string ( s )
continue
}
switch s {
case '-' , '_' , '.' , '~' , '/' : // §2.3 Unreserved characters (mark)
encodedPathname = encodedPathname + string ( s )
continue
default :
len := utf8 . RuneLen ( s )
if len < 0 {
// if utf8 cannot convert return the same string as is
return pathName
}
u := make ( [ ] byte , len )
utf8 . EncodeRune ( u , s )
for _ , r := range u {
hex := hex . EncodeToString ( [ ] byte { r } )
encodedPathname = encodedPathname + "%" + strings . ToUpper ( hex )
}
}
}
return encodedPathname
}
// construct URL for http requests for bucket operations.
func makeTestTargetURL ( endPoint , bucketName , objectName string , queryValues url . Values ) string {
urlStr := endPoint + "/"
if bucketName != "" {
urlStr = urlStr + bucketName + "/"
}
if objectName != "" {
urlStr = urlStr + urlEncodePath ( objectName )
}
if len ( queryValues ) > 0 {
urlStr = urlStr + "?" + queryEncode ( queryValues )
}
return urlStr
}
// return URL for uploading object into the bucket.
func getPutObjectURL ( endPoint , bucketName , objectName string ) string {
return makeTestTargetURL ( endPoint , bucketName , objectName , url . Values { } )
}
// return URL for fetching object from the bucket.
func getGetObjectURL ( endPoint , bucketName , objectName string ) string {
return makeTestTargetURL ( endPoint , bucketName , objectName , url . Values { } )
}
// return URL for deleting the object from the bucket.
func getDeleteObjectURL ( endPoint , bucketName , objectName string ) string {
return makeTestTargetURL ( endPoint , bucketName , objectName , url . Values { } )
}
2016-09-02 10:59:08 +02:00
// return URL for deleting multiple objects from a bucket.
func getMultiDeleteObjectURL ( endPoint , bucketName string ) string {
queryValue := url . Values { }
queryValue . Set ( "delete" , "" )
return makeTestTargetURL ( endPoint , bucketName , "" , queryValue )
}
2016-08-17 04:24:23 +02:00
// return URL for HEAD on the object.
2016-06-26 04:07:44 +02:00
func getHeadObjectURL ( endPoint , bucketName , objectName string ) string {
return makeTestTargetURL ( endPoint , bucketName , objectName , url . Values { } )
}
2016-08-17 04:24:23 +02:00
// return url to be used while copying the object.
func getCopyObjectURL ( endPoint , bucketName , objectName string ) string {
return makeTestTargetURL ( endPoint , bucketName , objectName , url . Values { } )
}
2016-08-05 07:01:58 +02:00
// return URL for inserting bucket notification.
func getPutNotificationURL ( endPoint , bucketName string ) string {
queryValue := url . Values { }
queryValue . Set ( "notification" , "" )
return makeTestTargetURL ( endPoint , bucketName , "" , queryValue )
}
// return URL for fetching bucket notification.
func getGetNotificationURL ( endPoint , bucketName string ) string {
queryValue := url . Values { }
queryValue . Set ( "notification" , "" )
return makeTestTargetURL ( endPoint , bucketName , "" , queryValue )
}
2016-06-26 04:07:44 +02:00
// return URL for inserting bucket policy.
func getPutPolicyURL ( endPoint , bucketName string ) string {
queryValue := url . Values { }
queryValue . Set ( "policy" , "" )
return makeTestTargetURL ( endPoint , bucketName , "" , queryValue )
}
// return URL for fetching bucket policy.
func getGetPolicyURL ( endPoint , bucketName string ) string {
queryValue := url . Values { }
queryValue . Set ( "policy" , "" )
return makeTestTargetURL ( endPoint , bucketName , "" , queryValue )
}
// return URL for deleting bucket policy.
func getDeletePolicyURL ( endPoint , bucketName string ) string {
2016-07-04 07:35:30 +02:00
queryValue := url . Values { }
queryValue . Set ( "policy" , "" )
return makeTestTargetURL ( endPoint , bucketName , "" , queryValue )
2016-06-26 04:07:44 +02:00
}
// return URL for creating the bucket.
func getMakeBucketURL ( endPoint , bucketName string ) string {
return makeTestTargetURL ( endPoint , bucketName , "" , url . Values { } )
}
// return URL for listing buckets.
func getListBucketURL ( endPoint string ) string {
return makeTestTargetURL ( endPoint , "" , "" , url . Values { } )
}
// return URL for HEAD on the bucket.
func getHEADBucketURL ( endPoint , bucketName string ) string {
return makeTestTargetURL ( endPoint , bucketName , "" , url . Values { } )
}
// return URL for deleting the bucket.
func getDeleteBucketURL ( endPoint , bucketName string ) string {
return makeTestTargetURL ( endPoint , bucketName , "" , url . Values { } )
2016-09-09 19:18:38 +02:00
}
2016-06-26 04:07:44 +02:00
2016-09-09 19:18:38 +02:00
// return URL For fetching location of the bucket.
func getBucketLocationURL ( endPoint , bucketName string ) string {
queryValue := url . Values { }
queryValue . Set ( "location" , "" )
return makeTestTargetURL ( endPoint , bucketName , "" , queryValue )
2016-06-26 04:07:44 +02:00
}
2016-07-17 21:32:05 +02:00
// return URL for listing objects in the bucket with V1 legacy API.
func getListObjectsV1URL ( endPoint , bucketName string , maxKeys string ) string {
2016-06-28 08:54:56 +02:00
queryValue := url . Values { }
if maxKeys != "" {
queryValue . Set ( "max-keys" , maxKeys )
}
return makeTestTargetURL ( endPoint , bucketName , "" , queryValue )
}
2016-07-17 21:32:05 +02:00
// return URL for listing objects in the bucket with V2 API.
2016-09-10 19:44:38 +02:00
func getListObjectsV2URL ( endPoint , bucketName string , maxKeys string , fetchOwner string ) string {
2016-07-17 21:32:05 +02:00
queryValue := url . Values { }
queryValue . Set ( "list-type" , "2" ) // Enables list objects V2 URL.
if maxKeys != "" {
queryValue . Set ( "max-keys" , maxKeys )
}
2016-09-10 19:44:38 +02:00
if fetchOwner != "" {
queryValue . Set ( "fetch-owner" , fetchOwner )
}
2016-07-17 21:32:05 +02:00
return makeTestTargetURL ( endPoint , bucketName , "" , queryValue )
}
2016-06-28 08:54:56 +02:00
// return URL for a new multipart upload.
func getNewMultipartURL ( endPoint , bucketName , objectName string ) string {
queryValue := url . Values { }
queryValue . Set ( "uploads" , "" )
return makeTestTargetURL ( endPoint , bucketName , objectName , queryValue )
}
// return URL for a new multipart upload.
func getPartUploadURL ( endPoint , bucketName , objectName , uploadID , partNumber string ) string {
queryValues := url . Values { }
queryValues . Set ( "uploadId" , uploadID )
queryValues . Set ( "partNumber" , partNumber )
return makeTestTargetURL ( endPoint , bucketName , objectName , queryValues )
}
// return URL for aborting multipart upload.
func getAbortMultipartUploadURL ( endPoint , bucketName , objectName , uploadID string ) string {
queryValue := url . Values { }
queryValue . Set ( "uploadId" , uploadID )
return makeTestTargetURL ( endPoint , bucketName , objectName , queryValue )
}
2016-09-14 04:00:01 +02:00
// return URL for a listing pending multipart uploads.
2016-06-28 08:54:56 +02:00
func getListMultipartURL ( endPoint , bucketName string ) string {
queryValue := url . Values { }
queryValue . Set ( "uploads" , "" )
return makeTestTargetURL ( endPoint , bucketName , "" , queryValue )
}
2016-09-14 04:00:01 +02:00
// return URL for listing pending multipart uploads with parameters.
func getListMultipartUploadsURLWithParams ( endPoint , bucketName , prefix , keyMarker , uploadIDMarker , delimiter , maxUploads string ) string {
queryValue := url . Values { }
queryValue . Set ( "uploads" , "" )
queryValue . Set ( "prefix" , prefix )
queryValue . Set ( "delimiter" , delimiter )
queryValue . Set ( "key-marker" , keyMarker )
queryValue . Set ( "upload-id-marker" , uploadIDMarker )
queryValue . Set ( "max-uploads" , maxUploads )
return makeTestTargetURL ( endPoint , bucketName , "" , queryValue )
}
// return URL for a listing parts on a given upload id.
2016-06-28 08:54:56 +02:00
func getListMultipartURLWithParams ( endPoint , bucketName , objectName , uploadID , maxParts string ) string {
queryValues := url . Values { }
queryValues . Set ( "uploadId" , uploadID )
queryValues . Set ( "max-parts" , maxParts )
return makeTestTargetURL ( endPoint , bucketName , objectName , queryValues )
}
// return URL for completing multipart upload.
// complete multipart upload request is sent after all parts are uploaded.
func getCompleteMultipartUploadURL ( endPoint , bucketName , objectName , uploadID string ) string {
queryValue := url . Values { }
queryValue . Set ( "uploadId" , uploadID )
return makeTestTargetURL ( endPoint , bucketName , objectName , queryValue )
}
2016-06-21 21:10:18 +02:00
// returns temp root directory. `
func getTestRoot ( ) ( string , error ) {
return ioutil . TempDir ( os . TempDir ( ) , "api-" )
}
2016-08-31 04:22:27 +02:00
// getRandomDisks - Creates a slice of N random disks, each of the form - minio-XXX
func getRandomDisks ( N int ) ( [ ] string , error ) {
2016-06-08 03:15:04 +02:00
var erasureDisks [ ] string
2016-08-31 04:22:27 +02:00
for i := 0 ; i < N ; i ++ {
2016-06-08 03:15:04 +02:00
path , err := ioutil . TempDir ( os . TempDir ( ) , "minio-" )
2016-05-06 20:57:04 +02:00
if err != nil {
2016-08-31 04:22:27 +02:00
// Remove directories created so far.
removeRoots ( erasureDisks )
return nil , err
2016-05-06 20:57:04 +02:00
}
2016-06-08 03:15:04 +02:00
erasureDisks = append ( erasureDisks , path )
2016-05-06 20:57:04 +02:00
}
2016-08-31 04:22:27 +02:00
return erasureDisks , nil
}
2016-05-06 20:57:04 +02:00
2016-08-31 04:22:27 +02:00
// getXLObjectLayer - Instantiates XL object layer and returns it.
2016-09-15 08:53:42 +02:00
func getXLObjectLayer ( erasureDisks [ ] string , ignoredDisks [ ] string ) ( ObjectLayer , error ) {
err := formatDisks ( erasureDisks , ignoredDisks )
2016-08-31 04:22:27 +02:00
if err != nil {
return nil , err
}
2016-09-15 08:53:42 +02:00
objLayer , err := newXLObjects ( erasureDisks , ignoredDisks )
2016-06-08 03:15:04 +02:00
if err != nil {
2016-08-31 04:22:27 +02:00
return nil , err
2016-06-08 03:15:04 +02:00
}
2016-08-17 04:24:23 +02:00
// Disabling the cache for integration tests.
// Should use the object layer tests for validating cache.
if xl , ok := objLayer . ( xlObjects ) ; ok {
xl . objCacheEnabled = false
}
2016-08-31 04:22:27 +02:00
return objLayer , nil
2016-06-08 03:15:04 +02:00
}
2016-05-21 05:48:47 +02:00
2016-06-08 03:15:04 +02:00
// getSingleNodeObjectLayer - Instantiates single node object layer and returns it.
2016-08-31 04:22:27 +02:00
func getSingleNodeObjectLayer ( disk string ) ( ObjectLayer , error ) {
// Create the object layer.
objLayer , err := newFSObjects ( disk )
2016-06-08 03:15:04 +02:00
if err != nil {
2016-08-31 04:22:27 +02:00
return nil , err
2016-06-08 03:15:04 +02:00
}
2016-08-31 04:22:27 +02:00
return objLayer , nil
2016-06-08 03:15:04 +02:00
}
// removeRoots - Cleans up initialized directories during tests.
func removeRoots ( roots [ ] string ) {
for _ , root := range roots {
2016-06-13 11:53:09 +02:00
removeAll ( root )
2016-05-06 20:57:04 +02:00
}
2016-06-08 03:15:04 +02:00
}
2016-05-06 20:57:04 +02:00
2016-06-29 07:32:00 +02:00
//removeDiskN - removes N disks from supplied disk slice.
func removeDiskN ( disks [ ] string , n int ) {
if n > len ( disks ) {
n = len ( disks )
}
for _ , disk := range disks [ : n ] {
removeAll ( disk )
2016-06-08 03:15:04 +02:00
}
}
2016-09-10 23:47:27 +02:00
// creates a bucket for the tests and returns the bucket name.
// initializes the specified API endpoints for the tests.
// initialies the root and returns its path.
// return credentials.
func initAPIHandlerTest ( obj ObjectLayer , endPoints [ ] string ) ( bucketName , rootPath string , apiRouter http . Handler , err error ) {
// get random bucket name.
bucketName = getRandomBucketName ( )
// Create bucket.
err = obj . MakeBucket ( bucketName )
if err != nil {
// failed to create newbucket, return err.
return "" , "" , nil , err
}
// Register the API end points with XL/FS object layer.
// Registering only the GetObject handler.
apiRouter = initTestAPIEndPoints ( obj , endPoints )
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
rootPath , err = newTestConfig ( "us-east-1" )
if err != nil {
return "" , "" , nil , err
}
return bucketName , rootPath , apiRouter , nil
}
// ExecObjectLayerAPITest - executes object layer API tests.
// Creates single node and XL ObjectLayer instance, registers the specified API end points and runs test for both the layers.
func ExecObjectLayerAPITest ( t TestErrHandler , objAPITest objAPITestType , endPoints [ ] string ) {
objLayer , fsDir , err := prepareFS ( )
if err != nil {
t . Fatalf ( "Initialization of object layer failed for single node setup: %s" , err )
}
bucketFS , fsRoot , fsAPIRouter , err := initAPIHandlerTest ( objLayer , endPoints )
if err != nil {
t . Fatalf ( "Initialzation of API handler tests failed: <ERROR> %s" , err )
}
credentials := serverConfig . GetCredential ( )
// Executing the object layer tests for single node setup.
objAPITest ( objLayer , singleNodeTestStr , bucketFS , fsAPIRouter , credentials , t )
objLayer , xlDisks , err := prepareXL ( )
if err != nil {
t . Fatalf ( "Initialization of object layer failed for XL setup: %s" , err )
}
bucketXL , xlRoot , xlAPIRouter , err := initAPIHandlerTest ( objLayer , endPoints )
if err != nil {
t . Fatalf ( "Initialzation of API handler tests failed: <ERROR> %s" , err )
}
credentials = serverConfig . GetCredential ( )
// Executing the object layer tests for XL.
objAPITest ( objLayer , xLTestStr , bucketXL , xlAPIRouter , credentials , t )
defer removeRoots ( append ( xlDisks , fsDir , fsRoot , xlRoot ) )
}
// function to be passed to ExecObjectLayerAPITest, for executing object layr API handler tests.
type objAPITestType func ( obj ObjectLayer , instanceType string , bucketName string ,
apiRouter http . Handler , credentials credential , t TestErrHandler )
2016-06-08 03:15:04 +02:00
// Regular object test type.
2016-07-08 00:05:51 +02:00
type objTestType func ( obj ObjectLayer , instanceType string , t TestErrHandler )
2016-06-08 03:15:04 +02:00
// Special object test type for disk not found situations.
type objTestDiskNotFoundType func ( obj ObjectLayer , instanceType string , dirs [ ] string , t * testing . T )
// ExecObjectLayerTest - executes object layer tests.
// Creates single node and XL ObjectLayer instance and runs test for both the layers.
2016-07-08 00:05:51 +02:00
func ExecObjectLayerTest ( t TestErrHandler , objTest objTestType ) {
2016-08-31 04:22:27 +02:00
objLayer , fsDir , err := prepareFS ( )
2016-05-06 20:57:04 +02:00
if err != nil {
2016-07-03 04:05:16 +02:00
t . Fatalf ( "Initialization of object layer failed for single node setup: %s" , err )
2016-05-06 20:57:04 +02:00
}
2016-05-26 23:43:17 +02:00
// Executing the object layer tests for single node setup.
objTest ( objLayer , singleNodeTestStr , t )
2016-05-21 05:48:47 +02:00
2016-08-31 04:22:27 +02:00
objLayer , fsDirs , err := prepareXL ( )
2016-05-06 20:57:04 +02:00
if err != nil {
2016-07-03 04:05:16 +02:00
t . Fatalf ( "Initialization of object layer failed for XL setup: %s" , err )
2016-05-06 20:57:04 +02:00
}
// Executing the object layer tests for XL.
objTest ( objLayer , xLTestStr , t )
defer removeRoots ( append ( fsDirs , fsDir ) )
}
2016-06-08 03:15:04 +02:00
// ExecObjectLayerDiskNotFoundTest - executes object layer tests while deleting
// disks in between tests. Creates XL ObjectLayer instance and runs test for XL layer.
func ExecObjectLayerDiskNotFoundTest ( t * testing . T , objTest objTestDiskNotFoundType ) {
2016-08-31 04:22:27 +02:00
objLayer , fsDirs , err := prepareXL ( )
2016-06-08 03:15:04 +02:00
if err != nil {
2016-07-03 04:05:16 +02:00
t . Fatalf ( "Initialization of object layer failed for XL setup: %s" , err )
2016-06-08 03:15:04 +02:00
}
// Executing the object layer tests for XL.
objTest ( objLayer , xLTestStr , fsDirs , t )
defer removeRoots ( fsDirs )
}
2016-06-29 11:28:46 +02:00
// Special object test type for stale files situations.
type objTestStaleFilesType func ( obj ObjectLayer , instanceType string , dirs [ ] string , t * testing . T )
// ExecObjectLayerStaleFilesTest - executes object layer tests those leaves stale
// files/directories under .minio/tmp. Creates XL ObjectLayer instance and runs test for XL layer.
func ExecObjectLayerStaleFilesTest ( t * testing . T , objTest objTestStaleFilesType ) {
2016-08-31 04:22:27 +02:00
nDisks := 16
erasureDisks , err := getRandomDisks ( nDisks )
if err != nil {
t . Fatalf ( "Initialization of disks for XL setup: %s" , err )
}
2016-09-15 08:53:42 +02:00
objLayer , err := getXLObjectLayer ( erasureDisks , nil )
2016-06-29 11:28:46 +02:00
if err != nil {
2016-07-03 04:05:16 +02:00
t . Fatalf ( "Initialization of object layer failed for XL setup: %s" , err )
2016-06-29 11:28:46 +02:00
}
// Executing the object layer tests for XL.
2016-08-31 04:22:27 +02:00
objTest ( objLayer , xLTestStr , erasureDisks , t )
defer removeRoots ( erasureDisks )
2016-06-29 11:28:46 +02:00
}
2016-07-03 04:05:16 +02:00
// Takes in XL/FS object layer, and the list of API end points to be tested/required, registers the API end points and returns the HTTP handler.
// Need isolated registration of API end points while writing unit tests for end points.
// All the API end points are registered only for the default case.
func initTestAPIEndPoints ( objLayer ObjectLayer , apiFunctions [ ] string ) http . Handler {
// initialize a new mux router.
// goriilla/mux is the library used to register all the routes and handle them.
muxRouter := router . NewRouter ( )
// All object storage operations are registered as HTTP handlers on `objectAPIHandlers`.
// When the handlers get a HTTP request they use the underlyting ObjectLayer to perform operations.
api := objectAPIHandlers {
2016-07-31 23:11:14 +02:00
ObjectAPI : func ( ) ObjectLayer { return objLayer } ,
2016-07-03 04:05:16 +02:00
}
// API Router.
apiRouter := muxRouter . NewRoute ( ) . PathPrefix ( "/" ) . Subrouter ( )
// Bucket router.
bucket := apiRouter . PathPrefix ( "/{bucket}" ) . Subrouter ( )
// Iterate the list of API functions requested for and register them in mux HTTP handler.
for _ , apiFunction := range apiFunctions {
switch apiFunction {
2016-09-15 08:53:42 +02:00
// Register ListBuckets handler.
case "ListBuckets" :
apiRouter . Methods ( "GET" ) . HandlerFunc ( api . ListBucketsHandler )
2016-08-17 04:24:23 +02:00
// Register GetObject handler.
case "GetObject`" :
bucket . Methods ( "GET" ) . Path ( "/{object:.+}" ) . HandlerFunc ( api . GetObjectHandler )
// Register GetObject handler.
case "CopyObject`" :
bucket . Methods ( "PUT" ) . Path ( "/{object:.+}" ) . HeadersRegexp ( "X-Amz-Copy-Source" , ".*?(\\/|%2F).*?" ) . HandlerFunc ( api . CopyObjectHandler )
2016-07-03 04:05:16 +02:00
// Register PutBucket Policy handler.
case "PutBucketPolicy" :
bucket . Methods ( "PUT" ) . HandlerFunc ( api . PutBucketPolicyHandler ) . Queries ( "policy" , "" )
2016-08-17 04:24:23 +02:00
// Register Delete bucket HTTP policy handler.
2016-07-03 04:05:16 +02:00
case "DeleteBucketPolicy" :
bucket . Methods ( "DELETE" ) . HandlerFunc ( api . DeleteBucketPolicyHandler ) . Queries ( "policy" , "" )
2016-09-09 19:18:38 +02:00
// Register Get Bucket policy HTTP Handler.
2016-07-03 04:05:16 +02:00
case "GetBucketPolicy" :
bucket . Methods ( "GET" ) . HandlerFunc ( api . GetBucketPolicyHandler ) . Queries ( "policy" , "" )
2016-09-09 19:18:38 +02:00
// Register GetBucketLocation handler.
case "GetBucketLocation" :
bucket . Methods ( "GET" ) . HandlerFunc ( api . GetBucketLocationHandler ) . Queries ( "location" , "" )
2016-09-14 04:00:01 +02:00
// Register HeadBucket handler.
2016-09-09 19:18:38 +02:00
case "HeadBucket" :
bucket . Methods ( "HEAD" ) . HandlerFunc ( api . HeadBucketHandler )
2016-09-14 04:00:01 +02:00
// Register ListMultipartUploads handler.
case "ListMultipartUploads" :
bucket . Methods ( "GET" ) . HandlerFunc ( api . ListMultipartUploadsHandler ) . Queries ( "uploads" , "" )
2016-09-09 19:18:38 +02:00
// Register all api endpoints by default.
2016-07-03 04:05:16 +02:00
default :
registerAPIRouter ( muxRouter , api )
// No need to register any more end points, all the end points are registered.
break
}
}
return muxRouter
}
2016-08-16 01:13:03 +02:00
2016-08-29 04:31:59 +02:00
// Initialize Web RPC Handlers for testing
2016-08-16 01:13:03 +02:00
func initTestWebRPCEndPoint ( objLayer ObjectLayer ) http . Handler {
// Initialize Web.
webHandlers := & webAPIHandlers {
2016-08-16 22:14:05 +02:00
ObjectAPI : func ( ) ObjectLayer { return objLayer } ,
2016-08-16 01:13:03 +02:00
}
// Initialize router.
muxRouter := router . NewRouter ( )
registerWebRouter ( muxRouter , webHandlers )
return muxRouter
}