diff --git a/cmd/api-errors.go b/cmd/api-errors.go index ca8800e20..0632682a5 100644 --- a/cmd/api-errors.go +++ b/cmd/api-errors.go @@ -134,6 +134,7 @@ const ( ErrObjectExistsAsDirectory ErrPolicyNesting ErrInvalidObjectName + ErrServerNotInitialized // Add new extended error codes here. // Please open a https://github.com/minio/minio/issues before adding // new error codes here. @@ -454,7 +455,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{ Description: "Request is not valid yet", HTTPStatusCode: http.StatusForbidden, }, - // FIXME: Actual XML error response also contains the header which missed in lsit of signed header parameters. + // FIXME: Actual XML error response also contains the header which missed in list of signed header parameters. ErrUnsignedHeaders: { Code: "AccessDenied", Description: "There were headers present in the request which were not signed", @@ -556,6 +557,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{ Description: "Object name contains unsupported characters. Unsupported characters are `^*|\\\"", HTTPStatusCode: http.StatusBadRequest, }, + ErrServerNotInitialized: { + Code: "XMinioServerNotInitialized", + Description: "Server not initialized, please try again.", + HTTPStatusCode: http.StatusServiceUnavailable, + }, // Add your error structure here. } @@ -566,6 +572,7 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) { if err == nil { return ErrNone } + err = errorCause(err) // Verify if the underlying error is signature mismatch. switch err { case errSignatureMismatch: diff --git a/cmd/api-response.go b/cmd/api-response.go index 68aa005cc..a797905a3 100644 --- a/cmd/api-response.go +++ b/cmd/api-response.go @@ -24,10 +24,10 @@ import ( ) const ( - timeFormatAMZ = "2006-01-02T15:04:05.000Z" // Reply date format - maxObjectList = 1000 // Limit number of objects in a listObjectsResponse. - maxUploadsList = 1000 // Limit number of uploads in a listUploadsResponse. - maxPartsList = 1000 // Limit number of parts in a listPartsResponse. + timeFormatAMZ = "2006-01-02T15:04:05Z" // Reply date format + maxObjectList = 1000 // Limit number of objects in a listObjectsResponse. + maxUploadsList = 1000 // Limit number of uploads in a listUploadsResponse. + maxPartsList = 1000 // Limit number of parts in a listPartsResponse. ) // LocationResponse - format for location response. diff --git a/cmd/api-router.go b/cmd/api-router.go index 40e282e76..fe2442d14 100644 --- a/cmd/api-router.go +++ b/cmd/api-router.go @@ -20,7 +20,7 @@ import router "github.com/gorilla/mux" // objectAPIHandler implements and provides http handlers for S3 API. type objectAPIHandlers struct { - ObjectAPI ObjectLayer + ObjectAPI func() ObjectLayer } // registerAPIRouter - registers S3 compatible APIs. diff --git a/cmd/auth-rpc-client.go b/cmd/auth-rpc-client.go new file mode 100644 index 000000000..1ea8014d4 --- /dev/null +++ b/cmd/auth-rpc-client.go @@ -0,0 +1,163 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "fmt" + "net/rpc" + "time" + + jwtgo "github.com/dgrijalva/jwt-go" +) + +// GenericReply represents any generic RPC reply. +type GenericReply struct{} + +// GenericArgs represents any generic RPC arguments. +type GenericArgs struct { + Token string // Used to authenticate every RPC call. + Timestamp time.Time // Used to verify if the RPC call was issued between the same Login() and disconnect event pair. +} + +// SetToken - sets the token to the supplied value. +func (ga *GenericArgs) SetToken(token string) { + ga.Token = token +} + +// SetTimestamp - sets the timestamp to the supplied value. +func (ga *GenericArgs) SetTimestamp(tstamp time.Time) { + ga.Timestamp = tstamp +} + +// RPCLoginArgs - login username and password for RPC. +type RPCLoginArgs struct { + Username string + Password string +} + +// RPCLoginReply - login reply provides generated token to be used +// with subsequent requests. +type RPCLoginReply struct { + Token string + ServerVersion string + Timestamp time.Time +} + +// Validates if incoming token is valid. +func isRPCTokenValid(tokenStr string) bool { + jwt, err := newJWT(defaultTokenExpiry) // Expiry set to 100yrs. + if err != nil { + errorIf(err, "Unable to initialize JWT") + return false + } + token, err := jwtgo.Parse(tokenStr, func(token *jwtgo.Token) (interface{}, error) { + if _, ok := token.Method.(*jwtgo.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"]) + } + return []byte(jwt.SecretAccessKey), nil + }) + if err != nil { + errorIf(err, "Unable to parse JWT token string") + return false + } + // Return if token is valid. + return token.Valid +} + +// Auth config represents authentication credentials and Login method name to be used +// for fetching JWT tokens from the RPC server. +type authConfig struct { + accessKey string // Username for the server. + secretKey string // Password for the server. + address string // Network address path of RPC server. + path string // Network path for HTTP dial. + loginMethod string // RPC service name for authenticating using JWT +} + +// AuthRPCClient is a wrapper type for RPCClient which provides JWT based authentication across reconnects. +type AuthRPCClient struct { + config *authConfig + rpc *RPCClient // reconnect'able rpc client built on top of net/rpc Client + isLoggedIn bool // Indicates if the auth client has been logged in and token is valid. + token string // JWT based token + tstamp time.Time // Timestamp as received on Login RPC. +} + +// newAuthClient - returns a jwt based authenticated (go) rpc client, which does automatic reconnect. +func newAuthClient(cfg *authConfig) *AuthRPCClient { + return &AuthRPCClient{ + // Save the config. + config: cfg, + // Initialize a new reconnectable rpc client. + rpc: newClient(cfg.address, cfg.path), + // Allocated auth client not logged in yet. + isLoggedIn: false, + } +} + +// Close - closes underlying rpc connection. +func (authClient *AuthRPCClient) Close() error { + // reset token on closing a connection + authClient.isLoggedIn = false + return authClient.rpc.Close() +} + +// Login - a jwt based authentication is performed with rpc server. +func (authClient *AuthRPCClient) Login() error { + // Return if already logged in. + if authClient.isLoggedIn { + return nil + } + reply := RPCLoginReply{} + if err := authClient.rpc.Call(authClient.config.loginMethod, RPCLoginArgs{ + Username: authClient.config.accessKey, + Password: authClient.config.secretKey, + }, &reply); err != nil { + return err + } + // Set token, time stamp as received from a successful login call. + authClient.token = reply.Token + authClient.tstamp = reply.Timestamp + authClient.isLoggedIn = true + return nil +} + +// Call - If rpc connection isn't established yet since previous disconnect, +// connection is established, a jwt authenticated login is performed and then +// the call is performed. +func (authClient *AuthRPCClient) Call(serviceMethod string, args interface { + SetToken(token string) + SetTimestamp(tstamp time.Time) +}, reply interface{}) (err error) { + // On successful login, attempt the call. + if err = authClient.Login(); err == nil { + // Set token and timestamp before the rpc call. + args.SetToken(authClient.token) + args.SetTimestamp(authClient.tstamp) + + // Call the underlying rpc. + err = authClient.rpc.Call(serviceMethod, args, reply) + + // Invalidate token to mark for re-login on subsequent reconnect. + if err != nil { + if err.Error() == rpc.ErrShutdown.Error() { + authClient.isLoggedIn = false + } + } + } + return err +} diff --git a/cmd/benchmark-utils_test.go b/cmd/benchmark-utils_test.go index 419e84d7c..5f8e5711d 100644 --- a/cmd/benchmark-utils_test.go +++ b/cmd/benchmark-utils_test.go @@ -28,6 +28,20 @@ import ( "time" ) +// Prepare benchmark backend +func prepareBenchmarkBackend(instanceType string) (ObjectLayer, []string, error) { + nDisks := 16 + disks, err := getRandomDisks(nDisks) + if err != nil { + return nil, nil, err + } + obj, err := makeTestBackend(disks, instanceType) + if err != nil { + return nil, nil, err + } + return obj, disks, nil +} + // Benchmark utility functions for ObjectLayer.PutObject(). // Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark. func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) { @@ -40,9 +54,6 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) { b.Fatal(err) } - // PutObject returns md5Sum of the object inserted. - // md5Sum variable is assigned with that value. - var md5Sum string // get text data generated for number of bytes equal to object size. textData := generateBytesData(objSize) // generate md5sum for the generated data. @@ -57,12 +68,12 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) { b.ResetTimer() for i := 0; i < b.N; i++ { // insert the object. - md5Sum, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata) + objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata) if err != nil { b.Fatal(err) } - if md5Sum != metadata["md5Sum"] { - b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"]) + if objInfo.MD5Sum != metadata["md5Sum"] { + b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.MD5Sum, metadata["md5Sum"]) } } // Benchmark ends here. Stop timer. @@ -135,7 +146,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) { // creates XL/FS backend setup, obtains the object layer and calls the runPutObjectPartBenchmark function. func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) { // create a temp XL/FS backend. - objLayer, disks, err := makeTestBackend(instanceType) + objLayer, disks, err := prepareBenchmarkBackend(instanceType) if err != nil { b.Fatalf("Failed obtaining Temp Backend: %s", err) } @@ -148,7 +159,7 @@ func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) { // creates XL/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function. func benchmarkPutObject(b *testing.B, instanceType string, objSize int) { // create a temp XL/FS backend. - objLayer, disks, err := makeTestBackend(instanceType) + objLayer, disks, err := prepareBenchmarkBackend(instanceType) if err != nil { b.Fatalf("Failed obtaining Temp Backend: %s", err) } @@ -161,7 +172,7 @@ func benchmarkPutObject(b *testing.B, instanceType string, objSize int) { // creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for put object. func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int) { // create a temp XL/FS backend. - objLayer, disks, err := makeTestBackend(instanceType) + objLayer, disks, err := prepareBenchmarkBackend(instanceType) if err != nil { b.Fatalf("Failed obtaining Temp Backend: %s", err) } @@ -183,9 +194,6 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) { b.Fatal(err) } - // PutObject returns md5Sum of the object inserted. - // md5Sum variable is assigned with that value. - var md5Sum string for i := 0; i < 10; i++ { // get text data generated for number of bytes equal to object size. textData := generateBytesData(objSize) @@ -197,12 +205,13 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) { metadata := make(map[string]string) metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil)) // insert the object. - md5Sum, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata) + var objInfo ObjectInfo + objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata) if err != nil { b.Fatal(err) } - if md5Sum != metadata["md5Sum"] { - b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"]) + if objInfo.MD5Sum != metadata["md5Sum"] { + b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.MD5Sum, metadata["md5Sum"]) } } @@ -242,7 +251,7 @@ func generateBytesData(size int) []byte { // creates XL/FS backend setup, obtains the object layer and calls the runGetObjectBenchmark function. func benchmarkGetObject(b *testing.B, instanceType string, objSize int) { // create a temp XL/FS backend. - objLayer, disks, err := makeTestBackend(instanceType) + objLayer, disks, err := prepareBenchmarkBackend(instanceType) if err != nil { b.Fatalf("Failed obtaining Temp Backend: %s", err) } @@ -255,7 +264,7 @@ func benchmarkGetObject(b *testing.B, instanceType string, objSize int) { // creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for ObjectLayer.GetObject() . func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int) { // create a temp XL/FS backend. - objLayer, disks, err := makeTestBackend(instanceType) + objLayer, disks, err := prepareBenchmarkBackend(instanceType) if err != nil { b.Fatalf("Failed obtaining Temp Backend: %s", err) } @@ -277,9 +286,6 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) { b.Fatal(err) } - // PutObject returns md5Sum of the object inserted. - // md5Sum variable is assigned with that value. - var md5Sum string // get text data generated for number of bytes equal to object size. textData := generateBytesData(objSize) // generate md5sum for the generated data. @@ -297,12 +303,12 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) { i := 0 for pb.Next() { // insert the object. - md5Sum, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata) + objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata) if err != nil { b.Fatal(err) } - if md5Sum != metadata["md5Sum"] { - b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", md5Sum, metadata["md5Sum"]) + if objInfo.MD5Sum != metadata["md5Sum"] { + b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", objInfo.MD5Sum, metadata["md5Sum"]) } i++ } @@ -324,9 +330,6 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) { b.Fatal(err) } - // PutObject returns md5Sum of the object inserted. - // md5Sum variable is assigned with that value. - var md5Sum string for i := 0; i < 10; i++ { // get text data generated for number of bytes equal to object size. textData := generateBytesData(objSize) @@ -338,12 +341,13 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) { metadata := make(map[string]string) metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil)) // insert the object. - md5Sum, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata) + var objInfo ObjectInfo + objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata) if err != nil { b.Fatal(err) } - if md5Sum != metadata["md5Sum"] { - b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"]) + if objInfo.MD5Sum != metadata["md5Sum"] { + b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.MD5Sum, metadata["md5Sum"]) } } diff --git a/cmd/bucket-handlers-listobjects.go b/cmd/bucket-handlers-listobjects.go index dd406e711..8e9dcd4ee 100644 --- a/cmd/bucket-handlers-listobjects.go +++ b/cmd/bucket-handlers-listobjects.go @@ -64,6 +64,12 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http vars := mux.Vars(r) bucket := vars["bucket"] + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + switch getRequestAuthType(r) { default: // For all unknown auth types return error. @@ -100,7 +106,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http // Inititate a list objects operation based on the input params. // On success would return back ListObjectsInfo object to be // marshalled into S3 compatible XML header. - listObjectsInfo, err := api.ObjectAPI.ListObjects(bucket, prefix, marker, delimiter, maxKeys) + listObjectsInfo, err := objectAPI.ListObjects(bucket, prefix, marker, delimiter, maxKeys) if err != nil { errorIf(err, "Unable to list objects.") writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) @@ -124,6 +130,12 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http vars := mux.Vars(r) bucket := vars["bucket"] + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + switch getRequestAuthType(r) { default: // For all unknown auth types return error. @@ -154,7 +166,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http // Inititate a list objects operation based on the input params. // On success would return back ListObjectsInfo object to be // marshalled into S3 compatible XML header. - listObjectsInfo, err := api.ObjectAPI.ListObjects(bucket, prefix, marker, delimiter, maxKeys) + listObjectsInfo, err := objectAPI.ListObjects(bucket, prefix, marker, delimiter, maxKeys) if err != nil { errorIf(err, "Unable to list objects.") writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index 83a21cfcb..14de46e68 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -21,7 +21,6 @@ import ( "io" "net/http" "net/url" - "path" "strings" "sync" @@ -64,6 +63,12 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r * vars := mux.Vars(r) bucket := vars["bucket"] + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + switch getRequestAuthType(r) { default: // For all unknown auth types return error. @@ -82,7 +87,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r * } } - if _, err := api.ObjectAPI.GetBucketInfo(bucket); err != nil { + if _, err := objectAPI.GetBucketInfo(bucket); err != nil { errorIf(err, "Unable to fetch bucket info.") writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) return @@ -113,6 +118,12 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, vars := mux.Vars(r) bucket := vars["bucket"] + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + switch getRequestAuthType(r) { default: // For all unknown auth types return error. @@ -144,7 +155,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, } } - listMultipartsInfo, err := api.ObjectAPI.ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads) + listMultipartsInfo, err := objectAPI.ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads) if err != nil { errorIf(err, "Unable to list multipart uploads.") writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) @@ -164,13 +175,20 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, // This implementation of the GET operation returns a list of all buckets // owned by the authenticated sender of the request. func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + // List buckets does not support bucket policies, no need to enforce it. if s3Error := checkAuth(r); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } - bucketsInfo, err := api.ObjectAPI.ListBuckets() + // Invoke the list buckets. + bucketsInfo, err := objectAPI.ListBuckets() if err != nil { errorIf(err, "Unable to list buckets.") writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) @@ -191,6 +209,12 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, vars := mux.Vars(r) bucket := vars["bucket"] + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + switch getRequestAuthType(r) { default: // For all unknown auth types return error. @@ -249,7 +273,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, wg.Add(1) go func(i int, obj ObjectIdentifier) { defer wg.Done() - dErr := api.ObjectAPI.DeleteObject(bucket, obj.ObjectName) + dErr := objectAPI.DeleteObject(bucket, obj.ObjectName) if dErr != nil { dErrs[i] = dErr } @@ -267,7 +291,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, deletedObjects = append(deletedObjects, object) continue } - if _, ok := err.(ObjectNotFound); ok { + if _, ok := errorCause(err).(ObjectNotFound); ok { // If the object is not found it should be // accounted as deleted as per S3 spec. deletedObjects = append(deletedObjects, object) @@ -311,6 +335,12 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, // ---------- // This implementation of the PUT operation creates a new bucket for authenticated request func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) { + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + // PutBucket does not support policies, use checkAuth to validate signature. if s3Error := checkAuth(r); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) @@ -328,7 +358,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req } // Proceed to creating a bucket. - err := api.ObjectAPI.MakeBucket(bucket) + err := objectAPI.MakeBucket(bucket) if err != nil { errorIf(err, "Unable to create a bucket.") writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) @@ -344,6 +374,12 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req // This implementation of the POST operation handles object creation with a specified // signature policy in multipart/form-data func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) { + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + // Here the parameter is the size of the form data that should // be loaded in memory, the remaining being put in temporary files. reader, err := r.MultipartReader() @@ -384,17 +420,13 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h metadata := make(map[string]string) // Nothing to store right now. - md5Sum, err := api.ObjectAPI.PutObject(bucket, object, -1, fileBody, metadata) + objInfo, err := objectAPI.PutObject(bucket, object, -1, fileBody, metadata) if err != nil { errorIf(err, "Unable to create object.") writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) return } - if md5Sum != "" { - w.Header().Set("ETag", "\""+md5Sum+"\"") - } - - // TODO full URL is preferred. + w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"") w.Header().Set("Location", getObjectLocation(bucket, object)) // Set common headers. @@ -404,13 +436,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h writeSuccessNoContent(w) if globalEventNotifier.IsBucketNotificationSet(bucket) { - // Fetch object info for notifications. - objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object) - if err != nil { - errorIf(err, "Unable to fetch object info for \"%s\"", path.Join(bucket, object)) - return - } - // Notify object created event. eventNotify(eventData{ Type: ObjectCreatedPost, @@ -433,6 +458,12 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re vars := mux.Vars(r) bucket := vars["bucket"] + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + switch getRequestAuthType(r) { default: // For all unknown auth types return error. @@ -451,7 +482,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re } } - if _, err := api.ObjectAPI.GetBucketInfo(bucket); err != nil { + if _, err := objectAPI.GetBucketInfo(bucket); err != nil { errorIf(err, "Unable to fetch bucket info.") writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) return @@ -461,6 +492,12 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re // DeleteBucketHandler - Delete bucket func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) { + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + // DeleteBucket does not support bucket policies, use checkAuth to validate signature. if s3Error := checkAuth(r); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) @@ -471,17 +508,17 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http. bucket := vars["bucket"] // Attempt to delete bucket. - if err := api.ObjectAPI.DeleteBucket(bucket); err != nil { + if err := objectAPI.DeleteBucket(bucket); err != nil { errorIf(err, "Unable to delete a bucket.") writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) return } // Delete bucket access policy, if present - ignore any errors. - removeBucketPolicy(bucket, api.ObjectAPI) + removeBucketPolicy(bucket, objectAPI) // Delete notification config, if present - ignore any errors. - removeNotificationConfig(bucket, api.ObjectAPI) + removeNotificationConfig(bucket, objectAPI) // Write success response. writeSuccessNoContent(w) diff --git a/cmd/bucket-handlers_test.go b/cmd/bucket-handlers_test.go new file mode 100644 index 000000000..82f1e2586 --- /dev/null +++ b/cmd/bucket-handlers_test.go @@ -0,0 +1,302 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "bytes" + "encoding/xml" + "net/http" + "net/http/httptest" + "testing" +) + +// Wrapper for calling GetBucketPolicy HTTP handler tests for both XL multiple disks and single node setup. +func TestGetBucketLocationHandler(t *testing.T) { + ExecObjectLayerTest(t, testGetBucketLocationHandler) +} + +func testGetBucketLocationHandler(obj ObjectLayer, instanceType string, t TestErrHandler) { + initBucketPolicies(obj) + + // get random bucket name. + bucketName := getRandomBucketName() + // Create bucket. + err := obj.MakeBucket(bucketName) + if err != nil { + // failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err) + } + // Register the API end points with XL/FS object layer. + apiRouter := initTestAPIEndPoints(obj, []string{"GetBucketLocation"}) + // initialize the server and obtain the credentials and root. + // credentials are necessary to sign the HTTP request. + rootPath, err := newTestConfig("us-east-1") + if err != nil { + t.Fatalf("Init Test config failed") + } + // remove the root folder after the test ends. + defer removeAll(rootPath) + + credentials := serverConfig.GetCredential() + // test cases with sample input and expected output. + testCases := []struct { + bucketName string + accessKey string + secretKey string + // expected Response. + expectedRespStatus int + locationResponse []byte + errorResponse APIErrorResponse + shouldPass bool + }{ + // Tests for authenticated request and proper response. + { + bucketName, + credentials.AccessKeyID, + credentials.SecretAccessKey, + http.StatusOK, + []byte(` +`), + APIErrorResponse{}, + true, + }, + // Tests for anonymous requests. + { + bucketName, + "", + "", + http.StatusForbidden, + []byte(""), + APIErrorResponse{ + Resource: "/" + bucketName + "/", + Code: "AccessDenied", + Message: "Access Denied.", + }, + false, + }, + } + + for i, testCase := range testCases { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + // construct HTTP request for Get bucket location. + req, err := newTestSignedRequest("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey) + if err != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for GetBucketLocationHandler: %v", i+1, instanceType, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + if rec.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + } + if !bytes.Equal(testCase.locationResponse, rec.Body.Bytes()) && testCase.shouldPass { + t.Errorf("Test %d: %s: Expected the response to be `%s`, but instead found `%s`", i+1, instanceType, string(testCase.locationResponse), string(rec.Body.Bytes())) + } + errorResponse := APIErrorResponse{} + err = xml.Unmarshal(rec.Body.Bytes(), &errorResponse) + if err != nil && !testCase.shouldPass { + t.Fatalf("Test %d: %s: Unable to marshal response body %s", i+1, instanceType, string(rec.Body.Bytes())) + } + if errorResponse.Resource != testCase.errorResponse.Resource { + t.Errorf("Test %d: %s: Expected the error resource to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Resource, errorResponse.Resource) + } + if errorResponse.Message != testCase.errorResponse.Message { + t.Errorf("Test %d: %s: Expected the error message to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Message, errorResponse.Message) + } + if errorResponse.Code != testCase.errorResponse.Code { + t.Errorf("Test %d: %s: Expected the error code to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Code, errorResponse.Code) + } + } +} + +// Wrapper for calling HeadBucket HTTP handler tests for both XL multiple disks and single node setup. +func TestHeadBucketHandler(t *testing.T) { + ExecObjectLayerTest(t, testHeadBucketHandler) +} + +func testHeadBucketHandler(obj ObjectLayer, instanceType string, t TestErrHandler) { + initBucketPolicies(obj) + + // get random bucket name. + bucketName := getRandomBucketName() + // Create bucket. + err := obj.MakeBucket(bucketName) + if err != nil { + // failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err) + } + // Register the API end points with XL/FS object layer. + apiRouter := initTestAPIEndPoints(obj, []string{"HeadBucket"}) + // initialize the server and obtain the credentials and root. + // credentials are necessary to sign the HTTP request. + rootPath, err := newTestConfig("us-east-1") + if err != nil { + t.Fatalf("Init Test config failed") + } + // remove the root folder after the test ends. + defer removeAll(rootPath) + + credentials := serverConfig.GetCredential() + // test cases with sample input and expected output. + testCases := []struct { + bucketName string + accessKey string + secretKey string + // expected Response. + expectedRespStatus int + }{ + // Bucket exists. + { + bucketName: bucketName, + accessKey: credentials.AccessKeyID, + secretKey: credentials.SecretAccessKey, + expectedRespStatus: http.StatusOK, + }, + // Non-existent bucket name. + { + bucketName: "2333", + accessKey: credentials.AccessKeyID, + secretKey: credentials.SecretAccessKey, + expectedRespStatus: http.StatusNotFound, + }, + // Un-authenticated request. + { + bucketName: bucketName, + accessKey: "", + secretKey: "", + expectedRespStatus: http.StatusForbidden, + }, + } + + for i, testCase := range testCases { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + // construct HTTP request for HEAD bucket. + req, err := newTestSignedRequest("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey) + if err != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for HeadBucketHandler: %v", i+1, instanceType, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + if rec.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + } + } +} + +// Wrapper for calling TestListMultipartUploadsHandler tests for both XL multiple disks and single node setup. +func TestListMultipartUploadsHandler(t *testing.T) { + ExecObjectLayerTest(t, testListMultipartUploads) +} + +// testListMultipartUploadsHandler - Tests validate listing of multipart uploads. +func testListMultipartUploadsHandler(obj ObjectLayer, instanceType string, t TestErrHandler) { + initBucketPolicies(obj) + + // get random bucket name. + bucketName := getRandomBucketName() + + // Register the API end points with XL/FS object layer. + apiRouter := initTestAPIEndPoints(obj, []string{"ListMultipartUploads"}) + // initialize the server and obtain the credentials and root. + // credentials are necessary to sign the HTTP request. + rootPath, err := newTestConfig("us-east-1") + if err != nil { + t.Fatalf("Init Test config failed") + } + // remove the root folder after the test ends. + defer removeAll(rootPath) + + credentials := serverConfig.GetCredential() + + // bucketnames[0]. + // objectNames[0]. + // uploadIds [0]. + // Create bucket before initiating NewMultipartUpload. + err = obj.MakeBucket(bucketName) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + // Collection of non-exhaustive ListMultipartUploads test cases, valid errors + // and success responses. + testCases := []struct { + // Inputs to ListMultipartUploads. + bucket string + prefix string + keyMarker string + uploadIDMarker string + delimiter string + maxUploads string + expectedRespStatus int + shouldPass bool + }{ + // 1 - invalid bucket name. + {".test", "", "", "", "", "0", http.StatusBadRequest, false}, + // 2 - bucket not found. + {"volatile-bucket-1", "", "", "", "", "0", http.StatusNotFound, false}, + // 3 - invalid delimiter. + {bucketName, "", "", "", "-", "0", http.StatusBadRequest, false}, + // 4 - invalid prefix and marker combination. + {bucketName, "asia", "europe-object", "", "", "0", http.StatusNotImplemented, false}, + // 5 - invalid upload id and marker combination. + {bucketName, "asia", "asia/europe/", "abc", "", "0", http.StatusBadRequest, false}, + // 6 - invalid max upload id. + {bucketName, "", "", "", "", "-1", http.StatusBadRequest, false}, + // 7 - good case delimiter. + {bucketName, "", "", "", "/", "100", http.StatusOK, true}, + // 8 - good case without delimiter. + {bucketName, "", "", "", "", "100", http.StatusOK, true}, + } + + for i, testCase := range testCases { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + + // construct HTTP request for List multipart uploads endpoint. + u := getListMultipartUploadsURLWithParams("", testCase.bucket, testCase.prefix, testCase.keyMarker, testCase.uploadIDMarker, testCase.delimiter, testCase.maxUploads) + req, gerr := newTestSignedRequest("GET", u, 0, nil, credentials.AccessKeyID, credentials.SecretAccessKey) + if gerr != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for ListMultipartUploadsHandler: %v", i+1, instanceType, gerr) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + if rec.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + } + } + + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + + // construct HTTP request for List multipart uploads endpoint. + u := getListMultipartUploadsURLWithParams("", bucketName, "", "", "", "", "") + req, err := newTestSignedRequest("GET", u, 0, nil, "", "") // Generate an anonymous request. + if err != nil { + t.Fatalf("Test %s: Failed to create HTTP request for ListMultipartUploadsHandler: %v", instanceType, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusForbidden { + t.Errorf("Test %s: Expected the response status to be `http.StatusForbidden`, but instead found `%d`", instanceType, rec.Code) + } +} diff --git a/cmd/bucket-notification-handlers.go b/cmd/bucket-notification-handlers.go index 73b276491..7dce07ba1 100644 --- a/cmd/bucket-notification-handlers.go +++ b/cmd/bucket-notification-handlers.go @@ -39,6 +39,12 @@ const ( // not enabled on the bucket, the operation returns an empty // NotificationConfiguration element. func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) { + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + // Validate request authorization. if s3Error := checkAuth(r); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) @@ -47,7 +53,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, vars := mux.Vars(r) bucket := vars["bucket"] // Attempt to successfully load notification config. - nConfig, err := loadNotificationConfig(bucket, api.ObjectAPI) + nConfig, err := loadNotificationConfig(bucket, objAPI) if err != nil && err != errNoSuchNotifications { errorIf(err, "Unable to read notification configuration.") writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) @@ -78,6 +84,12 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, // By default, your bucket has no event notifications configured. That is, // the notification configuration will be an empty NotificationConfiguration. func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) { + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + // Validate request authorization. if s3Error := checkAuth(r); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) @@ -86,7 +98,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, vars := mux.Vars(r) bucket := vars["bucket"] - _, err := api.ObjectAPI.GetBucketInfo(bucket) + _, err := objectAPI.GetBucketInfo(bucket) if err != nil { errorIf(err, "Unable to find bucket info.") writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) @@ -133,7 +145,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, // Proceed to save notification configuration. notificationConfigPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig) - _, err = api.ObjectAPI.PutObject(minioMetaBucket, notificationConfigPath, bufferSize, bytes.NewReader(buffer.Bytes()), nil) + _, err = objectAPI.PutObject(minioMetaBucket, notificationConfigPath, bufferSize, bytes.NewReader(buffer.Bytes()), nil) if err != nil { errorIf(err, "Unable to write bucket notification configuration.") writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) @@ -204,6 +216,13 @@ func sendBucketNotification(w http.ResponseWriter, arnListenerCh <-chan []Notifi // ListenBucketNotificationHandler - list bucket notifications. func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) { + // Validate if bucket exists. + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + // Validate request authorization. if s3Error := checkAuth(r); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) @@ -219,8 +238,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit return } - // Validate if bucket exists. - _, err := api.ObjectAPI.GetBucketInfo(bucket) + _, err := objAPI.GetBucketInfo(bucket) if err != nil { errorIf(err, "Unable to bucket info.") writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) diff --git a/cmd/bucket-notification-utils.go b/cmd/bucket-notification-utils.go index 27930440c..04b63ee02 100644 --- a/cmd/bucket-notification-utils.go +++ b/cmd/bucket-notification-utils.go @@ -268,18 +268,17 @@ func validateTopicConfigs(topicConfigs []topicConfig) APIErrorCode { // Check all the queue configs for any duplicates. func checkDuplicateQueueConfigs(configs []queueConfig) APIErrorCode { - configMaps := make(map[string]int) + var queueConfigARNS []string // Navigate through each configs and count the entries. for _, config := range configs { - configMaps[config.QueueARN]++ + queueConfigARNS = append(queueConfigARNS, config.QueueARN) } - // Validate if there are any duplicate counts. - for _, count := range configMaps { - if count != 1 { - return ErrOverlappingConfigs - } + // Check if there are any duplicate counts. + if err := checkDuplicates(queueConfigARNS); err != nil { + errorIf(err, "Invalid queue configs found.") + return ErrOverlappingConfigs } // Success. @@ -288,18 +287,17 @@ func checkDuplicateQueueConfigs(configs []queueConfig) APIErrorCode { // Check all the topic configs for any duplicates. func checkDuplicateTopicConfigs(configs []topicConfig) APIErrorCode { - configMaps := make(map[string]int) + var topicConfigARNS []string // Navigate through each configs and count the entries. for _, config := range configs { - configMaps[config.TopicARN]++ + topicConfigARNS = append(topicConfigARNS, config.TopicARN) } - // Validate if there are any duplicate counts. - for _, count := range configMaps { - if count != 1 { - return ErrOverlappingConfigs - } + // Check if there are any duplicate counts. + if err := checkDuplicates(topicConfigARNS); err != nil { + errorIf(err, "Invalid topic configs found.") + return ErrOverlappingConfigs } // Success. @@ -320,12 +318,17 @@ func validateNotificationConfig(nConfig notificationConfig) APIErrorCode { } // Check for duplicate queue configs. - if s3Error := checkDuplicateQueueConfigs(nConfig.QueueConfigs); s3Error != ErrNone { - return s3Error + if len(nConfig.QueueConfigs) > 1 { + if s3Error := checkDuplicateQueueConfigs(nConfig.QueueConfigs); s3Error != ErrNone { + return s3Error + } } + // Check for duplicate topic configs. - if s3Error := checkDuplicateTopicConfigs(nConfig.TopicConfigs); s3Error != ErrNone { - return s3Error + if len(nConfig.TopicConfigs) > 1 { + if s3Error := checkDuplicateTopicConfigs(nConfig.TopicConfigs); s3Error != ErrNone { + return s3Error + } } // Add validation for other configurations. diff --git a/cmd/bucket-policy-handlers.go b/cmd/bucket-policy-handlers.go index 5cb206115..e6a5b826e 100644 --- a/cmd/bucket-policy-handlers.go +++ b/cmd/bucket-policy-handlers.go @@ -126,6 +126,12 @@ func bucketPolicyConditionMatch(conditions map[string]set.StringSet, statement p // This implementation of the PUT operation uses the policy // subresource to add to or replace a policy on a bucket func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + vars := mux.Vars(r) bucket := vars["bucket"] switch getRequestAuthType(r) { @@ -180,8 +186,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht } // Save bucket policy. - if err = writeBucketPolicy(bucket, api.ObjectAPI, bytes.NewReader(policyBytes), int64(len(policyBytes))); err != nil { - errorIf(err, "Unable to write bucket policy.") + if err = writeBucketPolicy(bucket, objAPI, bytes.NewReader(policyBytes), int64(len(policyBytes))); err != nil { switch err.(type) { case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) @@ -203,6 +208,12 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht // This implementation of the DELETE operation uses the policy // subresource to add to remove a policy on a bucket. func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + vars := mux.Vars(r) bucket := vars["bucket"] @@ -219,8 +230,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r } // Delete bucket access policy. - if err := removeBucketPolicy(bucket, api.ObjectAPI); err != nil { - errorIf(err, "Unable to remove bucket policy.") + if err := removeBucketPolicy(bucket, objAPI); err != nil { switch err.(type) { case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) @@ -244,6 +254,12 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r // This operation uses the policy // subresource to return the policy of a specified bucket. func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + vars := mux.Vars(r) bucket := vars["bucket"] @@ -260,7 +276,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht } // Read bucket access policy. - policy, err := readBucketPolicy(bucket, api.ObjectAPI) + policy, err := readBucketPolicy(bucket, objAPI) if err != nil { errorIf(err, "Unable to read bucket policy.") switch err.(type) { diff --git a/cmd/bucket-policy-handlers_test.go b/cmd/bucket-policy-handlers_test.go index 0bca9ed4e..cc2b75c45 100644 --- a/cmd/bucket-policy-handlers_test.go +++ b/cmd/bucket-policy-handlers_test.go @@ -1,5 +1,5 @@ /* - * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc. + * Minio Cloud Storage, (C) 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -294,13 +294,13 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType string, t TestErrH req, err := newTestSignedRequest("PUT", getPutPolicyURL("", testCase.bucketName), int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testCase.accessKey, testCase.secretKey) if err != nil { - t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: %v", i+1, err) + t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: %v", i+1, instanceType, err) } // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler. // Call the ServeHTTP to execute the handler. apiRouter.ServeHTTP(rec, req) if rec.Code != testCase.expectedRespStatus { - t.Errorf("Test %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, rec.Code) + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) } } } diff --git a/cmd/bucket-policy.go b/cmd/bucket-policy.go index 4678fecf9..c03e90274 100644 --- a/cmd/bucket-policy.go +++ b/cmd/bucket-policy.go @@ -66,27 +66,38 @@ func (bp *bucketPolicies) RemoveBucketPolicy(bucket string) { func loadAllBucketPolicies(objAPI ObjectLayer) (policies map[string]*bucketPolicy, err error) { // List buckets to proceed loading all notification configuration. buckets, err := objAPI.ListBuckets() + errorIf(err, "Unable to list buckets.") + err = errorCause(err) if err != nil { return nil, err } + policies = make(map[string]*bucketPolicy) + var pErrs []error // Loads bucket policy. for _, bucket := range buckets { - var policy *bucketPolicy - policy, err = readBucketPolicy(bucket.Name, objAPI) - if err != nil { - switch err.(type) { + policy, pErr := readBucketPolicy(bucket.Name, objAPI) + if pErr != nil { + switch pErr.(type) { case BucketPolicyNotFound: continue } - return nil, err + pErrs = append(pErrs, pErr) + // Continue to load other bucket policies if possible. + continue } policies[bucket.Name] = policy } + // Look for any errors occurred while reading bucket policies. + for _, pErr := range pErrs { + if pErr != nil { + return policies, pErr + } + } + // Success. return policies, nil - } // Intialize all bucket policies. @@ -94,16 +105,20 @@ func initBucketPolicies(objAPI ObjectLayer) error { if objAPI == nil { return errInvalidArgument } + // Read all bucket policies. policies, err := loadAllBucketPolicies(objAPI) if err != nil { return err } + // Populate global bucket collection. globalBucketPolicies = &bucketPolicies{ rwMutex: &sync.RWMutex{}, bucketPolicyConfigs: policies, } + + // Success. return nil } @@ -125,18 +140,22 @@ func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader } policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON) objInfo, err := objAPI.GetObjectInfo(minioMetaBucket, policyPath) + err = errorCause(err) if err != nil { if _, ok := err.(ObjectNotFound); ok { return nil, BucketPolicyNotFound{Bucket: bucket} } + errorIf(err, "Unable to load policy for the bucket %s.", bucket) return nil, err } var buffer bytes.Buffer err = objAPI.GetObject(minioMetaBucket, policyPath, 0, objInfo.Size, &buffer) + err = errorCause(err) if err != nil { if _, ok := err.(ObjectNotFound); ok { return nil, BucketPolicyNotFound{Bucket: bucket} } + errorIf(err, "Unable to load policy for the bucket %s.", bucket) return nil, err } @@ -169,9 +188,10 @@ func removeBucketPolicy(bucket string, objAPI ObjectLayer) error { if !IsValidBucketName(bucket) { return BucketNameInvalid{Bucket: bucket} } - policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON) if err := objAPI.DeleteObject(minioMetaBucket, policyPath); err != nil { + errorIf(err, "Unable to remove bucket-policy on bucket %s.", bucket) + err = errorCause(err) if _, ok := err.(ObjectNotFound); ok { return BucketPolicyNotFound{Bucket: bucket} } @@ -188,6 +208,9 @@ func writeBucketPolicy(bucket string, objAPI ObjectLayer, reader io.Reader, size } policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON) - _, err := objAPI.PutObject(minioMetaBucket, policyPath, size, reader, nil) - return err + if _, err := objAPI.PutObject(minioMetaBucket, policyPath, size, reader, nil); err != nil { + errorIf(err, "Unable to set policy for the bucket %s", bucket) + return errorCause(err) + } + return nil } diff --git a/cmd/commands.go b/cmd/commands.go index f40e56553..96aa05ec0 100644 --- a/cmd/commands.go +++ b/cmd/commands.go @@ -24,19 +24,6 @@ var commands = []cli.Command{} // Collection of minio commands currently supported in a trie tree. var commandsTree = newTrie() -// Collection of minio flags currently supported. -var globalFlags = []cli.Flag{ - cli.StringFlag{ - Name: "config-dir, C", - Value: mustGetConfigPath(), - Usage: "Path to configuration folder.", - }, - cli.BoolFlag{ - Name: "quiet", - Usage: "Suppress chatty output.", - }, -} - // registerCommand registers a cli command. func registerCommand(command cli.Command) { commands = append(commands, command) diff --git a/cmd/control-heal-main.go b/cmd/control-heal-main.go index c25a9fda6..96d8c8035 100644 --- a/cmd/control-heal-main.go +++ b/cmd/control-heal-main.go @@ -18,7 +18,6 @@ package cmd import ( "fmt" - "net/rpc" "net/url" "path" "strings" @@ -30,13 +29,18 @@ var healCmd = cli.Command{ Name: "heal", Usage: "To heal objects.", Action: healControl, + Flags: globalFlags, CustomHelpTemplate: `NAME: minio control {{.Name}} - {{.Usage}} USAGE: minio control {{.Name}} -EAMPLES: +FLAGS: + {{range .Flags}}{{.}} + {{end}} + +EXAMPLES: 1. Heal an object. $ minio control {{.Name}} http://localhost:9000/songs/classical/western/piano.mp3 @@ -48,8 +52,17 @@ EAMPLES: `, } +func checkHealControlSyntax(ctx *cli.Context) { + if len(ctx.Args()) != 1 { + cli.ShowCommandHelpAndExit(ctx, "heal", 1) + } +} + // "minio control heal" entry point. func healControl(ctx *cli.Context) { + + checkHealControlSyntax(ctx) + // Parse bucket and object from url.URL.Path parseBucketObject := func(path string) (bucketName string, objectName string) { splits := strings.SplitN(path, string(slashSeparator), 3) @@ -68,29 +81,38 @@ func healControl(ctx *cli.Context) { return bucketName, objectName } - if len(ctx.Args()) != 1 { - cli.ShowCommandHelpAndExit(ctx, "heal", 1) - } - parsedURL, err := url.Parse(ctx.Args()[0]) fatalIf(err, "Unable to parse URL") + authCfg := &authConfig{ + accessKey: serverConfig.GetCredential().AccessKeyID, + secretKey: serverConfig.GetCredential().SecretAccessKey, + address: parsedURL.Host, + path: path.Join(reservedBucket, controlPath), + loginMethod: "Controller.LoginHandler", + } + client := newAuthClient(authCfg) + + // Always try to fix disk metadata + fmt.Print("Checking and healing disk metadata..") + args := &GenericArgs{} + reply := &GenericReply{} + err = client.Call("Controller.HealDiskMetadataHandler", args, reply) + fatalIf(err, "Unable to heal disk metadata.") + fmt.Println(" ok") + bucketName, objectName := parseBucketObject(parsedURL.Path) if bucketName == "" { - cli.ShowCommandHelpAndExit(ctx, "heal", 1) + return } - client, err := rpc.DialHTTPPath("tcp", parsedURL.Host, path.Join(reservedBucket, controlPath)) - fatalIf(err, "Unable to connect to %s", parsedURL.Host) - // If object does not have trailing "/" then it's an object, hence heal it. if objectName != "" && !strings.HasSuffix(objectName, slashSeparator) { - fmt.Printf("Healing : /%s/%s", bucketName, objectName) - args := &HealObjectArgs{bucketName, objectName} + fmt.Printf("Healing : /%s/%s\n", bucketName, objectName) + args := &HealObjectArgs{Bucket: bucketName, Object: objectName} reply := &HealObjectReply{} - err = client.Call("Control.HealObject", args, reply) - fatalIf(err, "RPC Control.HealObject call failed") - fmt.Println() + err = client.Call("Controller.HealObjectHandler", args, reply) + errorIf(err, "Healing object %s failed.", objectName) return } @@ -98,23 +120,32 @@ func healControl(ctx *cli.Context) { prefix := objectName marker := "" for { - args := HealListArgs{bucketName, prefix, marker, "", 1000} + args := &HealListArgs{ + Bucket: bucketName, + Prefix: prefix, + Marker: marker, + Delimiter: "", + MaxKeys: 1000, + } reply := &HealListReply{} - err = client.Call("Control.ListObjectsHeal", args, reply) - fatalIf(err, "RPC Heal.ListObjects call failed") + err = client.Call("Controller.ListObjectsHealHandler", args, reply) + fatalIf(err, "Unable to list objects for healing.") // Heal the objects returned in the ListObjects reply. for _, obj := range reply.Objects { - fmt.Printf("Healing : /%s/%s", bucketName, obj) - reply := &HealObjectReply{} - err = client.Call("Control.HealObject", HealObjectArgs{bucketName, obj}, reply) - fatalIf(err, "RPC Heal.HealObject call failed") - fmt.Println() + fmt.Printf("Healing : /%s/%s\n", bucketName, obj) + reply := &GenericReply{} + healArgs := &HealObjectArgs{Bucket: bucketName, Object: obj} + err = client.Call("Controller.HealObjectHandler", healArgs, reply) + errorIf(err, "Healing object %s failed.", obj) } + if !reply.IsTruncated { // End of listing. break } + + // Set the marker to list the next set of keys. marker = reply.NextMarker } } diff --git a/cmd/control-lock-main.go b/cmd/control-lock-main.go new file mode 100644 index 000000000..7843fa696 --- /dev/null +++ b/cmd/control-lock-main.go @@ -0,0 +1,144 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "encoding/json" + "fmt" + "net/url" + "path" + "time" + + "github.com/minio/cli" +) + +// SystemLockState - Structure to fill the lock state of entire object storage. +// That is the total locks held, total calls blocked on locks and state of all the locks for the entire system. +type SystemLockState struct { + TotalLocks int64 `json:"totalLocks"` + TotalBlockedLocks int64 `json:"totalBlockedLocks"` // count of operations which are blocked waiting for the lock to be released. + TotalAcquiredLocks int64 `json:"totalAcquiredLocks"` // count of operations which has successfully acquired the lock but hasn't unlocked yet( operation in progress). + LocksInfoPerObject []VolumeLockInfo `json:"locksInfoPerObject"` +} + +// VolumeLockInfo - Structure to contain the lock state info for volume, path pair. +type VolumeLockInfo struct { + Bucket string `json:"bucket"` + Object string `json:"object"` + LocksOnObject int64 `json:"locksOnObject"` // All locks blocked + running for given pair. + LocksAcquiredOnObject int64 `json:"locksAcquiredOnObject"` // count of operations which has successfully acquired the lock but hasn't unlocked yet( operation in progress). + TotalBlockedLocks int64 `json:"locksBlockedOnObject"` // count of operations which are blocked waiting for the lock to be released. + LockDetailsOnObject []OpsLockState `json:"lockDetailsOnObject"` // state information containing state of the locks for all operations on given pair. +} + +// OpsLockState - structure to fill in state information of the lock. +// structure to fill in status information for each operation with given operation ID. +type OpsLockState struct { + OperationID string `json:"opsID"` // string containing operation ID. + LockOrigin string `json:"lockOrigin"` // contant which mentions the operation type (Get Obejct, PutObject...) + LockType string `json:"lockType"` + Status string `json:"status"` // status can be running/ready/blocked. + StatusSince string `json:"statusSince"` // time info of the since how long the status holds true, value in seconds. +} + +// Read entire state of the locks in the system and return. +func generateSystemLockResponse() (SystemLockState, error) { + nsMutex.lockMapMutex.Lock() + defer nsMutex.lockMapMutex.Unlock() + + if nsMutex.debugLockMap == nil { + return SystemLockState{}, LockInfoNil{} + } + + lockState := SystemLockState{} + + lockState.TotalBlockedLocks = nsMutex.blockedCounter + lockState.TotalLocks = nsMutex.globalLockCounter + lockState.TotalAcquiredLocks = nsMutex.runningLockCounter + + for param := range nsMutex.debugLockMap { + volLockInfo := VolumeLockInfo{} + volLockInfo.Bucket = param.volume + volLockInfo.Object = param.path + volLockInfo.TotalBlockedLocks = nsMutex.debugLockMap[param].blocked + volLockInfo.LocksAcquiredOnObject = nsMutex.debugLockMap[param].running + volLockInfo.LocksOnObject = nsMutex.debugLockMap[param].ref + for opsID := range nsMutex.debugLockMap[param].lockInfo { + opsState := OpsLockState{} + opsState.OperationID = opsID + opsState.LockOrigin = nsMutex.debugLockMap[param].lockInfo[opsID].lockOrigin + opsState.LockType = nsMutex.debugLockMap[param].lockInfo[opsID].lockType + opsState.Status = nsMutex.debugLockMap[param].lockInfo[opsID].status + opsState.StatusSince = time.Now().Sub(nsMutex.debugLockMap[param].lockInfo[opsID].since).String() + + volLockInfo.LockDetailsOnObject = append(volLockInfo.LockDetailsOnObject, opsState) + } + lockState.LocksInfoPerObject = append(lockState.LocksInfoPerObject, volLockInfo) + } + + return lockState, nil +} + +var lockCmd = cli.Command{ + Name: "lock", + Usage: "info about the locks in the node.", + Action: lockControl, + Flags: globalFlags, + CustomHelpTemplate: `NAME: + minio control {{.Name}} - {{.Usage}} + +USAGE: + minio control {{.Name}} http://localhost:9000/ + +FLAGS: + {{range .Flags}}{{.}} + {{end}} + +EAMPLES: + 1. Get all the info about the blocked/held locks in the node: + $ minio control lock http://localhost:9000/ +`, +} + +// "minio control lock" entry point. +func lockControl(c *cli.Context) { + if len(c.Args()) != 1 { + cli.ShowCommandHelpAndExit(c, "lock", 1) + } + + parsedURL, err := url.Parse(c.Args()[0]) + fatalIf(err, "Unable to parse URL.") + + authCfg := &authConfig{ + accessKey: serverConfig.GetCredential().AccessKeyID, + secretKey: serverConfig.GetCredential().SecretAccessKey, + address: parsedURL.Host, + path: path.Join(reservedBucket, controlPath), + loginMethod: "Controller.LoginHandler", + } + client := newAuthClient(authCfg) + + args := &GenericArgs{} + reply := &SystemLockState{} + err = client.Call("Controller.LockInfo", args, reply) + // logs the error and returns if err != nil. + fatalIf(err, "RPC Controller.LockInfo call failed") + // print the lock info on the console. + b, err := json.MarshalIndent(*reply, "", " ") + fatalIf(err, "Failed to parse the RPC lock info response") + fmt.Print(string(b)) +} diff --git a/cmd/control-main.go b/cmd/control-main.go index a0494c628..d0eb917e5 100644 --- a/cmd/control-main.go +++ b/cmd/control-main.go @@ -22,8 +22,10 @@ import "github.com/minio/cli" var controlCmd = cli.Command{ Name: "control", Usage: "Control and manage minio server.", + Flags: globalFlags, Action: mainControl, Subcommands: []cli.Command{ + lockCmd, healCmd, shutdownCmd, }, diff --git a/cmd/control-shutdown-main.go b/cmd/control-shutdown-main.go index b24621d39..28b6d9457 100644 --- a/cmd/control-shutdown-main.go +++ b/cmd/control-shutdown-main.go @@ -17,30 +17,35 @@ package cmd import ( - "net/rpc" "net/url" "path" "github.com/minio/cli" ) +var shutdownFlags = []cli.Flag{ + cli.BoolFlag{ + Name: "restart", + Usage: "Restart the server.", + }, +} + var shutdownCmd = cli.Command{ Name: "shutdown", Usage: "Shutdown or restart the server.", Action: shutdownControl, - Flags: []cli.Flag{ - cli.BoolFlag{ - Name: "restart", - Usage: "Restart the server.", - }, - }, + Flags: append(shutdownFlags, globalFlags...), CustomHelpTemplate: `NAME: minio control {{.Name}} - {{.Usage}} USAGE: minio control {{.Name}} http://localhost:9000/ -EAMPLES: +FLAGS: + {{range .Flags}}{{.}} + {{end}} + +EXAMPLES: 1. Shutdown the server: $ minio control shutdown http://localhost:9000/ @@ -55,14 +60,19 @@ func shutdownControl(c *cli.Context) { cli.ShowCommandHelpAndExit(c, "shutdown", 1) } - parsedURL, err := url.ParseRequestURI(c.Args()[0]) - fatalIf(err, "Unable to parse URL") + parsedURL, err := url.Parse(c.Args()[0]) + fatalIf(err, "Unable to parse URL.") - client, err := rpc.DialHTTPPath("tcp", parsedURL.Host, path.Join(reservedBucket, controlPath)) - fatalIf(err, "Unable to connect to %s", parsedURL.Host) + authCfg := &authConfig{ + accessKey: serverConfig.GetCredential().AccessKeyID, + secretKey: serverConfig.GetCredential().SecretAccessKey, + address: parsedURL.Host, + path: path.Join(reservedBucket, controlPath), + loginMethod: "Controller.LoginHandler", + } + client := newAuthClient(authCfg) - args := &ShutdownArgs{Reboot: c.Bool("restart")} - reply := &ShutdownReply{} - err = client.Call("Control.Shutdown", args, reply) - fatalIf(err, "RPC Control.Shutdown call failed") + args := &ShutdownArgs{Restart: c.Bool("restart")} + err = client.Call("Controller.ShutdownHandler", args, &GenericReply{}) + errorIf(err, "Shutting down Minio server at %s failed.", parsedURL.Host) } diff --git a/cmd/controller-handlers.go b/cmd/controller-handlers.go index e6c0fa9c3..a5b395ad3 100644 --- a/cmd/controller-handlers.go +++ b/cmd/controller-handlers.go @@ -16,8 +16,31 @@ package cmd +/// Auth operations + +// Login - login handler. +func (c *controllerAPIHandlers) LoginHandler(args *RPCLoginArgs, reply *RPCLoginReply) error { + jwt, err := newJWT(defaultTokenExpiry) + if err != nil { + return err + } + if err = jwt.Authenticate(args.Username, args.Password); err != nil { + return err + } + token, err := jwt.GenerateToken(args.Username) + if err != nil { + return err + } + reply.Token = token + reply.ServerVersion = Version + return nil +} + // HealListArgs - argument for ListObjects RPC. type HealListArgs struct { + // Authentication token generated by Login. + GenericArgs + Bucket string Prefix string Marker string @@ -25,7 +48,7 @@ type HealListArgs struct { MaxKeys int } -// HealListReply - reply by ListObjects RPC. +// HealListReply - reply object by ListObjects RPC. type HealListReply struct { IsTruncated bool NextMarker string @@ -33,12 +56,15 @@ type HealListReply struct { } // ListObjects - list all objects that needs healing. -func (c *controllerAPIHandlers) ListObjectsHeal(arg *HealListArgs, reply *HealListReply) error { - objAPI := c.ObjectAPI +func (c *controllerAPIHandlers) ListObjectsHealHandler(args *HealListArgs, reply *HealListReply) error { + objAPI := c.ObjectAPI() if objAPI == nil { - return errInvalidArgument + return errVolumeBusy } - info, err := objAPI.ListObjectsHeal(arg.Bucket, arg.Prefix, arg.Marker, arg.Delimiter, arg.MaxKeys) + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + info, err := objAPI.ListObjectsHeal(args.Bucket, args.Prefix, args.Marker, args.Delimiter, args.MaxKeys) if err != nil { return err } @@ -52,7 +78,13 @@ func (c *controllerAPIHandlers) ListObjectsHeal(arg *HealListArgs, reply *HealLi // HealObjectArgs - argument for HealObject RPC. type HealObjectArgs struct { + // Authentication token generated by Login. + GenericArgs + + // Name of the bucket. Bucket string + + // Name of the object. Object string } @@ -60,29 +92,78 @@ type HealObjectArgs struct { type HealObjectReply struct{} // HealObject - heal the object. -func (c *controllerAPIHandlers) HealObject(arg *HealObjectArgs, reply *HealObjectReply) error { - objAPI := c.ObjectAPI +func (c *controllerAPIHandlers) HealObjectHandler(args *HealObjectArgs, reply *GenericReply) error { + objAPI := c.ObjectAPI() if objAPI == nil { - return errInvalidArgument + return errVolumeBusy } - return objAPI.HealObject(arg.Bucket, arg.Object) + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + return objAPI.HealObject(args.Bucket, args.Object) +} + +// HealObject - heal the object. +func (c *controllerAPIHandlers) HealDiskMetadataHandler(args *GenericArgs, reply *GenericReply) error { + objAPI := c.ObjectAPI() + if objAPI == nil { + return errVolumeBusy + } + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + err := objAPI.HealDiskMetadata() + if err != nil { + return err + } + go func() { + globalWakeupCh <- struct{}{} + }() + return err } // ShutdownArgs - argument for Shutdown RPC. type ShutdownArgs struct { - Reboot bool + // Authentication token generated by Login. + GenericArgs + + // Should the server be restarted, call active connections are served before server + // is restarted. + Restart bool } -// ShutdownReply - reply by Shutdown RPC. -type ShutdownReply struct{} - -// Shutdown - Shutdown the server. - -func (c *controllerAPIHandlers) Shutdown(arg *ShutdownArgs, reply *ShutdownReply) error { - if arg.Reboot { +// Shutdown - Shutsdown the server. +func (c *controllerAPIHandlers) ShutdownHandler(args *ShutdownArgs, reply *GenericReply) error { + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + if args.Restart { globalShutdownSignalCh <- shutdownRestart } else { globalShutdownSignalCh <- shutdownHalt } return nil } + +func (c *controllerAPIHandlers) TryInitHandler(args *GenericArgs, reply *GenericReply) error { + go func() { + globalWakeupCh <- struct{}{} + }() + *reply = GenericReply{} + return nil + +} + +// LockInfo - RPC control handler for `minio control lock`. +// Returns the info of the locks held in the system. +func (c *controllerAPIHandlers) LockInfo(arg *GenericArgs, reply *SystemLockState) error { + // obtain the lock state information. + lockInfo, err := generateSystemLockResponse() + // in case of error, return err to the RPC client. + if err != nil { + return err + } + // the response containing the lock info. + *reply = lockInfo + return nil +} diff --git a/cmd/controller-router.go b/cmd/controller-router.go index 8538d95c5..c782b929f 100644 --- a/cmd/controller-router.go +++ b/cmd/controller-router.go @@ -27,10 +27,10 @@ const ( controlPath = "/controller" ) -// Register control RPC handlers. -func registerControlRPCRouter(mux *router.Router, ctrlHandlers *controllerAPIHandlers) { +// Register controller RPC handlers. +func registerControllerRPCRouter(mux *router.Router, ctrlHandlers *controllerAPIHandlers) { ctrlRPCServer := rpc.NewServer() - ctrlRPCServer.RegisterName("Control", ctrlHandlers) + ctrlRPCServer.RegisterName("Controller", ctrlHandlers) ctrlRouter := mux.NewRoute().PathPrefix(reservedBucket).Subrouter() ctrlRouter.Path(controlPath).Handler(ctrlRPCServer) @@ -38,5 +38,5 @@ func registerControlRPCRouter(mux *router.Router, ctrlHandlers *controllerAPIHan // Handler for object healing. type controllerAPIHandlers struct { - ObjectAPI ObjectLayer + ObjectAPI func() ObjectLayer } diff --git a/cmd/controller_test.go b/cmd/controller_test.go new file mode 100644 index 000000000..477e1785d --- /dev/null +++ b/cmd/controller_test.go @@ -0,0 +1,298 @@ +/* + * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "path" + "strconv" + "sync" + "time" + + . "gopkg.in/check.v1" +) + +// API suite container common to both FS and XL. +type TestRPCControllerSuite struct { + serverType string + testServer TestServer + endPoint string + accessKey string + secretKey string +} + +// Init and run test on XL backend. +var _ = Suite(&TestRPCControllerSuite{serverType: "XL"}) + +// Setting up the test suite. +// Starting the Test server with temporary FS backend. +func (s *TestRPCControllerSuite) SetUpSuite(c *C) { + s.testServer = StartTestRPCServer(c, s.serverType) + s.endPoint = s.testServer.Server.Listener.Addr().String() + s.accessKey = s.testServer.AccessKey + s.secretKey = s.testServer.SecretKey +} + +// Called implicitly by "gopkg.in/check.v1" after all tests are run. +func (s *TestRPCControllerSuite) TearDownSuite(c *C) { + s.testServer.Stop() +} + +// Tests to validate the correctness of lock instrumentation control RPC end point. +func (s *TestRPCControllerSuite) TestRPCControlLock(c *C) { + // enabling lock instrumentation. + globalDebugLock = true + // initializing the locks. + initNSLock(false) + // set debug lock info to `nil` so that the next tests have to initialize them again. + defer func() { + globalDebugLock = false + nsMutex.debugLockMap = nil + }() + + expectedResult := []lockStateCase{ + // Test case - 1. + // Case where 10 read locks are held. + // Entry for any of the 10 reads locks has to be found. + // Since they held in a loop, Lock origin for first 10 read locks (opsID 0-9) should be the same. + { + + volume: "my-bucket", + path: "my-object", + opsID: "0", + readLock: true, + lockOrigin: "[lock held] in github.com/minio/minio/cmd.TestLockStats[/Users/hackintoshrao/mycode/go/src/github.com/minio/minio/cmd/namespace-lock_test.go:298]", + // expected metrics. + expectedErr: nil, + expectedLockStatus: "Running", + + expectedGlobalLockCount: 10, + expectedRunningLockCount: 10, + expectedBlockedLockCount: 0, + + expectedVolPathLockCount: 10, + expectedVolPathRunningCount: 10, + expectedVolPathBlockCount: 0, + }, + // Test case 2. + // Testing the existence of entry for the last read lock (read lock with opsID "9"). + { + + volume: "my-bucket", + path: "my-object", + opsID: "9", + readLock: true, + lockOrigin: "[lock held] in github.com/minio/minio/cmd.TestLockStats[/Users/hackintoshrao/mycode/go/src/github.com/minio/minio/cmd/namespace-lock_test.go:298]", + // expected metrics. + expectedErr: nil, + expectedLockStatus: "Running", + + expectedGlobalLockCount: 10, + expectedRunningLockCount: 10, + expectedBlockedLockCount: 0, + + expectedVolPathLockCount: 10, + expectedVolPathRunningCount: 10, + expectedVolPathBlockCount: 0, + }, + + // Test case 3. + // Hold a write lock, and it should block since 10 read locks + // on <"my-bucket", "my-object"> are still held. + { + + volume: "my-bucket", + path: "my-object", + opsID: "10", + readLock: false, + lockOrigin: "[lock held] in github.com/minio/minio/cmd.TestLockStats[/Users/hackintoshrao/mycode/go/src/github.com/minio/minio/cmd/namespace-lock_test.go:298]", + // expected metrics. + expectedErr: nil, + expectedLockStatus: "Blocked", + + expectedGlobalLockCount: 11, + expectedRunningLockCount: 10, + expectedBlockedLockCount: 1, + + expectedVolPathLockCount: 11, + expectedVolPathRunningCount: 10, + expectedVolPathBlockCount: 1, + }, + + // Test case 4. + // Expected result when all the read locks are released and the blocked write lock acquires the lock. + { + + volume: "my-bucket", + path: "my-object", + opsID: "10", + readLock: false, + lockOrigin: "[lock held] in github.com/minio/minio/cmd.TestLockStats[/Users/hackintoshrao/mycode/go/src/github.com/minio/minio/cmd/namespace-lock_test.go:298]", + // expected metrics. + expectedErr: nil, + expectedLockStatus: "Running", + + expectedGlobalLockCount: 1, + expectedRunningLockCount: 1, + expectedBlockedLockCount: 0, + + expectedVolPathLockCount: 1, + expectedVolPathRunningCount: 1, + expectedVolPathBlockCount: 0, + }, + // Test case - 5. + // At the end after locks are released, its verified whether the counters are set to 0. + { + + volume: "my-bucket", + path: "my-object", + // expected metrics. + expectedErr: nil, + expectedLockStatus: "Blocked", + + expectedGlobalLockCount: 0, + expectedRunningLockCount: 0, + expectedBlockedLockCount: 0, + }, + } + + // used to make sure that the tests don't end till locks held in other go routines are released. + var wg sync.WaitGroup + + // Hold 5 read locks. We should find the info about these in the RPC response. + + // hold 10 read locks. + // Then call the RPC control end point for obtaining lock instrumentation info. + + for i := 0; i < 10; i++ { + nsMutex.RLock("my-bucket", "my-object", strconv.Itoa(i)) + } + + authCfg := &authConfig{ + accessKey: s.accessKey, + secretKey: s.secretKey, + address: s.endPoint, + path: path.Join(reservedBucket, controlPath), + loginMethod: "Controller.LoginHandler", + } + + client := newAuthClient(authCfg) + + defer client.Close() + + args := &GenericArgs{} + reply := &SystemLockState{} + // Call the lock instrumentation RPC end point. + err := client.Call("Controller.LockInfo", args, reply) + if err != nil { + c.Errorf("Add: expected no error but got string %q", err.Error()) + } + // expected lock info. + expectedLockStats := expectedResult[0] + // verify the actual lock info with the expected one. + // verify the existence entry for first read lock (read lock with opsID "0"). + verifyRPCLockInfoResponse(expectedLockStats, *reply, c, 1) + expectedLockStats = expectedResult[1] + // verify the actual lock info with the expected one. + // verify the existence entry for last read lock (read lock with opsID "9"). + verifyRPCLockInfoResponse(expectedLockStats, *reply, c, 2) + + // now hold a write lock in a different go routine and it should block since 10 read locks are + // still held. + wg.Add(1) + go func() { + defer wg.Done() + // blocks till all read locks are released. + nsMutex.Lock("my-bucket", "my-object", strconv.Itoa(10)) + // Once the above attempt to lock is unblocked/acquired, we verify the stats and release the lock. + expectedWLockStats := expectedResult[3] + // Since the write lock acquired here, the number of blocked locks should reduce by 1 and + // count of running locks should increase by 1. + + // Call the RPC control handle to fetch the lock instrumentation info. + reply = &SystemLockState{} + // Call the lock instrumentation RPC end point. + err = client.Call("Controller.LockInfo", args, reply) + if err != nil { + c.Errorf("Add: expected no error but got string %q", err.Error()) + } + verifyRPCLockInfoResponse(expectedWLockStats, *reply, c, 4) + + // release the write lock. + nsMutex.Unlock("my-bucket", "my-object", strconv.Itoa(10)) + + }() + // waiting for a second so that the attempt to acquire the write lock in + // the above go routines gets blocked. + time.Sleep(1 * time.Second) + // The write lock should have got blocked by now, + // check whether the entry for one blocked lock exists. + expectedLockStats = expectedResult[2] + + // Call the RPC control handle to fetch the lock instrumentation info. + reply = &SystemLockState{} + // Call the lock instrumentation RPC end point. + err = client.Call("Controller.LockInfo", args, reply) + if err != nil { + c.Errorf("Add: expected no error but got string %q", err.Error()) + } + verifyRPCLockInfoResponse(expectedLockStats, *reply, c, 3) + // Release all the read locks held. + // the blocked write lock in the above go routines should get unblocked. + for i := 0; i < 10; i++ { + nsMutex.RUnlock("my-bucket", "my-object", strconv.Itoa(i)) + } + wg.Wait() + // Since all the locks are released. There should not be any entry in the lock info. + // and all the counters should be set to 0. + reply = &SystemLockState{} + // Call the lock instrumentation RPC end point. + err = client.Call("Controller.LockInfo", args, reply) + if err != nil { + c.Errorf("Add: expected no error but got string %q", err.Error()) + } + + if reply.TotalAcquiredLocks != 0 && reply.TotalLocks != 0 && reply.TotalBlockedLocks != 0 { + c.Fatalf("The counters are not reset properly after all locks are released") + } + if len(reply.LocksInfoPerObject) != 0 { + c.Fatalf("Since all locks are released there shouldn't have been any lock info entry, but found %d", len(reply.LocksInfoPerObject)) + } +} + +// TestControllerHandlerHealDiskMetadata - Registers and call the `HealDiskMetadataHandler`, +// asserts to validate the success. +func (s *TestRPCControllerSuite) TestControllerHandlerHealDiskMetadata(c *C) { + // The suite has already started the test RPC server, just send RPC calls. + authCfg := &authConfig{ + accessKey: s.accessKey, + secretKey: s.secretKey, + address: s.endPoint, + path: path.Join(reservedBucket, controlPath), + loginMethod: "Controller.LoginHandler", + } + + client := newAuthClient(authCfg) + defer client.Close() + + args := &GenericArgs{} + reply := &GenericReply{} + err := client.Call("Controller.HealDiskMetadataHandler", args, reply) + + if err != nil { + c.Errorf("Heal Meta Disk Handler test failed with %s", err.Error()) + } +} diff --git a/cmd/damerau-levenshtein.go b/cmd/damerau-levenshtein.go index fcf38fe26..1e275e78b 100644 --- a/cmd/damerau-levenshtein.go +++ b/cmd/damerau-levenshtein.go @@ -44,9 +44,9 @@ func DamerauLevenshteinDistance(a string, b string) int { for j := 0; j <= len(b); j++ { d[0][j] = j } + var cost int for i := 1; i <= len(a); i++ { for j := 1; j <= len(b); j++ { - cost := 0 if a[i-1] == b[j-1] { cost = 0 } else { diff --git a/cmd/erasure-createfile.go b/cmd/erasure-createfile.go index 28b807108..eeadef267 100644 --- a/cmd/erasure-createfile.go +++ b/cmd/erasure-createfile.go @@ -41,7 +41,7 @@ func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader // FIXME: this is a bug in Golang, n == 0 and err == // io.ErrUnexpectedEOF for io.ReadFull function. if n == 0 && rErr == io.ErrUnexpectedEOF { - return 0, nil, rErr + return 0, nil, traceError(rErr) } if rErr == io.EOF { // We have reached EOF on the first byte read, io.Reader @@ -58,7 +58,7 @@ func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader break } if rErr != nil && rErr != io.ErrUnexpectedEOF { - return 0, nil, rErr + return 0, nil, traceError(rErr) } if n > 0 { // Returns encoded blocks. @@ -88,19 +88,19 @@ func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader func encodeData(dataBuffer []byte, dataBlocks, parityBlocks int) ([][]byte, error) { rs, err := reedsolomon.New(dataBlocks, parityBlocks) if err != nil { - return nil, err + return nil, traceError(err) } // Split the input buffer into data and parity blocks. var blocks [][]byte blocks, err = rs.Split(dataBuffer) if err != nil { - return nil, err + return nil, traceError(err) } // Encode parity blocks using data blocks. err = rs.Encode(blocks) if err != nil { - return nil, err + return nil, traceError(err) } // Return encoded blocks. @@ -122,7 +122,7 @@ func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, hash defer wg.Done() wErr := disk.AppendFile(volume, path, enBlocks[index]) if wErr != nil { - wErrs[index] = wErr + wErrs[index] = traceError(wErr) return } @@ -139,7 +139,7 @@ func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, hash // Do we have write quorum?. if !isDiskQuorum(wErrs, writeQuorum) { - return errXLWriteQuorum + return traceError(errXLWriteQuorum) } return nil } diff --git a/cmd/erasure-createfile_test.go b/cmd/erasure-createfile_test.go index 5796202f2..6371473f5 100644 --- a/cmd/erasure-createfile_test.go +++ b/cmd/erasure-createfile_test.go @@ -93,8 +93,8 @@ func TestErasureCreateFile(t *testing.T) { // 1 more disk down. 7 disk down in total. Should return quorum error. disks[10] = AppendDiskDown{disks[10].(*posix)} _, _, err = erasureCreateFile(disks, "testbucket", "testobject4", bytes.NewReader(data), blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1) - if err != errXLWriteQuorum { - t.Errorf("erasureCreateFile returned expected errXLWriteQuorum error, got %s", err) + if errorCause(err) != errXLWriteQuorum { + t.Errorf("erasureCreateFile return value: expected errXLWriteQuorum, got %s", err) } } @@ -195,7 +195,7 @@ func TestErasureEncode(t *testing.T) { } // Failed as expected, but does it fail for the expected reason. if actualErr != nil && !testCase.shouldPass { - if testCase.expectedErr != actualErr { + if errorCause(actualErr) != testCase.expectedErr { t.Errorf("Test %d: Expected Error to be \"%v\", but instead found \"%v\" ", i+1, testCase.expectedErr, actualErr) } } diff --git a/cmd/erasure-healfile.go b/cmd/erasure-healfile.go index 56ae7de65..5d029ad5c 100644 --- a/cmd/erasure-healfile.go +++ b/cmd/erasure-healfile.go @@ -64,7 +64,7 @@ func erasureHealFile(latestDisks []StorageAPI, outDatedDisks []StorageAPI, volum } err := disk.AppendFile(healBucket, healPath, enBlocks[index]) if err != nil { - return nil, err + return nil, traceError(err) } hashWriters[index].Write(enBlocks[index]) } diff --git a/cmd/erasure-healfile_test.go b/cmd/erasure-healfile_test.go index 08c0bbc73..036df42ec 100644 --- a/cmd/erasure-healfile_test.go +++ b/cmd/erasure-healfile_test.go @@ -66,7 +66,11 @@ func TestErasureHealFile(t *testing.T) { copy(latest, disks) latest[0] = nil outDated[0] = disks[0] + healCheckSums, err := erasureHealFile(latest, outDated, "testbucket", "testobject1", "testbucket", "testobject1", 1*1024*1024, blockSize, dataBlocks, parityBlocks, bitRotAlgo) + if err != nil { + t.Fatal(err) + } // Checksum of the healed file should match. if checkSums[0] != healCheckSums[0] { t.Error("Healing failed, data does not match.") @@ -116,7 +120,7 @@ func TestErasureHealFile(t *testing.T) { latest[index] = nil outDated[index] = disks[index] } - healCheckSums, err = erasureHealFile(latest, outDated, "testbucket", "testobject1", "testbucket", "testobject1", 1*1024*1024, blockSize, dataBlocks, parityBlocks, bitRotAlgo) + _, err = erasureHealFile(latest, outDated, "testbucket", "testobject1", "testbucket", "testobject1", 1*1024*1024, blockSize, dataBlocks, parityBlocks, bitRotAlgo) if err == nil { t.Error("Expected erasureHealFile() to fail when the number of available disks <= parityBlocks") } diff --git a/cmd/erasure-readfile.go b/cmd/erasure-readfile.go index df20100ee..9f521eec6 100644 --- a/cmd/erasure-readfile.go +++ b/cmd/erasure-readfile.go @@ -84,10 +84,10 @@ func getReadDisks(orderedDisks []StorageAPI, index int, dataBlocks int) (readDis // Sanity checks - we should never have this situation. if dataDisks == dataBlocks { - return nil, 0, errUnexpected + return nil, 0, traceError(errUnexpected) } if dataDisks+parityDisks >= dataBlocks { - return nil, 0, errUnexpected + return nil, 0, traceError(errUnexpected) } // Find the disks from which next set of parallel reads should happen. @@ -107,7 +107,7 @@ func getReadDisks(orderedDisks []StorageAPI, index int, dataBlocks int) (readDis return readDisks, i + 1, nil } } - return nil, 0, errXLReadQuorum + return nil, 0, traceError(errXLReadQuorum) } // parallelRead - reads chunks in parallel from the disks specified in []readDisks. @@ -161,12 +161,12 @@ func parallelRead(volume, path string, readDisks []StorageAPI, orderedDisks []St func erasureReadFile(writer io.Writer, disks []StorageAPI, volume string, path string, offset int64, length int64, totalLength int64, blockSize int64, dataBlocks int, parityBlocks int, checkSums []string, algo string, pool *bpool.BytePool) (int64, error) { // Offset and length cannot be negative. if offset < 0 || length < 0 { - return 0, errUnexpected + return 0, traceError(errUnexpected) } // Can't request more data than what is available. if offset+length > totalLength { - return 0, errUnexpected + return 0, traceError(errUnexpected) } // chunkSize is the amount of data that needs to be read from each disk at a time. @@ -248,7 +248,7 @@ func erasureReadFile(writer io.Writer, disks []StorageAPI, volume string, path s } if nextIndex == len(disks) { // No more disks to read from. - return bytesWritten, errXLReadQuorum + return bytesWritten, traceError(errXLReadQuorum) } // We do not have enough enough data blocks to reconstruct the data // hence continue the for-loop till we have enough data blocks. @@ -325,24 +325,24 @@ func decodeData(enBlocks [][]byte, dataBlocks, parityBlocks int) error { // Initialized reedsolomon. rs, err := reedsolomon.New(dataBlocks, parityBlocks) if err != nil { - return err + return traceError(err) } // Reconstruct encoded blocks. err = rs.Reconstruct(enBlocks) if err != nil { - return err + return traceError(err) } // Verify reconstructed blocks (parity). ok, err := rs.Verify(enBlocks) if err != nil { - return err + return traceError(err) } if !ok { // Blocks cannot be reconstructed, corrupted data. err = errors.New("Verification failed after reconstruction, data likely corrupted.") - return err + return traceError(err) } // Success. diff --git a/cmd/erasure-readfile_test.go b/cmd/erasure-readfile_test.go index 7e2a6fb7b..0d972062f 100644 --- a/cmd/erasure-readfile_test.go +++ b/cmd/erasure-readfile_test.go @@ -104,7 +104,7 @@ func testGetReadDisks(t *testing.T, xl xlObjects) { for i, test := range testCases { disks, nextIndex, err := getReadDisks(test.argDisks, test.index, xl.dataBlocks) - if err != test.err { + if errorCause(err) != test.err { t.Errorf("test-case %d - expected error : %s, got : %s", i+1, test.err, err) continue } @@ -217,11 +217,16 @@ func TestIsSuccessBlocks(t *testing.T) { // Wrapper function for testGetReadDisks, testGetOrderedDisks. func TestErasureReadUtils(t *testing.T) { - objLayer, dirs, err := getXLObjectLayer() + nDisks := 16 + disks, err := getRandomDisks(nDisks) if err != nil { t.Fatal(err) } - defer removeRoots(dirs) + objLayer, err := getXLObjectLayer(disks) + if err != nil { + t.Fatal(err) + } + defer removeRoots(disks) xl := objLayer.(xlObjects) testGetReadDisks(t, xl) testGetOrderedDisks(t, xl) @@ -314,7 +319,7 @@ func TestErasureReadFileDiskFail(t *testing.T) { disks[13] = ReadDiskDown{disks[13].(*posix)} buf.Reset() _, err = erasureReadFile(buf, disks, "testbucket", "testobject", 0, length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool) - if err != errXLReadQuorum { + if errorCause(err) != errXLReadQuorum { t.Fatal("expected errXLReadQuorum error") } } diff --git a/cmd/erasure-utils.go b/cmd/erasure-utils.go index ec8a55f57..2f05f027a 100644 --- a/cmd/erasure-utils.go +++ b/cmd/erasure-utils.go @@ -76,17 +76,17 @@ func getDataBlockLen(enBlocks [][]byte, dataBlocks int) int { func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset int64, length int64) (int64, error) { // Offset and out size cannot be negative. if offset < 0 || length < 0 { - return 0, errUnexpected + return 0, traceError(errUnexpected) } // Do we have enough blocks? if len(enBlocks) < dataBlocks { - return 0, reedsolomon.ErrTooFewShards + return 0, traceError(reedsolomon.ErrTooFewShards) } // Do we have enough data? if int64(getDataBlockLen(enBlocks, dataBlocks)) < length { - return 0, reedsolomon.ErrShortData + return 0, traceError(reedsolomon.ErrShortData) } // Counter to decrement total left to write. @@ -114,7 +114,7 @@ func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset in if write < int64(len(block)) { n, err := io.Copy(dst, bytes.NewReader(block[:write])) if err != nil { - return 0, err + return 0, traceError(err) } totalWritten += n break @@ -122,7 +122,7 @@ func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset in // Copy the block. n, err := io.Copy(dst, bytes.NewReader(block)) if err != nil { - return 0, err + return 0, traceError(err) } // Decrement output size. diff --git a/cmd/errors.go b/cmd/errors.go new file mode 100644 index 000000000..b6fbf1984 --- /dev/null +++ b/cmd/errors.go @@ -0,0 +1,122 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "strings" +) + +// Holds the current directory path. Used for trimming path in traceError() +var rootPath string + +// Figure out the rootPath +func initError() { + // Root path is automatically determined from the calling function's source file location. + // Catch the calling function's source file path. + _, file, _, _ := runtime.Caller(1) + // Save the directory alone. + rootPath = filepath.Dir(file) +} + +// Represents a stack frame in the stack trace. +type traceInfo struct { + file string // File where error occurred + line int // Line where error occurred + name string // Name of the function where error occurred +} + +// Error - error type containing cause and the stack trace. +type Error struct { + e error // Holds the cause error + trace []traceInfo // stack trace + errs []error // Useful for XL to hold errors from all disks +} + +// Implement error interface. +func (e Error) Error() string { + return e.e.Error() +} + +// Trace - returns stack trace. +func (e Error) Trace() []string { + var traceArr []string + for _, info := range e.trace { + traceArr = append(traceArr, fmt.Sprintf("%s:%d:%s", + info.file, info.line, info.name)) + } + return traceArr +} + +// NewStorageError - return new Error type. +func traceError(e error, errs ...error) error { + if e == nil { + return nil + } + err := &Error{} + err.e = e + err.errs = errs + + stack := make([]uintptr, 40) + length := runtime.Callers(2, stack) + if length > len(stack) { + length = len(stack) + } + stack = stack[:length] + + for _, pc := range stack { + pc = pc - 1 + fn := runtime.FuncForPC(pc) + file, line := fn.FileLine(pc) + name := fn.Name() + if strings.HasSuffix(name, "ServeHTTP") { + break + } + if strings.HasSuffix(name, "runtime.") { + break + } + + file = strings.TrimPrefix(file, rootPath+string(os.PathSeparator)) + name = strings.TrimPrefix(name, "github.com/minio/minio/cmd.") + err.trace = append(err.trace, traceInfo{file, line, name}) + } + + return err +} + +// Returns the underlying cause error. +func errorCause(err error) error { + if e, ok := err.(*Error); ok { + err = e.e + } + return err +} + +// Returns slice of underlying cause error. +func errorsCause(errs []error) []error { + Errs := make([]error, len(errs)) + for i, err := range errs { + if err == nil { + continue + } + Errs[i] = errorCause(err) + } + return Errs +} diff --git a/cmd/event-notifier.go b/cmd/event-notifier.go index cef9e4438..c872d1891 100644 --- a/cmd/event-notifier.go +++ b/cmd/event-notifier.go @@ -20,6 +20,7 @@ import ( "bytes" "encoding/xml" "fmt" + "net" "net/url" "path" "sync" @@ -226,6 +227,7 @@ func loadNotificationConfig(bucket string, objAPI ObjectLayer) (*notificationCon // Construct the notification config path. notificationConfigPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig) objInfo, err := objAPI.GetObjectInfo(minioMetaBucket, notificationConfigPath) + err = errorCause(err) if err != nil { // 'notification.xml' not found return 'errNoSuchNotifications'. // This is default when no bucket notifications are found on the bucket. @@ -233,11 +235,13 @@ func loadNotificationConfig(bucket string, objAPI ObjectLayer) (*notificationCon case ObjectNotFound: return nil, errNoSuchNotifications } + errorIf(err, "Unable to load bucket-notification for bucket %s", bucket) // Returns error for other errors. return nil, err } var buffer bytes.Buffer err = objAPI.GetObject(minioMetaBucket, notificationConfigPath, 0, objInfo.Size, &buffer) + err = errorCause(err) if err != nil { // 'notification.xml' not found return 'errNoSuchNotifications'. // This is default when no bucket notifications are found on the bucket. @@ -245,6 +249,7 @@ func loadNotificationConfig(bucket string, objAPI ObjectLayer) (*notificationCon case ObjectNotFound: return nil, errNoSuchNotifications } + errorIf(err, "Unable to load bucket-notification for bucket %s", bucket) // Returns error for other errors. return nil, err } @@ -272,13 +277,12 @@ func loadAllBucketNotifications(objAPI ObjectLayer) (map[string]*notificationCon // Loads all bucket notifications. for _, bucket := range buckets { - var nCfg *notificationConfig - nCfg, err = loadNotificationConfig(bucket.Name, objAPI) - if err != nil { - if err == errNoSuchNotifications { + nCfg, nErr := loadNotificationConfig(bucket.Name, objAPI) + if nErr != nil { + if nErr == errNoSuchNotifications { continue } - return nil, err + return nil, nErr } configs[bucket.Name] = nCfg } @@ -308,6 +312,14 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) { // Using accountID we can now initialize a new AMQP logrus instance. amqpLog, err := newAMQPNotify(accountID) if err != nil { + // Encapsulate network error to be more informative. + if _, ok := err.(net.Error); ok { + return nil, &net.OpError{ + Op: "Connecting to " + queueARN, + Net: "tcp", + Err: err, + } + } return nil, err } queueTargets[queueARN] = amqpLog @@ -327,6 +339,14 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) { // Using accountID we can now initialize a new Redis logrus instance. redisLog, err := newRedisNotify(accountID) if err != nil { + // Encapsulate network error to be more informative. + if _, ok := err.(net.Error); ok { + return nil, &net.OpError{ + Op: "Connecting to " + queueARN, + Net: "tcp", + Err: err, + } + } return nil, err } queueTargets[queueARN] = redisLog @@ -345,6 +365,13 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) { // Using accountID we can now initialize a new ElasticSearch logrus instance. elasticLog, err := newElasticNotify(accountID) if err != nil { + // Encapsulate network error to be more informative. + if _, ok := err.(net.Error); ok { + return nil, &net.OpError{ + Op: "Connecting to " + queueARN, Net: "tcp", + Err: err, + } + } return nil, err } queueTargets[queueARN] = elasticLog diff --git a/cmd/event-notifier_test.go b/cmd/event-notifier_test.go index c43f8569a..6b077dba1 100644 --- a/cmd/event-notifier_test.go +++ b/cmd/event-notifier_test.go @@ -86,16 +86,25 @@ func testEventNotify(obj ObjectLayer, instanceType string, t TestErrHandler) { // Tests various forms of inititalization of event notifier. func TestInitEventNotifier(t *testing.T) { - fs, disk, err := getSingleNodeObjectLayer() + disk, err := getRandomDisks(1) + if err != nil { + t.Fatal("Unable to create directories for FS backend. ", err) + } + fs, err := getSingleNodeObjectLayer(disk[0]) if err != nil { t.Fatal("Unable to initialize FS backend.", err) } - xl, disks, err := getXLObjectLayer() + nDisks := 16 + disks, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal("Unable to create directories for XL backend. ", err) + } + xl, err := getXLObjectLayer(disks) if err != nil { t.Fatal("Unable to initialize XL backend.", err) } - disks = append(disks, disk) + disks = append(disks, disk...) for _, d := range disks { defer removeAll(d) } diff --git a/cmd/format-config-v1.go b/cmd/format-config-v1.go index 1036b21e4..f75a060d1 100644 --- a/cmd/format-config-v1.go +++ b/cmd/format-config-v1.go @@ -189,7 +189,7 @@ func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatConfigV1, []error) { } } // Return all formats and nil - return formatConfigs, nil + return formatConfigs, sErrs } // genericFormatCheck - validates and returns error. @@ -524,6 +524,11 @@ func healFormatXLFreshDisks(storageDisks []StorageAPI) error { } } + // Initialize meta volume, if volume already exists ignores it. + if err := initMetaVolume(orderedDisks); err != nil { + return fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err) + } + // Save new `format.json` across all disks, in JBOD order. return saveFormatXL(orderedDisks, newFormatConfigs) } @@ -870,6 +875,11 @@ func initFormatXL(storageDisks []StorageAPI) (err error) { formats[index].XL.JBOD = jbod } + // Initialize meta volume, if volume already exists ignores it. + if err := initMetaVolume(storageDisks); err != nil { + return fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err) + } + // Save formats `format.json` across all disks. return saveFormatXL(storageDisks, formats) } diff --git a/cmd/format-config-v1_test.go b/cmd/format-config-v1_test.go index 726f9958a..a9c44cc37 100644 --- a/cmd/format-config-v1_test.go +++ b/cmd/format-config-v1_test.go @@ -215,7 +215,6 @@ func genFormatXLInvalidDisksOrder() []*formatConfigV1 { } func prepareFormatXLHealFreshDisks(obj ObjectLayer) ([]StorageAPI, error) { - var err error xl := obj.(xlObjects) @@ -263,8 +262,13 @@ func prepareFormatXLHealFreshDisks(obj ObjectLayer) ([]StorageAPI, error) { } func TestFormatXLHealFreshDisks(t *testing.T) { + nDisks := 16 + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } // Create an instance of xl backend. - obj, fsDirs, err := getXLObjectLayer() + obj, err := getXLObjectLayer(fsDirs) if err != nil { t.Error(err) } @@ -290,8 +294,13 @@ func TestFormatXLHealFreshDisks(t *testing.T) { } func TestFormatXLHealFreshDisksErrorExpected(t *testing.T) { + nDisks := 16 + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } // Create an instance of xl backend. - obj, fsDirs, err := getXLObjectLayer() + obj, err := getXLObjectLayer(fsDirs) if err != nil { t.Error(err) } @@ -326,7 +335,7 @@ func TestFormatXLHealFreshDisksErrorExpected(t *testing.T) { // a given disk to test healing a corrupted disk func TestFormatXLHealCorruptedDisks(t *testing.T) { // Create an instance of xl backend. - obj, fsDirs, err := getXLObjectLayer() + obj, fsDirs, err := prepareXL() if err != nil { t.Fatal(err) } @@ -398,7 +407,7 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) { // some of format.json func TestFormatXLReorderByInspection(t *testing.T) { // Create an instance of xl backend. - obj, fsDirs, err := getXLObjectLayer() + obj, fsDirs, err := prepareXL() if err != nil { t.Fatal(err) } @@ -569,8 +578,13 @@ func TestSavedUUIDOrder(t *testing.T) { // Test initFormatXL() when disks are expected to return errors func TestInitFormatXLErrors(t *testing.T) { + nDisks := 16 + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } // Create an instance of xl backend. - obj, fsDirs, err := getXLObjectLayer() + obj, err := getXLObjectLayer(fsDirs) if err != nil { t.Fatal(err) } @@ -659,8 +673,14 @@ func TestGenericFormatCheck(t *testing.T) { } func TestLoadFormatXLErrs(t *testing.T) { + nDisks := 16 + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + // Create an instance of xl backend. - obj, fsDirs, err := getXLObjectLayer() + obj, err := getXLObjectLayer(fsDirs) if err != nil { t.Fatal(err) } @@ -680,7 +700,12 @@ func TestLoadFormatXLErrs(t *testing.T) { removeRoots(fsDirs) - obj, fsDirs, err = getXLObjectLayer() + fsDirs, err = getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + + obj, err = getXLObjectLayer(fsDirs) if err != nil { t.Fatal(err) } @@ -700,7 +725,12 @@ func TestLoadFormatXLErrs(t *testing.T) { removeRoots(fsDirs) - obj, fsDirs, err = getXLObjectLayer() + fsDirs, err = getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + + obj, err = getXLObjectLayer(fsDirs) if err != nil { t.Fatal(err) } @@ -718,7 +748,12 @@ func TestLoadFormatXLErrs(t *testing.T) { removeRoots(fsDirs) - obj, fsDirs, err = getXLObjectLayer() + fsDirs, err = getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + + obj, err = getXLObjectLayer(fsDirs) if err != nil { t.Fatal(err) } @@ -737,8 +772,14 @@ func TestLoadFormatXLErrs(t *testing.T) { // Tests for healFormatXLCorruptedDisks() with cases which lead to errors func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { + nDisks := 16 + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + // Everything is fine, should return nil - obj, fsDirs, err := getXLObjectLayer() + obj, err := getXLObjectLayer(fsDirs) if err != nil { t.Fatal(err) } @@ -746,10 +787,16 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil { t.Fatal("Got an unexpected error: ", err) } + removeRoots(fsDirs) + fsDirs, err = getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + // Disks 0..15 are nil - obj, fsDirs, err = getXLObjectLayer() + obj, err = getXLObjectLayer(fsDirs) if err != nil { t.Fatal(err) } @@ -762,8 +809,13 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } removeRoots(fsDirs) + fsDirs, err = getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + // One disk returns Faulty Disk - obj, fsDirs, err = getXLObjectLayer() + obj, err = getXLObjectLayer(fsDirs) if err != nil { t.Fatal(err) } @@ -778,8 +830,13 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } removeRoots(fsDirs) + fsDirs, err = getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + // One disk is not found, heal corrupted disks should return nil - obj, fsDirs, err = getXLObjectLayer() + obj, err = getXLObjectLayer(fsDirs) if err != nil { t.Fatal(err) } @@ -790,8 +847,13 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } removeRoots(fsDirs) + fsDirs, err = getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + // Remove format.json of all disks - obj, fsDirs, err = getXLObjectLayer() + obj, err = getXLObjectLayer(fsDirs) if err != nil { t.Fatal(err) } @@ -806,8 +868,13 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } removeRoots(fsDirs) + fsDirs, err = getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + // Corrupted format json in one disk - obj, fsDirs, err = getXLObjectLayer() + obj, err = getXLObjectLayer(fsDirs) if err != nil { t.Fatal(err) } @@ -825,8 +892,14 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { // Tests for healFormatXLFreshDisks() with cases which lead to errors func TestHealFormatXLFreshDisksErrs(t *testing.T) { + nDisks := 16 + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + // Everything is fine, should return nil - obj, fsDirs, err := getXLObjectLayer() + obj, err := getXLObjectLayer(fsDirs) if err != nil { t.Fatal(err) } @@ -836,8 +909,13 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } removeRoots(fsDirs) + fsDirs, err = getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + // Disks 0..15 are nil - obj, fsDirs, err = getXLObjectLayer() + obj, err = getXLObjectLayer(fsDirs) if err != nil { t.Fatal(err) } @@ -850,8 +928,13 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } removeRoots(fsDirs) + fsDirs, err = getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + // One disk returns Faulty Disk - obj, fsDirs, err = getXLObjectLayer() + obj, err = getXLObjectLayer(fsDirs) if err != nil { t.Fatal(err) } @@ -866,8 +949,13 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } removeRoots(fsDirs) + fsDirs, err = getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + // One disk is not found, heal corrupted disks should return nil - obj, fsDirs, err = getXLObjectLayer() + obj, err = getXLObjectLayer(fsDirs) if err != nil { t.Fatal(err) } @@ -878,8 +966,13 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } removeRoots(fsDirs) + fsDirs, err = getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + // Remove format.json of all disks - obj, fsDirs, err = getXLObjectLayer() + obj, err = getXLObjectLayer(fsDirs) if err != nil { t.Fatal(err) } @@ -894,8 +987,13 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } removeRoots(fsDirs) + fsDirs, err = getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + // Remove format.json of all disks - obj, fsDirs, err = getXLObjectLayer() + obj, err = getXLObjectLayer(fsDirs) if err != nil { t.Fatal(err) } diff --git a/cmd/fs-createfile.go b/cmd/fs-createfile.go index 2e32c6fbb..a0ec0a6c7 100644 --- a/cmd/fs-createfile.go +++ b/cmd/fs-createfile.go @@ -24,13 +24,13 @@ func fsCreateFile(disk StorageAPI, reader io.Reader, buf []byte, tmpBucket, temp for { n, rErr := reader.Read(buf) if rErr != nil && rErr != io.EOF { - return 0, rErr + return 0, traceError(rErr) } bytesWritten += int64(n) if n > 0 { wErr := disk.AppendFile(tmpBucket, tempObj, buf[0:n]) if wErr != nil { - return 0, wErr + return 0, traceError(wErr) } } if rErr == io.EOF { diff --git a/cmd/fs-v1-metadata.go b/cmd/fs-v1-metadata.go index 365b800c7..6f1f6346d 100644 --- a/cmd/fs-v1-metadata.go +++ b/cmd/fs-v1-metadata.go @@ -81,12 +81,12 @@ func readFSMetadata(disk StorageAPI, bucket, filePath string) (fsMeta fsMetaV1, // Read all `fs.json`. buf, err := disk.ReadAll(bucket, filePath) if err != nil { - return fsMetaV1{}, err + return fsMetaV1{}, traceError(err) } // Decode `fs.json` into fsMeta structure. if err = json.Unmarshal(buf, &fsMeta); err != nil { - return fsMetaV1{}, err + return fsMetaV1{}, traceError(err) } // Success. @@ -94,16 +94,23 @@ func readFSMetadata(disk StorageAPI, bucket, filePath string) (fsMeta fsMetaV1, } // Write fsMeta to fs.json or fs-append.json. -func writeFSMetadata(disk StorageAPI, bucket, filePath string, fsMeta fsMetaV1) (err error) { +func writeFSMetadata(disk StorageAPI, bucket, filePath string, fsMeta fsMetaV1) error { tmpPath := path.Join(tmpMetaPrefix, getUUID()) metadataBytes, err := json.Marshal(fsMeta) if err != nil { - return err + return traceError(err) } if err = disk.AppendFile(minioMetaBucket, tmpPath, metadataBytes); err != nil { - return err + return traceError(err) } - return disk.RenameFile(minioMetaBucket, tmpPath, bucket, filePath) + err = disk.RenameFile(minioMetaBucket, tmpPath, bucket, filePath) + if err != nil { + err = disk.DeleteFile(minioMetaBucket, tmpPath) + if err != nil { + return traceError(err) + } + } + return nil } // newFSMetaV1 - initializes new fsMetaV1. diff --git a/cmd/fs-v1-multipart-common.go b/cmd/fs-v1-multipart-common.go index c3ccdbdf7..3baf39242 100644 --- a/cmd/fs-v1-multipart-common.go +++ b/cmd/fs-v1-multipart-common.go @@ -64,8 +64,8 @@ func (fs fsObjects) writeUploadJSON(bucket, object, uploadID string, initiated t var uploadsJSON uploadsV1 uploadsJSON, err = readUploadsJSON(bucket, object, fs.storage) if err != nil { - // For any other errors. - if err != errFileNotFound { + // uploads.json might not exist hence ignore errFileNotFound. + if errorCause(err) != errFileNotFound { return err } // Set uploads format to `fs`. @@ -77,18 +77,18 @@ func (fs fsObjects) writeUploadJSON(bucket, object, uploadID string, initiated t // Update `uploads.json` on all disks. uploadsJSONBytes, wErr := json.Marshal(&uploadsJSON) if wErr != nil { - return wErr + return traceError(wErr) } // Write `uploads.json` to disk. if wErr = fs.storage.AppendFile(minioMetaBucket, tmpUploadsPath, uploadsJSONBytes); wErr != nil { - return wErr + return traceError(wErr) } wErr = fs.storage.RenameFile(minioMetaBucket, tmpUploadsPath, minioMetaBucket, uploadsPath) if wErr != nil { if dErr := fs.storage.DeleteFile(minioMetaBucket, tmpUploadsPath); dErr != nil { - return dErr + return traceError(dErr) } - return wErr + return traceError(wErr) } return nil } @@ -100,13 +100,13 @@ func (fs fsObjects) updateUploadsJSON(bucket, object string, uploadsJSON uploads tmpUploadsPath := path.Join(tmpMetaPrefix, uniqueID) uploadsBytes, wErr := json.Marshal(uploadsJSON) if wErr != nil { - return wErr + return traceError(wErr) } if wErr = fs.storage.AppendFile(minioMetaBucket, tmpUploadsPath, uploadsBytes); wErr != nil { - return wErr + return traceError(wErr) } if wErr = fs.storage.RenameFile(minioMetaBucket, tmpUploadsPath, minioMetaBucket, uploadsPath); wErr != nil { - return wErr + return traceError(wErr) } return nil } diff --git a/cmd/fs-v1-multipart.go b/cmd/fs-v1-multipart.go index 0eff2f38b..37d9965fb 100644 --- a/cmd/fs-v1-multipart.go +++ b/cmd/fs-v1-multipart.go @@ -58,9 +58,12 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark var err error var eof bool if uploadIDMarker != "" { - nsMutex.RLock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, keyMarker)) + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + nsMutex.RLock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, keyMarker), opsID) uploads, _, err = listMultipartUploadIDs(bucket, keyMarker, uploadIDMarker, maxUploads, fs.storage) - nsMutex.RUnlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, keyMarker)) + nsMutex.RUnlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, keyMarker), opsID) if err != nil { return ListMultipartsInfo{}, err } @@ -91,7 +94,7 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark eof = true break } - return ListMultipartsInfo{}, err + return ListMultipartsInfo{}, walkResult.err } entry := strings.TrimPrefix(walkResult.entry, retainSlash(pathJoin(mpartMetaPrefix, bucket))) if strings.HasSuffix(walkResult.entry, slashSeparator) { @@ -110,9 +113,14 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark var tmpUploads []uploadMetadata var end bool uploadIDMarker = "" - nsMutex.RLock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, entry)) + + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + + nsMutex.RLock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, entry), opsID) tmpUploads, end, err = listMultipartUploadIDs(bucket, entry, uploadIDMarker, maxUploads, fs.storage) - nsMutex.RUnlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, entry)) + nsMutex.RUnlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, entry), opsID) if err != nil { return ListMultipartsInfo{}, err } @@ -168,42 +176,42 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark func (fs fsObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) { // Validate input arguments. if !IsValidBucketName(bucket) { - return ListMultipartsInfo{}, BucketNameInvalid{Bucket: bucket} + return ListMultipartsInfo{}, traceError(BucketNameInvalid{Bucket: bucket}) } if !fs.isBucketExist(bucket) { - return ListMultipartsInfo{}, BucketNotFound{Bucket: bucket} + return ListMultipartsInfo{}, traceError(BucketNotFound{Bucket: bucket}) } if !IsValidObjectPrefix(prefix) { - return ListMultipartsInfo{}, ObjectNameInvalid{Bucket: bucket, Object: prefix} + return ListMultipartsInfo{}, traceError(ObjectNameInvalid{Bucket: bucket, Object: prefix}) } // Verify if delimiter is anything other than '/', which we do not support. if delimiter != "" && delimiter != slashSeparator { - return ListMultipartsInfo{}, UnsupportedDelimiter{ + return ListMultipartsInfo{}, traceError(UnsupportedDelimiter{ Delimiter: delimiter, - } + }) } // Verify if marker has prefix. if keyMarker != "" && !strings.HasPrefix(keyMarker, prefix) { - return ListMultipartsInfo{}, InvalidMarkerPrefixCombination{ + return ListMultipartsInfo{}, traceError(InvalidMarkerPrefixCombination{ Marker: keyMarker, Prefix: prefix, - } + }) } if uploadIDMarker != "" { if strings.HasSuffix(keyMarker, slashSeparator) { - return ListMultipartsInfo{}, InvalidUploadIDKeyCombination{ + return ListMultipartsInfo{}, traceError(InvalidUploadIDKeyCombination{ UploadIDMarker: uploadIDMarker, KeyMarker: keyMarker, - } + }) } id, err := uuid.Parse(uploadIDMarker) if err != nil { - return ListMultipartsInfo{}, err + return ListMultipartsInfo{}, traceError(err) } if id.IsZero() { - return ListMultipartsInfo{}, MalformedUploadID{ + return ListMultipartsInfo{}, traceError(MalformedUploadID{ UploadID: uploadIDMarker, - } + }) } } return fs.listMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads) @@ -213,7 +221,7 @@ func (fs fsObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMark // request, returns back a unique upload id. // // Internally this function creates 'uploads.json' associated for the -// incoming object at '.minio/multipart/bucket/object/uploads.json' on +// incoming object at '.minio.sys/multipart/bucket/object/uploads.json' on // all the disks. `uploads.json` carries metadata regarding on going // multipart operation on the object. func (fs fsObjects) newMultipartUpload(bucket string, object string, meta map[string]string) (uploadID string, err error) { @@ -225,9 +233,13 @@ func (fs fsObjects) newMultipartUpload(bucket string, object string, meta map[st fsMeta.Meta = meta } - // This lock needs to be held for any changes to the directory contents of ".minio/multipart/object/" - nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object)) - defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object)) + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + + // This lock needs to be held for any changes to the directory contents of ".minio.sys/multipart/object/" + nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object), opsID) + defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object), opsID) uploadID = getUUID() initiated := time.Now().UTC() @@ -235,9 +247,9 @@ func (fs fsObjects) newMultipartUpload(bucket string, object string, meta map[st if err = fs.writeUploadJSON(bucket, object, uploadID, initiated); err != nil { return "", err } - fsMetaPath := path.Join(mpartMetaPrefix, bucket, object, uploadID, fsMetaJSONFile) - if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil { - return "", toObjectErr(err, minioMetaBucket, fsMetaPath) + uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID) + if err = writeFSMetadata(fs.storage, minioMetaBucket, path.Join(uploadIDPath, fsMetaJSONFile), fsMeta); err != nil { + return "", toObjectErr(err, minioMetaBucket, uploadIDPath) } // Return success. return uploadID, nil @@ -251,15 +263,15 @@ func (fs fsObjects) newMultipartUpload(bucket string, object string, meta map[st func (fs fsObjects) NewMultipartUpload(bucket, object string, meta map[string]string) (string, error) { // Verify if bucket name is valid. if !IsValidBucketName(bucket) { - return "", BucketNameInvalid{Bucket: bucket} + return "", traceError(BucketNameInvalid{Bucket: bucket}) } // Verify whether the bucket exists. if !fs.isBucketExist(bucket) { - return "", BucketNotFound{Bucket: bucket} + return "", traceError(BucketNotFound{Bucket: bucket}) } // Verify if object name is valid. if !IsValidObjectName(object) { - return "", ObjectNameInvalid{Bucket: bucket, Object: object} + return "", traceError(ObjectNameInvalid{Bucket: bucket, Object: object}) } return fs.newMultipartUpload(bucket, object, meta) } @@ -290,7 +302,14 @@ func getFSAppendDataPath(uploadID string) string { } // Append parts to fsAppendDataFile. -func appendParts(disk StorageAPI, bucket, object, uploadID string) { +func appendParts(disk StorageAPI, bucket, object, uploadID, opsID string) { + cleanupAppendPaths := func() { + // In case of any error, cleanup the append data and json files + // from the tmp so that we do not have any inconsistent append + // data/json files. + disk.DeleteFile(bucket, getFSAppendDataPath(uploadID)) + disk.DeleteFile(bucket, getFSAppendMetaPath(uploadID)) + } uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID) // fs-append.json path fsAppendMetaPath := getFSAppendMetaPath(uploadID) @@ -298,20 +317,21 @@ func appendParts(disk StorageAPI, bucket, object, uploadID string) { fsMetaPath := path.Join(mpartMetaPrefix, bucket, object, uploadID, fsMetaJSONFile) // Lock the uploadID so that no one modifies fs.json - nsMutex.RLock(minioMetaBucket, uploadIDPath) + nsMutex.RLock(minioMetaBucket, uploadIDPath, opsID) fsMeta, err := readFSMetadata(disk, minioMetaBucket, fsMetaPath) - nsMutex.RUnlock(minioMetaBucket, uploadIDPath) + nsMutex.RUnlock(minioMetaBucket, uploadIDPath, opsID) if err != nil { return } // Lock fs-append.json so that there is no parallel append to the file. - nsMutex.Lock(minioMetaBucket, fsAppendMetaPath) - defer nsMutex.Unlock(minioMetaBucket, fsAppendMetaPath) + nsMutex.Lock(minioMetaBucket, fsAppendMetaPath, opsID) + defer nsMutex.Unlock(minioMetaBucket, fsAppendMetaPath, opsID) fsAppendMeta, err := readFSMetadata(disk, minioMetaBucket, fsAppendMetaPath) if err != nil { - if err != errFileNotFound { + if errorCause(err) != errFileNotFound { + cleanupAppendPaths() return } fsAppendMeta = fsMeta @@ -324,28 +344,14 @@ func appendParts(disk StorageAPI, bucket, object, uploadID string) { return } // Hold write lock on the part so that there is no parallel upload on the part. - nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID, strconv.Itoa(part.Number))) - defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID, strconv.Itoa(part.Number))) + partPath := pathJoin(mpartMetaPrefix, bucket, object, uploadID, strconv.Itoa(part.Number)) + nsMutex.Lock(minioMetaBucket, partPath, opsID) + defer nsMutex.Unlock(minioMetaBucket, partPath, opsID) // Proceed to append "part" fsAppendDataPath := getFSAppendDataPath(uploadID) - tmpDataPath := path.Join(tmpMetaPrefix, getUUID()) - if part.Number != 1 { - // Move it to tmp location before appending so that we don't leave inconsitent data - // if server crashes during append operation. - err = disk.RenameFile(minioMetaBucket, fsAppendDataPath, minioMetaBucket, tmpDataPath) - if err != nil { - return - } - // Delete fs-append.json so that we don't leave a stale file if server crashes - // when the part is being appended to the tmp file. - err = disk.DeleteFile(minioMetaBucket, fsAppendMetaPath) - if err != nil { - return - } - } // Path to the part that needs to be appended. - partPath := path.Join(mpartMetaPrefix, bucket, object, uploadID, part.Name) + partPath = path.Join(mpartMetaPrefix, bucket, object, uploadID, part.Name) offset := int64(0) totalLeft := part.Size buf := make([]byte, readSizeV1) @@ -357,7 +363,8 @@ func appendParts(disk StorageAPI, bucket, object, uploadID string) { var n int64 n, err = disk.ReadFile(minioMetaBucket, partPath, offset, buf[:curLeft]) if n > 0 { - if err = disk.AppendFile(minioMetaBucket, tmpDataPath, buf[:n]); err != nil { + if err = disk.AppendFile(minioMetaBucket, fsAppendDataPath, buf[:n]); err != nil { + cleanupAppendPaths() return } } @@ -365,51 +372,54 @@ func appendParts(disk StorageAPI, bucket, object, uploadID string) { if err == io.EOF || err == io.ErrUnexpectedEOF { break } + cleanupAppendPaths() return } offset += n totalLeft -= n } - // All good, the part has been appended to the tmp file, rename it back. - if err = disk.RenameFile(minioMetaBucket, tmpDataPath, minioMetaBucket, fsAppendDataPath); err != nil { - return - } fsAppendMeta.AddObjectPart(part.Number, part.Name, part.ETag, part.Size) + // Overwrite previous fs-append.json if err = writeFSMetadata(disk, minioMetaBucket, fsAppendMetaPath, fsAppendMeta); err != nil { + cleanupAppendPaths() return } // If there are more parts that need to be appended to fsAppendDataFile _, appendNeeded = partToAppend(fsMeta, fsAppendMeta) if appendNeeded { - go appendParts(disk, bucket, object, uploadID) + go appendParts(disk, bucket, object, uploadID, opsID) } } // PutObjectPart - reads incoming data until EOF for the part file on // an ongoing multipart transaction. Internally incoming data is -// written to '.minio/tmp' location and safely renamed to -// '.minio/multipart' for reach parts. +// written to '.minio.sys/tmp' location and safely renamed to +// '.minio.sys/multipart' for reach parts. func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return "", BucketNameInvalid{Bucket: bucket} + return "", traceError(BucketNameInvalid{Bucket: bucket}) } // Verify whether the bucket exists. if !fs.isBucketExist(bucket) { - return "", BucketNotFound{Bucket: bucket} + return "", traceError(BucketNotFound{Bucket: bucket}) } if !IsValidObjectName(object) { - return "", ObjectNameInvalid{Bucket: bucket, Object: object} + return "", traceError(ObjectNameInvalid{Bucket: bucket, Object: object}) } uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID) - nsMutex.RLock(minioMetaBucket, uploadIDPath) + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + + nsMutex.RLock(minioMetaBucket, uploadIDPath, opsID) // Just check if the uploadID exists to avoid copy if it doesn't. uploadIDExists := fs.isUploadIDExists(bucket, object, uploadID) - nsMutex.RUnlock(minioMetaBucket, uploadIDPath) + nsMutex.RUnlock(minioMetaBucket, uploadIDPath, opsID) if !uploadIDExists { - return "", InvalidUploadID{UploadID: uploadID} + return "", traceError(InvalidUploadID{UploadID: uploadID}) } partSuffix := fmt.Sprintf("object%d", partID) @@ -443,7 +453,7 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s // bytes than specified in request header. if bytesWritten < size { fs.storage.DeleteFile(minioMetaBucket, tmpPartPath) - return "", IncompleteBody{} + return "", traceError(IncompleteBody{}) } // Validate if payload is valid. @@ -452,7 +462,7 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s // Incoming payload wrong, delete the temporary object. fs.storage.DeleteFile(minioMetaBucket, tmpPartPath) // Error return. - return "", toObjectErr(err, bucket, object) + return "", toObjectErr(traceError(err), bucket, object) } } @@ -462,17 +472,21 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s // MD5 mismatch, delete the temporary object. fs.storage.DeleteFile(minioMetaBucket, tmpPartPath) // Returns md5 mismatch. - return "", BadDigest{md5Hex, newMD5Hex} + return "", traceError(BadDigest{md5Hex, newMD5Hex}) } } + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID = getOpsID() + // Hold write lock as we are updating fs.json - nsMutex.Lock(minioMetaBucket, uploadIDPath) - defer nsMutex.Unlock(minioMetaBucket, uploadIDPath) + nsMutex.Lock(minioMetaBucket, uploadIDPath, opsID) + defer nsMutex.Unlock(minioMetaBucket, uploadIDPath, opsID) // Just check if the uploadID exists to avoid copy if it doesn't. if !fs.isUploadIDExists(bucket, object, uploadID) { - return "", InvalidUploadID{UploadID: uploadID} + return "", traceError(InvalidUploadID{UploadID: uploadID}) } fsMetaPath := pathJoin(uploadIDPath, fsMetaJSONFile) @@ -486,21 +500,21 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s err = fs.storage.RenameFile(minioMetaBucket, tmpPartPath, minioMetaBucket, partPath) if err != nil { if dErr := fs.storage.DeleteFile(minioMetaBucket, tmpPartPath); dErr != nil { - return "", toObjectErr(dErr, minioMetaBucket, tmpPartPath) + return "", toObjectErr(traceError(dErr), minioMetaBucket, tmpPartPath) } - return "", toObjectErr(err, minioMetaBucket, partPath) + return "", toObjectErr(traceError(err), minioMetaBucket, partPath) } - - if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil { - return "", toObjectErr(err, minioMetaBucket, fsMetaPath) + uploadIDPath = path.Join(mpartMetaPrefix, bucket, object, uploadID) + if err = writeFSMetadata(fs.storage, minioMetaBucket, path.Join(uploadIDPath, fsMetaJSONFile), fsMeta); err != nil { + return "", toObjectErr(err, minioMetaBucket, uploadIDPath) } - go appendParts(fs.storage, bucket, object, uploadID) + go appendParts(fs.storage, bucket, object, uploadID, opsID) return newMD5Hex, nil } // listObjectParts - wrapper scanning through -// '.minio/multipart/bucket/object/UPLOADID'. Lists all the parts -// saved inside '.minio/multipart/bucket/object/UPLOADID'. +// '.minio.sys/multipart/bucket/object/UPLOADID'. Lists all the parts +// saved inside '.minio.sys/multipart/bucket/object/UPLOADID'. func (fs fsObjects) listObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, error) { result := ListPartsInfo{} @@ -521,7 +535,7 @@ func (fs fsObjects) listObjectParts(bucket, object, uploadID string, partNumberM partNamePath := path.Join(mpartMetaPrefix, bucket, object, uploadID, part.Name) fi, err = fs.storage.StatFile(minioMetaBucket, partNamePath) if err != nil { - return ListPartsInfo{}, toObjectErr(err, minioMetaBucket, partNamePath) + return ListPartsInfo{}, toObjectErr(traceError(err), minioMetaBucket, partNamePath) } result.Parts = append(result.Parts, partInfo{ PartNumber: part.Number, @@ -559,21 +573,26 @@ func (fs fsObjects) listObjectParts(bucket, object, uploadID string, partNumberM func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return ListPartsInfo{}, BucketNameInvalid{Bucket: bucket} + return ListPartsInfo{}, traceError(BucketNameInvalid{Bucket: bucket}) } // Verify whether the bucket exists. if !fs.isBucketExist(bucket) { - return ListPartsInfo{}, BucketNotFound{Bucket: bucket} + return ListPartsInfo{}, traceError(BucketNotFound{Bucket: bucket}) } if !IsValidObjectName(object) { - return ListPartsInfo{}, ObjectNameInvalid{Bucket: bucket, Object: object} + return ListPartsInfo{}, traceError(ObjectNameInvalid{Bucket: bucket, Object: object}) } + + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + // Hold lock so that there is no competing abort-multipart-upload or complete-multipart-upload. - nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID)) - defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID)) + nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID), opsID) + defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID), opsID) if !fs.isUploadIDExists(bucket, object, uploadID) { - return ListPartsInfo{}, InvalidUploadID{UploadID: uploadID} + return ListPartsInfo{}, traceError(InvalidUploadID{UploadID: uploadID}) } return fs.listObjectParts(bucket, object, uploadID, partNumberMarker, maxParts) } @@ -587,55 +606,59 @@ func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberM func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (string, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return "", BucketNameInvalid{Bucket: bucket} + return "", traceError(BucketNameInvalid{Bucket: bucket}) } // Verify whether the bucket exists. if !fs.isBucketExist(bucket) { - return "", BucketNotFound{Bucket: bucket} + return "", traceError(BucketNotFound{Bucket: bucket}) } if !IsValidObjectName(object) { - return "", ObjectNameInvalid{ + return "", traceError(ObjectNameInvalid{ Bucket: bucket, Object: object, - } + }) } uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID) + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + // Hold lock so that // 1) no one aborts this multipart upload // 2) no one does a parallel complete-multipart-upload on this // multipart upload - nsMutex.Lock(minioMetaBucket, uploadIDPath) - defer nsMutex.Unlock(minioMetaBucket, uploadIDPath) + nsMutex.Lock(minioMetaBucket, uploadIDPath, opsID) + defer nsMutex.Unlock(minioMetaBucket, uploadIDPath, opsID) if !fs.isUploadIDExists(bucket, object, uploadID) { - return "", InvalidUploadID{UploadID: uploadID} + return "", traceError(InvalidUploadID{UploadID: uploadID}) } // fs-append.json path fsAppendMetaPath := getFSAppendMetaPath(uploadID) // Lock fs-append.json so that no parallel appendParts() is being done. - nsMutex.Lock(minioMetaBucket, fsAppendMetaPath) - defer nsMutex.Unlock(minioMetaBucket, fsAppendMetaPath) + nsMutex.Lock(minioMetaBucket, fsAppendMetaPath, opsID) + defer nsMutex.Unlock(minioMetaBucket, fsAppendMetaPath, opsID) // Calculate s3 compatible md5sum for complete multipart. s3MD5, err := completeMultipartMD5(parts...) if err != nil { - return "", err + return "", traceError(err) } // Read saved fs metadata for ongoing multipart. fsMetaPath := pathJoin(uploadIDPath, fsMetaJSONFile) fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, fsMetaPath) if err != nil { - return "", toObjectErr(err, minioMetaBucket, fsMetaPath) + return "", toObjectErr(traceError(err), minioMetaBucket, fsMetaPath) } fsAppendMeta, err := readFSMetadata(fs.storage, minioMetaBucket, fsAppendMetaPath) if err == nil && isPartsSame(fsAppendMeta.Parts, parts) { fsAppendDataPath := getFSAppendDataPath(uploadID) if err = fs.storage.RenameFile(minioMetaBucket, fsAppendDataPath, bucket, object); err != nil { - return "", toObjectErr(err, minioMetaBucket, fsAppendDataPath) + return "", toObjectErr(traceError(err), minioMetaBucket, fsAppendDataPath) } // Remove the append-file metadata file in tmp location as we no longer need it. fs.storage.DeleteFile(minioMetaBucket, fsAppendMetaPath) @@ -649,18 +672,18 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload for i, part := range parts { partIdx := fsMeta.ObjectPartIndex(part.PartNumber) if partIdx == -1 { - return "", InvalidPart{} + return "", traceError(InvalidPart{}) } if fsMeta.Parts[partIdx].ETag != part.ETag { - return "", BadDigest{} + return "", traceError(BadDigest{}) } // All parts except the last part has to be atleast 5MB. if (i < len(parts)-1) && !isMinAllowedPartSize(fsMeta.Parts[partIdx].Size) { - return "", PartTooSmall{ + return "", traceError(PartTooSmall{ PartNumber: part.PartNumber, PartSize: fsMeta.Parts[partIdx].Size, PartETag: part.ETag, - } + }) } // Construct part suffix. partSuffix := fmt.Sprintf("object%d", part.PartNumber) @@ -676,7 +699,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload n, err = fs.storage.ReadFile(minioMetaBucket, multipartPartFile, offset, buf[:curLeft]) if n > 0 { if err = fs.storage.AppendFile(minioMetaBucket, tempObj, buf[:n]); err != nil { - return "", toObjectErr(err, minioMetaBucket, tempObj) + return "", toObjectErr(traceError(err), minioMetaBucket, tempObj) } } if err != nil { @@ -684,9 +707,9 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload break } if err == errFileNotFound { - return "", InvalidPart{} + return "", traceError(InvalidPart{}) } - return "", toObjectErr(err, minioMetaBucket, multipartPartFile) + return "", toObjectErr(traceError(err), minioMetaBucket, multipartPartFile) } offset += n totalLeft -= n @@ -697,9 +720,9 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload err = fs.storage.RenameFile(minioMetaBucket, tempObj, bucket, object) if err != nil { if dErr := fs.storage.DeleteFile(minioMetaBucket, tempObj); dErr != nil { - return "", toObjectErr(dErr, minioMetaBucket, tempObj) + return "", toObjectErr(traceError(dErr), minioMetaBucket, tempObj) } - return "", toObjectErr(err, bucket, object) + return "", toObjectErr(traceError(err), bucket, object) } } @@ -713,7 +736,8 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload } fsMeta.Meta["md5Sum"] = s3MD5 - fsMetaPath = path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile) + fsMetaPath := path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile) + // Write the metadata to a temp file and rename it to the actual location. if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil { return "", toObjectErr(err, bucket, object) } @@ -721,19 +745,23 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload // Cleanup all the parts if everything else has been safely committed. if err = cleanupUploadedParts(bucket, object, uploadID, fs.storage); err != nil { - return "", toObjectErr(err, bucket, object) + return "", toObjectErr(traceError(err), bucket, object) } + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID = getOpsID() + // Hold the lock so that two parallel complete-multipart-uploads do not // leave a stale uploads.json behind. - nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object)) - defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object)) + nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object), opsID) + defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object), opsID) // Validate if there are other incomplete upload-id's present for // the object, if yes do not attempt to delete 'uploads.json'. uploadsJSON, err := readUploadsJSON(bucket, object, fs.storage) if err != nil { - return "", toObjectErr(err, minioMetaBucket, object) + return "", toObjectErr(traceError(err), minioMetaBucket, object) } // If we have successfully read `uploads.json`, then we proceed to // purge or update `uploads.json`. @@ -743,14 +771,14 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload } if len(uploadsJSON.Uploads) > 0 { if err = fs.updateUploadsJSON(bucket, object, uploadsJSON); err != nil { - return "", toObjectErr(err, minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object)) + return "", toObjectErr(traceError(err), minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object)) } // Return success. return s3MD5, nil } if err = fs.storage.DeleteFile(minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object, uploadsJSONFile)); err != nil { - return "", toObjectErr(err, minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object)) + return "", toObjectErr(traceError(err), minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object)) } // Return md5sum. @@ -759,7 +787,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload // abortMultipartUpload - wrapper for purging an ongoing multipart // transaction, deletes uploadID entry from `uploads.json` and purges -// the directory at '.minio/multipart/bucket/object/uploadID' holding +// the directory at '.minio.sys/multipart/bucket/object/uploadID' holding // all the upload parts. func (fs fsObjects) abortMultipartUpload(bucket, object, uploadID string) error { // Cleanup all uploaded parts. @@ -785,9 +813,9 @@ func (fs fsObjects) abortMultipartUpload(bucket, object, uploadID string) error return nil } } // No more pending uploads for the object, we purge the entire - // entry at '.minio/multipart/bucket/object'. + // entry at '.minio.sys/multipart/bucket/object'. if err = fs.storage.DeleteFile(minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object, uploadsJSONFile)); err != nil { - return toObjectErr(err, minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object)) + return toObjectErr(traceError(err), minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object)) } return nil } @@ -807,27 +835,31 @@ func (fs fsObjects) abortMultipartUpload(bucket, object, uploadID string) error func (fs fsObjects) AbortMultipartUpload(bucket, object, uploadID string) error { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return BucketNameInvalid{Bucket: bucket} + return traceError(BucketNameInvalid{Bucket: bucket}) } if !fs.isBucketExist(bucket) { - return BucketNotFound{Bucket: bucket} + return traceError(BucketNotFound{Bucket: bucket}) } if !IsValidObjectName(object) { - return ObjectNameInvalid{Bucket: bucket, Object: object} + return traceError(ObjectNameInvalid{Bucket: bucket, Object: object}) } + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + // Hold lock so that there is no competing complete-multipart-upload or put-object-part. - nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID)) - defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID)) + nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID), opsID) + defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID), opsID) if !fs.isUploadIDExists(bucket, object, uploadID) { - return InvalidUploadID{UploadID: uploadID} + return traceError(InvalidUploadID{UploadID: uploadID}) } fsAppendMetaPath := getFSAppendMetaPath(uploadID) // Lock fs-append.json so that no parallel appendParts() is being done. - nsMutex.Lock(minioMetaBucket, fsAppendMetaPath) - defer nsMutex.Unlock(minioMetaBucket, fsAppendMetaPath) + nsMutex.Lock(minioMetaBucket, fsAppendMetaPath, opsID) + defer nsMutex.Unlock(minioMetaBucket, fsAppendMetaPath, opsID) err := fs.abortMultipartUpload(bucket, object, uploadID) return err diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index 8927f7e89..b9ee3380e 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -26,7 +26,6 @@ import ( "sort" "strings" - "github.com/minio/minio/pkg/disk" "github.com/minio/minio/pkg/mimedb" ) @@ -68,7 +67,7 @@ func newFSObjects(disk string) (ObjectLayer, error) { return nil, err } - // Attempt to create `.minio`. + // Attempt to create `.minio.sys`. err = storage.MakeVol(minioMetaBucket) if err != nil { switch err { @@ -146,8 +145,8 @@ func (fs fsObjects) Shutdown() error { // StorageInfo - returns underlying storage statistics. func (fs fsObjects) StorageInfo() StorageInfo { - info, err := disk.GetInfo(fs.physicalDisk) - fatalIf(err, "Unable to get disk info "+fs.physicalDisk) + info, err := fs.storage.DiskInfo() + errorIf(err, "Unable to get disk info %#v", fs.storage) return StorageInfo{ Total: info.Total, Free: info.Free, @@ -160,10 +159,10 @@ func (fs fsObjects) StorageInfo() StorageInfo { func (fs fsObjects) MakeBucket(bucket string) error { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return BucketNameInvalid{Bucket: bucket} + return traceError(BucketNameInvalid{Bucket: bucket}) } if err := fs.storage.MakeVol(bucket); err != nil { - return toObjectErr(err, bucket) + return toObjectErr(traceError(err), bucket) } return nil } @@ -172,11 +171,11 @@ func (fs fsObjects) MakeBucket(bucket string) error { func (fs fsObjects) GetBucketInfo(bucket string) (BucketInfo, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return BucketInfo{}, BucketNameInvalid{Bucket: bucket} + return BucketInfo{}, traceError(BucketNameInvalid{Bucket: bucket}) } vi, err := fs.storage.StatVol(bucket) if err != nil { - return BucketInfo{}, toObjectErr(err, bucket) + return BucketInfo{}, toObjectErr(traceError(err), bucket) } return BucketInfo{ Name: bucket, @@ -189,7 +188,7 @@ func (fs fsObjects) ListBuckets() ([]BucketInfo, error) { var bucketInfos []BucketInfo vols, err := fs.storage.ListVols() if err != nil { - return nil, toObjectErr(err) + return nil, toObjectErr(traceError(err)) } for _, vol := range vols { // StorageAPI can send volume names which are incompatible @@ -214,11 +213,11 @@ func (fs fsObjects) ListBuckets() ([]BucketInfo, error) { func (fs fsObjects) DeleteBucket(bucket string) error { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return BucketNameInvalid{Bucket: bucket} + return traceError(BucketNameInvalid{Bucket: bucket}) } // Attempt to delete regular bucket. if err := fs.storage.DeleteVol(bucket); err != nil { - return toObjectErr(err, bucket) + return toObjectErr(traceError(err), bucket) } // Cleanup all the previously incomplete multiparts. if err := cleanupDir(fs.storage, path.Join(minioMetaBucket, mpartMetaPrefix), bucket); err != nil && err != errVolumeNotFound { @@ -233,34 +232,34 @@ func (fs fsObjects) DeleteBucket(bucket string) error { func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64, writer io.Writer) (err error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return BucketNameInvalid{Bucket: bucket} + return traceError(BucketNameInvalid{Bucket: bucket}) } // Verify if object is valid. if !IsValidObjectName(object) { - return ObjectNameInvalid{Bucket: bucket, Object: object} + return traceError(ObjectNameInvalid{Bucket: bucket, Object: object}) } // Offset and length cannot be negative. if offset < 0 || length < 0 { - return toObjectErr(errUnexpected, bucket, object) + return toObjectErr(traceError(errUnexpected), bucket, object) } // Writer cannot be nil. if writer == nil { - return toObjectErr(errUnexpected, bucket, object) + return toObjectErr(traceError(errUnexpected), bucket, object) } // Stat the file to get file size. fi, err := fs.storage.StatFile(bucket, object) if err != nil { - return toObjectErr(err, bucket, object) + return toObjectErr(traceError(err), bucket, object) } // Reply back invalid range if the input offset and length fall out of range. if offset > fi.Size || length > fi.Size { - return InvalidRange{offset, length, fi.Size} + return traceError(InvalidRange{offset, length, fi.Size}) } // Reply if we have inputs with offset and length falling out of file size range. if offset+length > fi.Size { - return InvalidRange{offset, length, fi.Size} + return traceError(InvalidRange{offset, length, fi.Size}) } var totalLeft = length @@ -289,11 +288,11 @@ func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64, offset += int64(nw) } if ew != nil { - err = ew + err = traceError(ew) break } if nr != int64(nw) { - err = io.ErrShortWrite + err = traceError(io.ErrShortWrite) break } } @@ -301,7 +300,7 @@ func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64, break } if er != nil { - err = er + err = traceError(er) break } if totalLeft == 0 { @@ -312,22 +311,15 @@ func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64, return toObjectErr(err, bucket, object) } -// GetObjectInfo - get object info. -func (fs fsObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) { - // Verify if bucket is valid. - if !IsValidBucketName(bucket) { - return ObjectInfo{}, (BucketNameInvalid{Bucket: bucket}) - } - // Verify if object is valid. - if !IsValidObjectName(object) { - return ObjectInfo{}, (ObjectNameInvalid{Bucket: bucket, Object: object}) - } +// getObjectInfo - get object info. +func (fs fsObjects) getObjectInfo(bucket, object string) (ObjectInfo, error) { fi, err := fs.storage.StatFile(bucket, object) if err != nil { - return ObjectInfo{}, toObjectErr(err, bucket, object) + return ObjectInfo{}, toObjectErr(traceError(err), bucket, object) } fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile)) - if err != nil && err != errFileNotFound { + // Ignore error if the metadata file is not found, other errors must be returned. + if err != nil && errorCause(err) != errFileNotFound { return ObjectInfo{}, toObjectErr(err, bucket, object) } @@ -358,17 +350,30 @@ func (fs fsObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) { }, nil } -// PutObject - create an object. -func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (string, error) { +// GetObjectInfo - get object info. +func (fs fsObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return "", BucketNameInvalid{Bucket: bucket} + return ObjectInfo{}, traceError(BucketNameInvalid{Bucket: bucket}) + } + // Verify if object is valid. + if !IsValidObjectName(object) { + return ObjectInfo{}, traceError(ObjectNameInvalid{Bucket: bucket, Object: object}) + } + return fs.getObjectInfo(bucket, object) +} + +// PutObject - create an object. +func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return ObjectInfo{}, traceError(BucketNameInvalid{Bucket: bucket}) } if !IsValidObjectName(object) { - return "", ObjectNameInvalid{ + return ObjectInfo{}, traceError(ObjectNameInvalid{ Bucket: bucket, Object: object, - } + }) } // No metadata is set, allocate a new one. if metadata == nil { @@ -397,9 +402,9 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io. if size == 0 { // For size 0 we write a 0byte file. - err := fs.storage.AppendFile(minioMetaBucket, tempObj, []byte("")) + err = fs.storage.AppendFile(minioMetaBucket, tempObj, []byte("")) if err != nil { - return "", toObjectErr(err, bucket, object) + return ObjectInfo{}, toObjectErr(traceError(err), bucket, object) } } else { // Allocate a buffer to Read() from request body @@ -409,17 +414,18 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io. } buf := make([]byte, int(bufSize)) teeReader := io.TeeReader(limitDataReader, md5Writer) - bytesWritten, err := fsCreateFile(fs.storage, teeReader, buf, minioMetaBucket, tempObj) + var bytesWritten int64 + bytesWritten, err = fsCreateFile(fs.storage, teeReader, buf, minioMetaBucket, tempObj) if err != nil { fs.storage.DeleteFile(minioMetaBucket, tempObj) - return "", toObjectErr(err, bucket, object) + return ObjectInfo{}, toObjectErr(traceError(err), bucket, object) } // Should return IncompleteBody{} error when reader has fewer // bytes than specified in request header. if bytesWritten < size { fs.storage.DeleteFile(minioMetaBucket, tempObj) - return "", IncompleteBody{} + return ObjectInfo{}, traceError(IncompleteBody{}) } } @@ -435,7 +441,7 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io. // Incoming payload wrong, delete the temporary object. fs.storage.DeleteFile(minioMetaBucket, tempObj) // Error return. - return "", toObjectErr(vErr, bucket, object) + return ObjectInfo{}, toObjectErr(traceError(vErr), bucket, object) } } @@ -446,14 +452,14 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io. // MD5 mismatch, delete the temporary object. fs.storage.DeleteFile(minioMetaBucket, tempObj) // Returns md5 mismatch. - return "", BadDigest{md5Hex, newMD5Hex} + return ObjectInfo{}, traceError(BadDigest{md5Hex, newMD5Hex}) } } // Entire object was written to the temp location, now it's safe to rename it to the actual location. - err := fs.storage.RenameFile(minioMetaBucket, tempObj, bucket, object) + err = fs.storage.RenameFile(minioMetaBucket, tempObj, bucket, object) if err != nil { - return "", toObjectErr(err, bucket, object) + return ObjectInfo{}, toObjectErr(traceError(err), bucket, object) } // Save additional metadata only if extended headers such as "X-Amz-Meta-" are set. @@ -464,12 +470,15 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io. fsMetaPath := path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile) if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil { - return "", toObjectErr(err, bucket, object) + return ObjectInfo{}, toObjectErr(traceError(err), bucket, object) } } - - // Return md5sum, successfully wrote object. - return newMD5Hex, nil + objInfo, err = fs.getObjectInfo(bucket, object) + if err == nil { + // If MINIO_ENABLE_FSMETA is not enabled objInfo.MD5Sum will be empty. + objInfo.MD5Sum = newMD5Hex + } + return objInfo, err } // DeleteObject - deletes an object from a bucket, this operation is destructive @@ -477,17 +486,17 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io. func (fs fsObjects) DeleteObject(bucket, object string) error { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return BucketNameInvalid{Bucket: bucket} + return traceError(BucketNameInvalid{Bucket: bucket}) } if !IsValidObjectName(object) { - return ObjectNameInvalid{Bucket: bucket, Object: object} + return traceError(ObjectNameInvalid{Bucket: bucket, Object: object}) } err := fs.storage.DeleteFile(minioMetaBucket, path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile)) if err != nil && err != errFileNotFound { - return toObjectErr(err, bucket, object) + return toObjectErr(traceError(err), bucket, object) } if err = fs.storage.DeleteFile(bucket, object); err != nil { - return toObjectErr(err, bucket, object) + return toObjectErr(traceError(err), bucket, object) } return nil } @@ -518,11 +527,11 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey return } if fileInfo, err = fs.storage.StatFile(bucket, entry); err != nil { - return + return FileInfo{}, traceError(err) } fsMeta, mErr := readFSMetadata(fs.storage, minioMetaBucket, path.Join(bucketMetaPrefix, bucket, entry, fsMetaJSONFile)) - if mErr != nil && mErr != errFileNotFound { - return FileInfo{}, mErr + if mErr != nil && errorCause(mErr) != errFileNotFound { + return FileInfo{}, traceError(mErr) } if len(fsMeta.Meta) == 0 { fsMeta.Meta = make(map[string]string) @@ -535,28 +544,28 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return ListObjectsInfo{}, BucketNameInvalid{Bucket: bucket} + return ListObjectsInfo{}, traceError(BucketNameInvalid{Bucket: bucket}) } // Verify if bucket exists. if !isBucketExist(fs.storage, bucket) { - return ListObjectsInfo{}, BucketNotFound{Bucket: bucket} + return ListObjectsInfo{}, traceError(BucketNotFound{Bucket: bucket}) } if !IsValidObjectPrefix(prefix) { - return ListObjectsInfo{}, ObjectNameInvalid{Bucket: bucket, Object: prefix} + return ListObjectsInfo{}, traceError(ObjectNameInvalid{Bucket: bucket, Object: prefix}) } // Verify if delimiter is anything other than '/', which we do not support. if delimiter != "" && delimiter != slashSeparator { - return ListObjectsInfo{}, UnsupportedDelimiter{ + return ListObjectsInfo{}, traceError(UnsupportedDelimiter{ Delimiter: delimiter, - } + }) } // Verify if marker has prefix. if marker != "" { if !strings.HasPrefix(marker, prefix) { - return ListObjectsInfo{}, InvalidMarkerPrefixCombination{ + return ListObjectsInfo{}, traceError(InvalidMarkerPrefixCombination{ Marker: marker, Prefix: prefix, - } + }) } } @@ -611,7 +620,7 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey // For any walk error return right away. if walkResult.err != nil { // File not found is a valid case. - if walkResult.err == errFileNotFound { + if errorCause(walkResult.err) == errFileNotFound { return ListObjectsInfo{}, nil } return ListObjectsInfo{}, toObjectErr(walkResult.err, bucket, prefix) @@ -653,10 +662,15 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey // HealObject - no-op for fs. Valid only for XL. func (fs fsObjects) HealObject(bucket, object string) error { - return NotImplemented{} + return traceError(NotImplemented{}) } // HealListObjects - list objects for healing. Valid only for XL func (fs fsObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) { - return ListObjectsInfo{}, NotImplemented{} + return ListObjectsInfo{}, traceError(NotImplemented{}) +} + +// HealDiskMetadata -- heal disk metadata, not supported in FS +func (fs fsObjects) HealDiskMetadata() error { + return NotImplemented{} } diff --git a/cmd/fs-v1_test.go b/cmd/fs-v1_test.go index 275cdb008..37ebd0e6b 100644 --- a/cmd/fs-v1_test.go +++ b/cmd/fs-v1_test.go @@ -41,7 +41,11 @@ func TestNewFS(t *testing.T) { } // Initializes all disks with XL - _, err := newXLObjects(disks, nil) + err := formatDisks(disks, nil) + if err != nil { + t.Fatalf("Unable to format XL %s", err) + } + _, err = newXLObjects(disks, nil) if err != nil { t.Fatalf("Unable to initialize XL object, %s", err) } @@ -89,7 +93,7 @@ func TestFSShutdown(t *testing.T) { for i := 1; i <= 5; i++ { naughty := newNaughtyDisk(fsStorage, map[int]error{i: errFaultyDisk}, nil) fs.storage = naughty - if err := fs.Shutdown(); err != errFaultyDisk { + if err := fs.Shutdown(); errorCause(err) != errFaultyDisk { t.Fatal(i, ", Got unexpected fs shutdown error: ", err) } } diff --git a/cmd/globals.go b/cmd/globals.go index 154b519c9..d24354184 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -21,6 +21,7 @@ import ( "github.com/fatih/color" "github.com/minio/minio/pkg/objcache" + "os" ) // Global constants for Minio. @@ -42,6 +43,10 @@ const ( var ( globalQuiet = false // Quiet flag set via command line globalTrace = false // Trace flag set via environment setting. + + globalDebug = false // Debug flag set to print debug info. + globalDebugLock = false // Lock debug info set via environment variable MINIO_DEBUG=lock . + globalDebugMemory = false // Memory debug info set via environment variable MINIO_DEBUG=mem // Add new global flags here. // Maximum connections handled per @@ -70,3 +75,15 @@ var ( colorBlue = color.New(color.FgBlue).SprintfFunc() colorBold = color.New(color.Bold).SprintFunc() ) + +// fetch from environment variables and set the global values related to locks. +func setGlobalsDebugFromEnv() { + debugEnv := os.Getenv("MINIO_DEBUG") + switch debugEnv { + case "lock": + globalDebugLock = true + case "mem": + globalDebugMemory = true + } + globalDebug = globalDebugLock || globalDebugMemory +} diff --git a/cmd/lock-instrument.go b/cmd/lock-instrument.go new file mode 100644 index 000000000..5b0bd139e --- /dev/null +++ b/cmd/lock-instrument.go @@ -0,0 +1,283 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "fmt" + "time" +) + +const ( + debugRLockStr = "RLock" + debugWLockStr = "WLock" +) + +// struct containing information of status (ready/running/blocked) of an operation with given operation ID. +type debugLockInfo struct { + lockType string // "Rlock" or "WLock". + lockOrigin string // contains the trace of the function which invoked the lock, obtained from runtime. + status string // status can be running/ready/blocked. + since time.Time // time info of the since how long the status holds true. +} + +// debugLockInfo - container for storing locking information for unique copy (volume,path) pair. +// ref variable holds the reference count for locks held for. +// `ref` values helps us understand the n locks held for given pair. +// `running` value helps us understand the total successful locks held (not blocked) for given pair and the operation is under execution. +// `blocked` value helps us understand the total number of operations blocked waiting on locks for given pair. +type debugLockInfoPerVolumePath struct { + ref int64 // running + blocked operations. + running int64 // count of successful lock acquire and running operations. + blocked int64 // count of number of operations blocked waiting on lock. + lockInfo (map[string]debugLockInfo) // map of [operationID] debugLockInfo{operation, status, since} . +} + +// returns an instance of debugLockInfo. +// need to create this for every unique pair of {volume,path}. +// total locks, number of calls blocked on locks, and number of successful locks held but not unlocked yet. +func newDebugLockInfoPerVolumePath() *debugLockInfoPerVolumePath { + return &debugLockInfoPerVolumePath{ + lockInfo: make(map[string]debugLockInfo), + ref: 0, + blocked: 0, + running: 0, + } +} + +// LockInfoNil - Returned if the lock info map is not initialized. +type LockInfoNil struct { +} + +func (l LockInfoNil) Error() string { + return fmt.Sprintf("Debug Lock Map not initialized:\n1. Enable Lock Debugging using right ENV settings \n2. Make sure initNSLock() is called.") +} + +// LockInfoOriginNotFound - While changing the state of the lock info its important that the entry for +// lock at a given origin exists, if not `LockInfoOriginNotFound` is returned. +type LockInfoOriginNotFound struct { + volume string + path string + operationID string + lockOrigin string +} + +func (l LockInfoOriginNotFound) Error() string { + return fmt.Sprintf("No lock state stored for the lock origined at \"%s\", for %s, %s, %s.", + l.lockOrigin, l.volume, l.path, l.operationID) +} + +// LockInfoVolPathMssing - Error interface. Returned when the info the +type LockInfoVolPathMssing struct { + volume string + path string +} + +func (l LockInfoVolPathMssing) Error() string { + return fmt.Sprintf("No entry in debug Lock Map for Volume: %s, path: %s.", l.volume, l.path) +} + +// LockInfoOpsIDNotFound - Returned when the lock state info exists, but the entry for +// given operation ID doesn't exist. +type LockInfoOpsIDNotFound struct { + volume string + path string + operationID string +} + +func (l LockInfoOpsIDNotFound) Error() string { + return fmt.Sprintf("No entry in lock info for %s, %s, %s.", l.operationID, l.volume, l.path) +} + +// LockInfoStateNotBlocked - When an attempt to change the state of the lock form `blocked` to `running` is done, +// its necessary that the state before the transsition is "blocked", otherwise LockInfoStateNotBlocked returned. +type LockInfoStateNotBlocked struct { + volume string + path string + operationID string +} + +func (l LockInfoStateNotBlocked) Error() string { + return fmt.Sprintf("Lock state should be \"Blocked\" for %s, %s, %s.", l.volume, l.path, l.operationID) +} + +// change the state of the lock from Blocked to Running. +func (n *nsLockMap) statusBlockedToRunning(param nsParam, lockOrigin, operationID string, readLock bool) error { + // This operation is not executed under the scope nsLockMap.mutex.Lock(), lock has to be explicitly held here. + n.lockMapMutex.Lock() + defer n.lockMapMutex.Unlock() + if n.debugLockMap == nil { + return LockInfoNil{} + } + // new state info to be set for the lock. + newLockInfo := debugLockInfo{ + lockOrigin: lockOrigin, + status: "Running", + since: time.Now().UTC(), + } + + // set lock type. + if readLock { + newLockInfo.lockType = debugRLockStr + } else { + newLockInfo.lockType = debugWLockStr + } + + // check whether the lock info entry for pair already exists and its not `nil`. + if debugLockMap, ok := n.debugLockMap[param]; ok { + // ``*debugLockInfoPerVolumePath` entry containing lock info for `param ` is `nil`. + if debugLockMap == nil { + return LockInfoNil{} + } + } else { + // The lock state info foe given pair should already exist. + // If not return `LockInfoVolPathMssing`. + return LockInfoVolPathMssing{param.volume, param.path} + } + + // Lock info the for the given operation ID shouldn't be `nil`. + if n.debugLockMap[param].lockInfo == nil { + return LockInfoOpsIDNotFound{param.volume, param.path, operationID} + } + + if lockInfo, ok := n.debugLockMap[param].lockInfo[operationID]; ok { + // The entry for the lock origined at `lockOrigin` should already exist. + // If not return `LockInfoOriginNotFound`. + if lockInfo.lockOrigin != lockOrigin { + return LockInfoOriginNotFound{param.volume, param.path, operationID, lockOrigin} + } + // Status of the lock should already be set to "Blocked". + // If not return `LockInfoStateNotBlocked`. + if lockInfo.status != "Blocked" { + return LockInfoStateNotBlocked{param.volume, param.path, operationID} + } + } else { + // The lock info entry for given `opsID` should already exist for given pair. + // If not return `LockInfoOpsIDNotFound`. + return LockInfoOpsIDNotFound{param.volume, param.path, operationID} + } + + // All checks finished. + // changing the status of the operation from blocked to running and updating the time. + n.debugLockMap[param].lockInfo[operationID] = newLockInfo + + // After locking unblocks decrease the blocked counter. + n.blockedCounter-- + // Increase the running counter. + n.runningLockCounter++ + n.debugLockMap[param].blocked-- + n.debugLockMap[param].running++ + return nil +} + +// change the state of the lock from Ready to Blocked. +func (n *nsLockMap) statusNoneToBlocked(param nsParam, lockOrigin, operationID string, readLock bool) error { + if n.debugLockMap == nil { + return LockInfoNil{} + } + + newLockInfo := debugLockInfo{ + lockOrigin: lockOrigin, + status: "Blocked", + since: time.Now().UTC(), + } + if readLock { + newLockInfo.lockType = debugRLockStr + } else { + newLockInfo.lockType = debugWLockStr + } + + if lockInfo, ok := n.debugLockMap[param]; ok { + if lockInfo == nil { + // *debugLockInfoPerVolumePath entry is nil, initialize here to avoid any case of `nil` pointer access. + n.initLockInfoForVolumePath(param) + } + } else { + // State info entry for the given doesn't exist, initializing it. + n.initLockInfoForVolumePath(param) + } + + // lockInfo is a map[string]debugLockInfo, which holds map[OperationID]{status,time, origin} of the lock. + if n.debugLockMap[param].lockInfo == nil { + n.debugLockMap[param].lockInfo = make(map[string]debugLockInfo) + } + // The status of the operation with the given operation ID is marked blocked till its gets unblocked from the lock. + n.debugLockMap[param].lockInfo[operationID] = newLockInfo + // Increment the Global lock counter. + n.globalLockCounter++ + // Increment the counter for number of blocked opertions, decrement it after the locking unblocks. + n.blockedCounter++ + // increment the reference of the lock for the given pair. + n.debugLockMap[param].ref++ + // increment the blocked counter for the given pair. + n.debugLockMap[param].blocked++ + return nil +} + +// deleteLockInfoEntry - Deletes the lock state information for given pair. Called when nsLk.ref count is 0. +func (n *nsLockMap) deleteLockInfoEntryForVolumePath(param nsParam) error { + if n.debugLockMap == nil { + return LockInfoNil{} + } + // delete the lock info for the given operation. + if _, found := n.debugLockMap[param]; found { + // Remove from the map if there are no more references for the given (volume,path) pair. + delete(n.debugLockMap, param) + } else { + return LockInfoVolPathMssing{param.volume, param.path} + } + return nil +} + +// deleteLockInfoEntry - Deletes the entry for given opsID in the lock state information of given pair. +// called when the nsLk ref count for the given pair is not 0. +func (n *nsLockMap) deleteLockInfoEntryForOps(param nsParam, operationID string) error { + if n.debugLockMap == nil { + return LockInfoNil{} + } + // delete the lock info for the given operation. + if infoMap, found := n.debugLockMap[param]; found { + // the opertion finished holding the lock on the resource, remove the entry for the given operation with the operation ID. + if _, foundInfo := infoMap.lockInfo[operationID]; foundInfo { + // decrease the global running and lock reference counter. + n.runningLockCounter-- + n.globalLockCounter-- + // decrease the lock referee counter for the lock info for given pair. + // decrease the running operation number. Its assumed that the operation is over once an attempt to release the lock is made. + infoMap.running-- + // decrease the total reference count of locks jeld on pair. + infoMap.ref-- + delete(infoMap.lockInfo, operationID) + } else { + // Unlock request with invalid opertion ID not accepted. + return LockInfoOpsIDNotFound{param.volume, param.path, operationID} + } + } else { + return LockInfoVolPathMssing{param.volume, param.path} + } + return nil +} + +// return randomly generated string ID if lock debug is enabled, +// else returns empty string +func getOpsID() (opsID string) { + // check if lock debug is enabled. + if globalDebugLock { + // generated random ID. + opsID = string(generateRequestID()) + } + return opsID +} diff --git a/cmd/lock-instrument_test.go b/cmd/lock-instrument_test.go new file mode 100644 index 000000000..966d48e2e --- /dev/null +++ b/cmd/lock-instrument_test.go @@ -0,0 +1,744 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "testing" + "time" +) + +type lockStateCase struct { + volume string + path string + lockOrigin string + opsID string + readLock bool // lock type. + setBlocked bool // initialize the initial state to blocked. + expectedErr error + // expected global lock stats. + expectedLockStatus string // Status of the lock Blocked/Running. + + expectedGlobalLockCount int // Total number of locks held across the system, includes blocked + held locks. + expectedBlockedLockCount int // Total blocked lock across the system. + expectedRunningLockCount int // Total successfully held locks (non-blocking). + // expected lock statu for given pair. + expectedVolPathLockCount int // Total locks held for given pair, includes blocked locks. + expectedVolPathRunningCount int // Total succcesfully held locks for given pair. + expectedVolPathBlockCount int // Total locks blocked on the given pair. +} + +// Used for validating the Lock info obtaining from contol RPC end point for obtaining lock related info. +func verifyRPCLockInfoResponse(l lockStateCase, rpcLockInfoResponse SystemLockState, t TestErrHandler, testNum int) { + // Assert the total number of locks (locked + acquired) in the system. + if rpcLockInfoResponse.TotalLocks != int64(l.expectedGlobalLockCount) { + t.Fatalf("Test %d: Expected the global lock counter to be %v, but got %v", testNum, int64(l.expectedGlobalLockCount), + rpcLockInfoResponse.TotalLocks) + } + + // verify the count for total blocked locks. + if rpcLockInfoResponse.TotalBlockedLocks != int64(l.expectedBlockedLockCount) { + t.Fatalf("Test %d: Expected the total blocked lock counter to be %v, but got %v", testNum, int64(l.expectedBlockedLockCount), + rpcLockInfoResponse.TotalBlockedLocks) + } + + // verify the count for total running locks. + if rpcLockInfoResponse.TotalAcquiredLocks != int64(l.expectedRunningLockCount) { + t.Fatalf("Test %d: Expected the total running lock counter to be %v, but got %v", testNum, int64(l.expectedRunningLockCount), + rpcLockInfoResponse.TotalAcquiredLocks) + } + + for _, locksInfoPerObject := range rpcLockInfoResponse.LocksInfoPerObject { + // See whether the entry for the exists in the RPC response. + if locksInfoPerObject.Bucket == l.volume && locksInfoPerObject.Object == l.path { + // Assert the total number of locks (blocked + acquired) for the given pair. + if locksInfoPerObject.LocksOnObject != int64(l.expectedVolPathLockCount) { + t.Errorf("Test %d: Expected the total lock count for bucket: \"%s\", object: \"%s\" to be %v, but got %v", testNum, + l.volume, l.path, int64(l.expectedVolPathLockCount), locksInfoPerObject.LocksOnObject) + } + // Assert the total number of acquired locks for the given pair. + if locksInfoPerObject.LocksAcquiredOnObject != int64(l.expectedVolPathRunningCount) { + t.Errorf("Test %d: Expected the acquired lock count for bucket: \"%s\", object: \"%s\" to be %v, but got %v", testNum, + l.volume, l.path, int64(l.expectedVolPathRunningCount), locksInfoPerObject.LocksAcquiredOnObject) + } + // Assert the total number of blocked locks for the given pair. + if locksInfoPerObject.TotalBlockedLocks != int64(l.expectedVolPathBlockCount) { + t.Errorf("Test %d: Expected the blocked lock count for bucket: \"%s\", object: \"%s\" to be %v, but got %v", testNum, + l.volume, l.path, int64(l.expectedVolPathBlockCount), locksInfoPerObject.TotalBlockedLocks) + } + // Flag to mark whether there's an entry in the RPC lock info response for given opsID. + var opsIDfound bool + for _, opsLockState := range locksInfoPerObject.LockDetailsOnObject { + // first check whether the entry for the given operation ID exists. + if opsLockState.OperationID == l.opsID { + opsIDfound = true + // asserting the type of lock (RLock/WLock) from the RPC lock info response. + if l.readLock { + if opsLockState.LockType != debugRLockStr { + t.Errorf("Test case %d: Expected the lock type to be \"%s\"", testNum, debugRLockStr) + } + } else { + if opsLockState.LockType != debugWLockStr { + t.Errorf("Test case %d: Expected the lock type to be \"%s\"", testNum, debugWLockStr) + } + } + + if opsLockState.Status != l.expectedLockStatus { + t.Errorf("Test case %d: Expected the status of the operation to be \"%s\", got \"%s\"", testNum, l.expectedLockStatus, opsLockState.Status) + } + + // if opsLockState.LockOrigin != l.lockOrigin { + // t.Fatalf("Test case %d: Expected the origin of the lock to be \"%s\", got \"%s\"", testNum, opsLockState.LockOrigin, l.lockOrigin) + // } + // all check satisfied, return here. + // Any mismatch in the earlier checks would have ended the tests due to `Fatalf`, + // control reaching here implies that all checks are satisfied. + return + } + } + // opsID not found. + // No entry for an operation with given operation ID exists. + if !opsIDfound { + t.Fatalf("Test case %d: Entry for OpsId: \"%s\" not found in : \"%s\", : \"%s\" doesn't exist in the RPC response", testNum, l.opsID, l.volume, l.path) + } + } + } + // No entry exists for given pair in the RPC response. + t.Errorf("Test case %d: Entry for : \"%s\", : \"%s\" doesn't exist in the RPC response", testNum, l.volume, l.path) +} + +// Asserts the lock counter from the global nsMutex inmemory lock with the expected one. +func verifyGlobalLockStats(l lockStateCase, t *testing.T, testNum int) { + nsMutex.lockMapMutex.Lock() + + // Verifying the lock stats. + if nsMutex.globalLockCounter != int64(l.expectedGlobalLockCount) { + t.Errorf("Test %d: Expected the global lock counter to be %v, but got %v", testNum, int64(l.expectedGlobalLockCount), + nsMutex.globalLockCounter) + } + // verify the count for total blocked locks. + if nsMutex.blockedCounter != int64(l.expectedBlockedLockCount) { + t.Errorf("Test %d: Expected the total blocked lock counter to be %v, but got %v", testNum, int64(l.expectedBlockedLockCount), + nsMutex.blockedCounter) + } + // verify the count for total running locks. + if nsMutex.runningLockCounter != int64(l.expectedRunningLockCount) { + t.Errorf("Test %d: Expected the total running lock counter to be %v, but got %v", testNum, int64(l.expectedRunningLockCount), + nsMutex.runningLockCounter) + } + nsMutex.lockMapMutex.Unlock() + // Verifying again with the JSON response of the lock info. + // Verifying the lock stats. + sysLockState, err := generateSystemLockResponse() + if err != nil { + t.Fatalf("Obtaining lock info failed with %s", err) + + } + if sysLockState.TotalLocks != int64(l.expectedGlobalLockCount) { + t.Errorf("Test %d: Expected the global lock counter to be %v, but got %v", testNum, int64(l.expectedGlobalLockCount), + sysLockState.TotalLocks) + } + // verify the count for total blocked locks. + if sysLockState.TotalBlockedLocks != int64(l.expectedBlockedLockCount) { + t.Errorf("Test %d: Expected the total blocked lock counter to be %v, but got %v", testNum, int64(l.expectedBlockedLockCount), + sysLockState.TotalBlockedLocks) + } + // verify the count for total running locks. + if sysLockState.TotalAcquiredLocks != int64(l.expectedRunningLockCount) { + t.Errorf("Test %d: Expected the total running lock counter to be %v, but got %v", testNum, int64(l.expectedRunningLockCount), + sysLockState.TotalAcquiredLocks) + } +} + +// Verify the lock counter for entries of given pair. +func verifyLockStats(l lockStateCase, t *testing.T, testNum int) { + nsMutex.lockMapMutex.Lock() + defer nsMutex.lockMapMutex.Unlock() + param := nsParam{l.volume, l.path} + + // Verify the total locks (blocked+running) for given pair. + if nsMutex.debugLockMap[param].ref != int64(l.expectedVolPathLockCount) { + t.Errorf("Test %d: Expected the total lock count for volume: \"%s\", path: \"%s\" to be %v, but got %v", testNum, + param.volume, param.path, int64(l.expectedVolPathLockCount), nsMutex.debugLockMap[param].ref) + } + // Verify the total running locks for given pair. + if nsMutex.debugLockMap[param].running != int64(l.expectedVolPathRunningCount) { + t.Errorf("Test %d: Expected the total running locks for volume: \"%s\", path: \"%s\" to be %v, but got %v", testNum, param.volume, param.path, + int64(l.expectedVolPathRunningCount), nsMutex.debugLockMap[param].running) + } + // Verify the total blocked locks for givne pair. + if nsMutex.debugLockMap[param].blocked != int64(l.expectedVolPathBlockCount) { + t.Errorf("Test %d: Expected the total blocked locks for volume: \"%s\", path: \"%s\" to be %v, but got %v", testNum, param.volume, param.path, + int64(l.expectedVolPathBlockCount), nsMutex.debugLockMap[param].blocked) + } +} + +// verifyLockState - function which asserts the expected lock info in the system with the actual values in the nsMutex. +func verifyLockState(l lockStateCase, t *testing.T, testNum int) { + param := nsParam{l.volume, l.path} + + verifyGlobalLockStats(l, t, testNum) + nsMutex.lockMapMutex.Lock() + // Verifying the lock statuS fields. + if debugLockMap, ok := nsMutex.debugLockMap[param]; ok { + if lockInfo, ok := debugLockMap.lockInfo[l.opsID]; ok { + // Validating the lock type filed in the debug lock information. + if l.readLock { + if lockInfo.lockType != debugRLockStr { + t.Errorf("Test case %d: Expected the lock type in the lock debug info to be \"%s\"", testNum, debugRLockStr) + } + } else { + if lockInfo.lockType != debugWLockStr { + t.Errorf("Test case %d: Expected the lock type in the lock debug info to be \"%s\"", testNum, debugWLockStr) + } + } + + // // validating the lock origin. + // if l.lockOrigin != lockInfo.lockOrigin { + // t.Fatalf("Test %d: Expected the lock origin info to be \"%s\", but got \"%s\"", testNum, l.lockOrigin, lockInfo.lockOrigin) + // } + // validating the status of the lock. + if lockInfo.status != l.expectedLockStatus { + t.Errorf("Test %d: Expected the status of the lock to be \"%s\", but got \"%s\"", testNum, l.expectedLockStatus, lockInfo.status) + } + } else { + // Stop the tests if lock debug entry for given pair is not found. + t.Errorf("Test case %d: Expected an debug lock entry for opsID \"%s\"", testNum, l.opsID) + } + } else { + // To change the status the entry for given should exist in the lock info struct. + t.Errorf("Test case %d: Debug lock entry for volume: %s, path: %s doesn't exist", testNum, param.volume, param.path) + } + // verifyLockStats holds its own lock. + nsMutex.lockMapMutex.Unlock() + + // verify the lock count. + verifyLockStats(l, t, testNum) +} + +// TestNewDebugLockInfoPerVolumePath - Validates the values initialized by newDebugLockInfoPerVolumePath(). +func TestNewDebugLockInfoPerVolumePath(t *testing.T) { + lockInfo := newDebugLockInfoPerVolumePath() + + if lockInfo.ref != 0 { + t.Errorf("Expected initial reference value of total locks to be 0, got %d", lockInfo.ref) + } + if lockInfo.blocked != 0 { + t.Errorf("Expected initial reference of blocked locks to be 0, got %d", lockInfo.blocked) + } + if lockInfo.running != 0 { + t.Errorf("Expected initial reference value of held locks to be 0, got %d", lockInfo.running) + } +} + +// TestNsLockMapStatusBlockedToRunning - Validates the function for changing the lock state from blocked to running. +func TestNsLockMapStatusBlockedToRunning(t *testing.T) { + + testCases := []struct { + volume string + path string + lockOrigin string + opsID string + readLock bool // lock type. + setBlocked bool // initialize the initial state to blocked. + expectedErr error + }{ + // Test case - 1. + { + + volume: "my-bucket", + path: "my-object", + lockOrigin: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a", + opsID: "abcd1234", + readLock: true, + setBlocked: true, + // expected metrics. + expectedErr: nil, + }, + // Test case - 2. + // No entry for pair. + // So an attempt to change the state of the lock from `Blocked`->`Running` should fail. + { + + volume: "my-bucket", + path: "my-object-2", + lockOrigin: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a", + opsID: "abcd1234", + readLock: false, + setBlocked: false, + // expected metrics. + expectedErr: LockInfoVolPathMssing{"my-bucket", "my-object-2"}, + }, + // Test case - 3. + // Entry for the given operationID doesn't exist in the lock state info. + { + volume: "my-bucket", + path: "my-object", + lockOrigin: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a", + opsID: "ops-Id-not-registered", + readLock: true, + setBlocked: false, + // expected metrics. + expectedErr: LockInfoOpsIDNotFound{"my-bucket", "my-object", "ops-Id-not-registered"}, + }, + // Test case - 4. + // Test case with non-existent lock origin. + { + volume: "my-bucket", + path: "my-object", + lockOrigin: "Bad Origin", + opsID: "abcd1234", + readLock: true, + setBlocked: false, + // expected metrics. + expectedErr: LockInfoOriginNotFound{"my-bucket", "my-object", "abcd1234", "Bad Origin"}, + }, + // Test case - 5. + // Test case with write lock. + { + + volume: "my-bucket", + path: "my-object", + lockOrigin: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a", + opsID: "abcd1234", + readLock: false, + setBlocked: true, + // expected metrics. + expectedErr: nil, + }, + } + + param := nsParam{testCases[0].volume, testCases[0].path} + // Testing before the initialization done. + // Since the data structures for + actualErr := nsMutex.statusBlockedToRunning(param, testCases[0].lockOrigin, + testCases[0].opsID, testCases[0].readLock) + + expectedNilErr := LockInfoNil{} + if actualErr != expectedNilErr { + t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedNilErr, actualErr) + } + + nsMutex = &nsLockMap{ + // entries of -> stateInfo of locks, for instrumentation purpose. + debugLockMap: make(map[nsParam]*debugLockInfoPerVolumePath), + lockMap: make(map[nsParam]*nsLock), + } + // Entry for pair is set to nil. + // Should fail with `LockInfoNil{}`. + nsMutex.debugLockMap[param] = nil + actualErr = nsMutex.statusBlockedToRunning(param, testCases[0].lockOrigin, + testCases[0].opsID, testCases[0].readLock) + + expectedNilErr = LockInfoNil{} + if actualErr != expectedNilErr { + t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedNilErr, actualErr) + } + + // Setting the lock info the be `nil`. + nsMutex.debugLockMap[param] = &debugLockInfoPerVolumePath{ + lockInfo: nil, // setting the lockinfo to nil. + ref: 0, + blocked: 0, + running: 0, + } + + actualErr = nsMutex.statusBlockedToRunning(param, testCases[0].lockOrigin, + testCases[0].opsID, testCases[0].readLock) + + expectedOpsErr := LockInfoOpsIDNotFound{testCases[0].volume, testCases[0].path, testCases[0].opsID} + if actualErr != expectedOpsErr { + t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedOpsErr, actualErr) + } + + // Next case: ase whether an attempt to change the state of the lock to "Running" done, + // but the initial state if already "Running". Such an attempt should fail + nsMutex.debugLockMap[param] = &debugLockInfoPerVolumePath{ + lockInfo: make(map[string]debugLockInfo), + ref: 0, + blocked: 0, + running: 0, + } + + // Setting the status of the lock to be "Running". + // The initial state of the lock should set to "Blocked", otherwise its not possible to change the state from "Blocked" -> "Running". + nsMutex.debugLockMap[param].lockInfo[testCases[0].opsID] = debugLockInfo{ + lockOrigin: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a", + status: "Running", // State set to "Running". Should fail with `LockInfoStateNotBlocked`. + since: time.Now().UTC(), + } + + actualErr = nsMutex.statusBlockedToRunning(param, testCases[0].lockOrigin, + testCases[0].opsID, testCases[0].readLock) + + expectedBlockErr := LockInfoStateNotBlocked{testCases[0].volume, testCases[0].path, testCases[0].opsID} + if actualErr != expectedBlockErr { + t.Fatalf("Errors mismatch: Expected: \"%s\", got: \"%s\"", expectedBlockErr, actualErr) + } + + // enabling lock instrumentation. + globalDebugLock = true + // initializing the locks. + initNSLock(false) + // set debug lock info to `nil` so that the next tests have to initialize them again. + defer func() { + globalDebugLock = false + nsMutex.debugLockMap = nil + }() + // Iterate over the cases and assert the result. + for i, testCase := range testCases { + param := nsParam{testCase.volume, testCase.path} + // status of the lock to be set to "Blocked", before setting Blocked->Running. + if testCase.setBlocked { + nsMutex.lockMapMutex.Lock() + err := nsMutex.statusNoneToBlocked(param, testCase.lockOrigin, testCase.opsID, testCase.readLock) + if err != nil { + t.Fatalf("Test %d: Initializing the initial state to Blocked failed %s", i+1, err) + } + nsMutex.lockMapMutex.Unlock() + } + // invoking the method under test. + actualErr = nsMutex.statusBlockedToRunning(param, testCase.lockOrigin, testCase.opsID, testCase.readLock) + if actualErr != testCase.expectedErr { + t.Fatalf("Test %d: Errors mismatch: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, actualErr) + } + // In case of no error proceed with validating the lock state information. + if actualErr == nil { + // debug entry for given pair should exist. + if debugLockMap, ok := nsMutex.debugLockMap[param]; ok { + if lockInfo, ok := debugLockMap.lockInfo[testCase.opsID]; ok { + // Validating the lock type filed in the debug lock information. + if testCase.readLock { + if lockInfo.lockType != debugRLockStr { + t.Errorf("Test case %d: Expected the lock type in the lock debug info to be \"%s\"", i+1, debugRLockStr) + } + } else { + if lockInfo.lockType != debugWLockStr { + t.Errorf("Test case %d: Expected the lock type in the lock debug info to be \"%s\"", i+1, debugWLockStr) + } + } + + // validating the lock origin. + if testCase.lockOrigin != lockInfo.lockOrigin { + t.Errorf("Test %d: Expected the lock origin info to be \"%s\", but got \"%s\"", i+1, testCase.lockOrigin, lockInfo.lockOrigin) + } + // validating the status of the lock. + if lockInfo.status != "Running" { + t.Errorf("Test %d: Expected the status of the lock to be \"%s\", but got \"%s\"", i+1, "Running", lockInfo.status) + } + } else { + // Stop the tests if lock debug entry for given pair is not found. + t.Fatalf("Test case %d: Expected an debug lock entry for opsID \"%s\"", i+1, testCase.opsID) + } + } else { + // To change the status the entry for given should exist in the lock info struct. + t.Fatalf("Test case %d: Debug lock entry for volume: %s, path: %s doesn't exist", i+1, param.volume, param.path) + } + } + } + +} + +// TestNsLockMapStatusNoneToBlocked - Validates the function for changing the lock state to blocked +func TestNsLockMapStatusNoneToBlocked(t *testing.T) { + + testCases := []lockStateCase{ + // Test case - 1. + { + + volume: "my-bucket", + path: "my-object", + lockOrigin: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a", + opsID: "abcd1234", + readLock: true, + // expected metrics. + expectedErr: nil, + expectedLockStatus: "Blocked", + + expectedGlobalLockCount: 1, + expectedRunningLockCount: 0, + expectedBlockedLockCount: 1, + + expectedVolPathLockCount: 1, + expectedVolPathRunningCount: 0, + expectedVolPathBlockCount: 1, + }, + // Test case - 2. + // No entry for pair. + // So an attempt to change the state of the lock from `Blocked`->`Running` should fail. + { + + volume: "my-bucket", + path: "my-object-2", + lockOrigin: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a", + opsID: "abcd1234", + readLock: false, + // expected metrics. + expectedErr: nil, + expectedLockStatus: "Blocked", + + expectedGlobalLockCount: 2, + expectedRunningLockCount: 0, + expectedBlockedLockCount: 2, + + expectedVolPathLockCount: 1, + expectedVolPathRunningCount: 0, + expectedVolPathBlockCount: 1, + }, + // Test case - 3. + // Entry for the given operationID doesn't exist in the lock state info. + // The entry should be created and relevant counters should be set. + { + volume: "my-bucket", + path: "my-object", + lockOrigin: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a", + opsID: "ops-Id-not-registered", + readLock: true, + // expected metrics. + expectedErr: nil, + expectedLockStatus: "Blocked", + + expectedGlobalLockCount: 3, + expectedRunningLockCount: 0, + expectedBlockedLockCount: 3, + + expectedVolPathLockCount: 2, + expectedVolPathRunningCount: 0, + expectedVolPathBlockCount: 2, + }, + } + + param := nsParam{testCases[0].volume, testCases[0].path} + // Testing before the initialization done. + // Since the data structures for + actualErr := nsMutex.statusBlockedToRunning(param, testCases[0].lockOrigin, + testCases[0].opsID, testCases[0].readLock) + + expectedNilErr := LockInfoNil{} + if actualErr != expectedNilErr { + t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedNilErr, actualErr) + } + // enabling lock instrumentation. + globalDebugLock = true + // initializing the locks. + initNSLock(false) + // set debug lock info to `nil` so that the next tests have to initialize them again. + defer func() { + globalDebugLock = false + nsMutex.debugLockMap = nil + }() + // Iterate over the cases and assert the result. + for i, testCase := range testCases { + nsMutex.lockMapMutex.Lock() + param := nsParam{testCase.volume, testCase.path} + actualErr := nsMutex.statusNoneToBlocked(param, testCase.lockOrigin, testCase.opsID, testCase.readLock) + if actualErr != testCase.expectedErr { + t.Fatalf("Test %d: Errors mismatch: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, actualErr) + } + nsMutex.lockMapMutex.Unlock() + if actualErr == nil { + verifyLockState(testCase, t, i+1) + } + } +} + +// TestNsLockMapDeleteLockInfoEntryForOps - Validates the removal of entry for given Operational ID from the lock info. +func TestNsLockMapDeleteLockInfoEntryForOps(t *testing.T) { + testCases := []lockStateCase{ + // Test case - 1. + { + volume: "my-bucket", + path: "my-object", + lockOrigin: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a", + opsID: "abcd1234", + readLock: true, + // expected metrics. + }, + } + // case - 1. + // Testing the case where delete lock info is attempted even before the lock is initialized. + param := nsParam{testCases[0].volume, testCases[0].path} + // Testing before the initialization done. + + actualErr := nsMutex.deleteLockInfoEntryForOps(param, testCases[0].opsID) + + expectedNilErr := LockInfoNil{} + if actualErr != expectedNilErr { + t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedNilErr, actualErr) + } + + // enabling lock instrumentation. + globalDebugLock = true + // initializing the locks. + initNSLock(false) + // set debug lock info to `nil` so that the next tests have to initialize them again. + defer func() { + globalDebugLock = false + nsMutex.debugLockMap = nil + }() + // case - 2. + // Case where an attempt to delete the entry for non-existent pair is done. + // Set the status of the lock to blocked and then to running. + nonExistParam := nsParam{volume: "non-exist-volume", path: "non-exist-path"} + actualErr = nsMutex.deleteLockInfoEntryForOps(nonExistParam, testCases[0].opsID) + + expectedVolPathErr := LockInfoVolPathMssing{nonExistParam.volume, nonExistParam.path} + if actualErr != expectedVolPathErr { + t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedVolPathErr, actualErr) + } + + // Case - 3. + // Lock state is set to Running and then an attempt to delete the info for non-existent opsID done. + nsMutex.lockMapMutex.Lock() + err := nsMutex.statusNoneToBlocked(param, testCases[0].lockOrigin, testCases[0].opsID, testCases[0].readLock) + if err != nil { + t.Fatalf("Setting lock status to Blocked failed: %s", err) + } + nsMutex.lockMapMutex.Unlock() + err = nsMutex.statusBlockedToRunning(param, testCases[0].lockOrigin, testCases[0].opsID, testCases[0].readLock) + if err != nil { + t.Fatalf("Setting lock status to Running failed: %s", err) + } + actualErr = nsMutex.deleteLockInfoEntryForOps(param, "non-existent-OpsID") + + expectedOpsIDErr := LockInfoOpsIDNotFound{param.volume, param.path, "non-existent-OpsID"} + if actualErr != expectedOpsIDErr { + t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedOpsIDErr, actualErr) + } + // case - 4. + // Attempt to delete an registered entry is done. + // All metrics should be 0 after deleting the entry. + + // Verify that the entry the opsID exists. + if debugLockMap, ok := nsMutex.debugLockMap[param]; ok { + if _, ok := debugLockMap.lockInfo[testCases[0].opsID]; !ok { + t.Fatalf("Entry for OpsID \"%s\" in %s, %s should have existed. ", testCases[0].opsID, param.volume, param.path) + } + } else { + t.Fatalf("Entry for %s, %s should have existed. ", param.volume, param.path) + } + + actualErr = nsMutex.deleteLockInfoEntryForOps(param, testCases[0].opsID) + if actualErr != nil { + t.Fatalf("Expected the error to be , but got %s", actualErr) + } + + // Verify that the entry for the opsId doesn't exists. + if debugLockMap, ok := nsMutex.debugLockMap[param]; ok { + if _, ok := debugLockMap.lockInfo[testCases[0].opsID]; ok { + t.Fatalf("The entry for opsID \"%s\" should have been deleted", testCases[0].opsID) + } + } else { + t.Fatalf("Entry for %s, %s should have existed. ", param.volume, param.path) + } + if nsMutex.runningLockCounter != int64(0) { + t.Errorf("Expected the count of total running locks to be %v, but got %v", int64(0), nsMutex.runningLockCounter) + } + if nsMutex.blockedCounter != int64(0) { + t.Errorf("Expected the count of total blocked locks to be %v, but got %v", int64(0), nsMutex.blockedCounter) + } + if nsMutex.globalLockCounter != int64(0) { + t.Errorf("Expected the count of all locks to be %v, but got %v", int64(0), nsMutex.globalLockCounter) + } +} + +// TestNsLockMapDeleteLockInfoEntryForVolumePath - Tests validate the logic for removal +// of entry for given pair from lock info. +func TestNsLockMapDeleteLockInfoEntryForVolumePath(t *testing.T) { + testCases := []lockStateCase{ + // Test case - 1. + { + volume: "my-bucket", + path: "my-object", + lockOrigin: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a", + opsID: "abcd1234", + readLock: true, + // expected metrics. + }, + } + // case - 1. + // Testing the case where delete lock info is attempted even before the lock is initialized. + param := nsParam{testCases[0].volume, testCases[0].path} + // Testing before the initialization done. + + actualErr := nsMutex.deleteLockInfoEntryForVolumePath(param) + + expectedNilErr := LockInfoNil{} + if actualErr != expectedNilErr { + t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedNilErr, actualErr) + } + + // enabling lock instrumentation. + globalDebugLock = true + // initializing the locks. + initNSLock(false) + // set debug lock info to `nil` so that the next tests have to initialize them again. + defer func() { + globalDebugLock = false + nsMutex.debugLockMap = nil + }() + // case - 2. + // Case where an attempt to delete the entry for non-existent pair is done. + // Set the status of the lock to blocked and then to running. + nonExistParam := nsParam{volume: "non-exist-volume", path: "non-exist-path"} + actualErr = nsMutex.deleteLockInfoEntryForVolumePath(nonExistParam) + + expectedVolPathErr := LockInfoVolPathMssing{nonExistParam.volume, nonExistParam.path} + if actualErr != expectedVolPathErr { + t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedVolPathErr, actualErr) + } + + // case - 3. + // Attempt to delete an registered entry is done. + // All metrics should be 0 after deleting the entry. + + // Registering the entry first. + nsMutex.lockMapMutex.Lock() + err := nsMutex.statusNoneToBlocked(param, testCases[0].lockOrigin, testCases[0].opsID, testCases[0].readLock) + if err != nil { + t.Fatalf("Setting lock status to Blocked failed: %s", err) + } + nsMutex.lockMapMutex.Unlock() + err = nsMutex.statusBlockedToRunning(param, testCases[0].lockOrigin, testCases[0].opsID, testCases[0].readLock) + if err != nil { + t.Fatalf("Setting lock status to Running failed: %s", err) + } + // Verify that the entry the for given exists. + if _, ok := nsMutex.debugLockMap[param]; !ok { + t.Fatalf("Entry for %s, %s should have existed.", param.volume, param.path) + } + // first delete the entry for the operation ID. + _ = nsMutex.deleteLockInfoEntryForOps(param, testCases[0].opsID) + actualErr = nsMutex.deleteLockInfoEntryForVolumePath(param) + if actualErr != nil { + t.Fatalf("Expected the error to be , but got %s", actualErr) + } + + // Verify that the entry for the opsId doesn't exists. + if _, ok := nsMutex.debugLockMap[param]; ok { + t.Fatalf("Entry for %s, %s should have been deleted. ", param.volume, param.path) + } + // The lock count values should be 0. + if nsMutex.runningLockCounter != int64(0) { + t.Errorf("Expected the count of total running locks to be %v, but got %v", int64(0), nsMutex.runningLockCounter) + } + if nsMutex.blockedCounter != int64(0) { + t.Errorf("Expected the count of total blocked locks to be %v, but got %v", int64(0), nsMutex.blockedCounter) + } + if nsMutex.globalLockCounter != int64(0) { + t.Errorf("Expected the count of all locks to be %v, but got %v", int64(0), nsMutex.globalLockCounter) + } +} diff --git a/cmd/lock-rpc-server.go b/cmd/lock-rpc-server.go new file mode 100644 index 000000000..d80aeb998 --- /dev/null +++ b/cmd/lock-rpc-server.go @@ -0,0 +1,227 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "fmt" + "net/rpc" + "path" + "strings" + "sync" + "time" + + router "github.com/gorilla/mux" +) + +const lockRPCPath = "/minio/lock" + +// LockArgs besides lock name, holds Token and Timestamp for session +// authentication and validation server restart. +type LockArgs struct { + Name string + Token string + Timestamp time.Time +} + +// SetToken - sets the token to the supplied value. +func (l *LockArgs) SetToken(token string) { + l.Token = token +} + +// SetTimestamp - sets the timestamp to the supplied value. +func (l *LockArgs) SetTimestamp(tstamp time.Time) { + l.Timestamp = tstamp +} + +type lockServer struct { + rpcPath string + mutex sync.Mutex + // e.g, when a Lock(name) is held, map[string][]bool{"name" : []bool{true}} + // when one or more RLock() is held, map[string][]bool{"name" : []bool{false, false}} + lockMap map[string][]bool + timestamp time.Time // Timestamp set at the time of initialization. Resets naturally on minio server restart. +} + +func (l *lockServer) verifyArgs(args *LockArgs) error { + if !l.timestamp.Equal(args.Timestamp) { + return errInvalidTimestamp + } + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + return nil +} + +/// Distributed lock handlers + +// LoginHandler - handles LoginHandler RPC call. +func (l *lockServer) LoginHandler(args *RPCLoginArgs, reply *RPCLoginReply) error { + jwt, err := newJWT(defaultTokenExpiry) + if err != nil { + return err + } + if err = jwt.Authenticate(args.Username, args.Password); err != nil { + return err + } + token, err := jwt.GenerateToken(args.Username) + if err != nil { + return err + } + reply.Token = token + reply.Timestamp = l.timestamp + return nil +} + +// Lock - rpc handler for (single) write lock operation. +func (l *lockServer) Lock(args *LockArgs, reply *bool) error { + l.mutex.Lock() + defer l.mutex.Unlock() + if err := l.verifyArgs(args); err != nil { + return err + } + _, ok := l.lockMap[args.Name] + // No locks held on the given name. + if !ok { + *reply = true + l.lockMap[args.Name] = []bool{true} + } else { + // Either a read or write lock is held on the given name. + *reply = false + } + return nil +} + +// Unlock - rpc handler for (single) write unlock operation. +func (l *lockServer) Unlock(args *LockArgs, reply *bool) error { + l.mutex.Lock() + defer l.mutex.Unlock() + if err := l.verifyArgs(args); err != nil { + return err + } + locksHeld, ok := l.lockMap[args.Name] + // No lock is held on the given name, there must be some issue at the lock client side. + if !ok { + *reply = false + return fmt.Errorf("Unlock attempted on an un-locked entity: %s", args.Name) + } else if len(locksHeld) == 1 && locksHeld[0] == true { + *reply = true + delete(l.lockMap, args.Name) + return nil + } else { + *reply = false + return fmt.Errorf("Unlock attempted on a read locked entity: %s (%d read locks active)", args.Name, len(locksHeld)) + } +} + +// RLock - rpc handler for read lock operation. +func (l *lockServer) RLock(args *LockArgs, reply *bool) error { + l.mutex.Lock() + defer l.mutex.Unlock() + if err := l.verifyArgs(args); err != nil { + return err + } + locksHeld, ok := l.lockMap[args.Name] + // No locks held on the given name. + if !ok { + // First read-lock to be held on *name. + l.lockMap[args.Name] = []bool{false} + *reply = true + } else if len(locksHeld) == 1 && locksHeld[0] == true { + // A write-lock is held, read lock can't be granted. + *reply = false + } else { + // Add an entry for this read lock. + l.lockMap[args.Name] = append(locksHeld, false) + *reply = true + } + return nil +} + +// RUnlock - rpc handler for read unlock operation. +func (l *lockServer) RUnlock(args *LockArgs, reply *bool) error { + l.mutex.Lock() + defer l.mutex.Unlock() + if err := l.verifyArgs(args); err != nil { + return err + } + locksHeld, ok := l.lockMap[args.Name] + if !ok { + *reply = false + return fmt.Errorf("RUnlock attempted on an un-locked entity: %s", args.Name) + } else if len(locksHeld) == 1 && locksHeld[0] == true { + // A write-lock is held, cannot release a read lock + *reply = false + return fmt.Errorf("RUnlock attempted on a write locked entity: %s", args.Name) + } else if len(locksHeld) > 1 { + // Remove one of the read locks held. + locksHeld = locksHeld[1:] + l.lockMap[args.Name] = locksHeld + *reply = true + } else { + // Delete the map entry since this is the last read lock held + // on *name. + delete(l.lockMap, args.Name) + *reply = true + } + return nil +} + +// Initialize distributed lock. +func initDistributedNSLock(mux *router.Router, serverConfig serverCmdConfig) { + lockServers := newLockServers(serverConfig) + registerStorageLockers(mux, lockServers) +} + +// Create one lock server for every local storage rpc server. +func newLockServers(serverConfig serverCmdConfig) (lockServers []*lockServer) { + // Initialize posix storage API. + exports := serverConfig.disks + ignoredExports := serverConfig.ignoredDisks + + // Save ignored disks in a map + skipDisks := make(map[string]bool) + for _, ignoredExport := range ignoredExports { + skipDisks[ignoredExport] = true + } + for _, export := range exports { + if skipDisks[export] { + continue + } + if isLocalStorage(export) { + if idx := strings.LastIndex(export, ":"); idx != -1 { + export = export[idx+1:] + } + lockServers = append(lockServers, &lockServer{ + rpcPath: export, + mutex: sync.Mutex{}, + lockMap: make(map[string][]bool), + timestamp: time.Now().UTC(), + }) + } + } + return lockServers +} + +// registerStorageLockers - register locker rpc handlers for net/rpc library clients +func registerStorageLockers(mux *router.Router, lockServers []*lockServer) { + for _, lockServer := range lockServers { + lockRPCServer := rpc.NewServer() + lockRPCServer.RegisterName("Dsync", lockServer) + lockRouter := mux.PathPrefix(reservedBucket).Subrouter() + lockRouter.Path(path.Join("/lock", lockServer.rpcPath)).Handler(lockRPCServer) + } +} diff --git a/cmd/logger.go b/cmd/logger.go index dd0191104..d2405807b 100644 --- a/cmd/logger.go +++ b/cmd/logger.go @@ -67,9 +67,10 @@ func errorIf(err error, msg string, data ...interface{}) { fields := logrus.Fields{ "cause": err.Error(), } - if globalTrace { - fields["stack"] = "\n" + stackInfo() + if e, ok := err.(*Error); ok { + fields["stack"] = strings.Join(e.Trace(), " ") } + log.WithFields(fields).Errorf(msg, data...) } diff --git a/cmd/main.go b/cmd/main.go index 578aefd70..2d7d0fe1a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "sort" + "strings" "github.com/minio/cli" "github.com/minio/mc/pkg/console" @@ -27,11 +28,20 @@ import ( var ( // global flags for minio. - minioFlags = []cli.Flag{ + globalFlags = []cli.Flag{ cli.BoolFlag{ Name: "help, h", Usage: "Show help.", }, + cli.StringFlag{ + Name: "config-dir, C", + Value: mustGetConfigPath(), + Usage: "Path to configuration folder.", + }, + cli.BoolFlag{ + Name: "quiet", + Usage: "Suppress chatty output.", + }, } ) @@ -62,6 +72,9 @@ func init() { // Set global trace flag. globalTrace = os.Getenv("MINIO_TRACE") == "1" + + // Set all the debug flags from ENV if any. + setGlobalsDebugFromEnv() } func migrate() { @@ -112,7 +125,7 @@ func registerApp() *cli.App { app.Author = "Minio.io" app.Usage = "Cloud Storage Server." app.Description = `Minio is an Amazon S3 compatible object storage server. Use it to store photos, videos, VMs, containers, log files, or any blob of data as objects.` - app.Flags = append(minioFlags, globalFlags...) + app.Flags = globalFlags app.Commands = commands app.CustomAppHelpTemplate = minioHelpTemplate app.CommandNotFound = func(ctx *cli.Context, command string) { @@ -160,19 +173,24 @@ func Main() { // Enable all loggers by now. enableLoggers() + // Init the error tracing module. + initError() + // Set global quiet flag. globalQuiet = c.Bool("quiet") || c.GlobalBool("quiet") // Do not print update messages, if quiet flag is set. if !globalQuiet { - // Do not print any errors in release update function. - noError := true - updateMsg := getReleaseUpdate(minioUpdateStableURL, noError) - if updateMsg.Update { + if strings.HasPrefix(Version, "RELEASE.") { + updateMsg, _, err := getReleaseUpdate(minioUpdateStableURL) + if err != nil { + // Ignore any errors during getReleaseUpdate() because + // the internet might not be available. + return nil + } console.Println(updateMsg) } } - return nil } diff --git a/cmd/namespace-lock.go b/cmd/namespace-lock.go index 44cf0df87..3eb4ce059 100644 --- a/cmd/namespace-lock.go +++ b/cmd/namespace-lock.go @@ -18,9 +18,66 @@ package cmd import ( "errors" + "fmt" + pathutil "path" + "runtime" + "strconv" + "strings" "sync" + + "github.com/minio/dsync" ) +// Global name space lock. +var nsMutex *nsLockMap + +// Initialize distributed locking only in case of distributed setup. +// Returns if the setup is distributed or not on success. +func initDsyncNodes(disks []string, port int) error { + serverPort := strconv.Itoa(port) + cred := serverConfig.GetCredential() + // Initialize rpc lock client information only if this instance is a distributed setup. + var clnts []dsync.RPC + for _, disk := range disks { + if idx := strings.LastIndex(disk, ":"); idx != -1 { + clnts = append(clnts, newAuthClient(&authConfig{ + accessKey: cred.AccessKeyID, + secretKey: cred.SecretAccessKey, + // Construct a new dsync server addr. + address: disk[:idx] + ":" + serverPort, + // Construct a new rpc path for the disk. + path: pathutil.Join(lockRPCPath, disk[idx+1:]), + loginMethod: "Dsync.LoginHandler", + })) + } + } + return dsync.SetNodesWithClients(clnts) +} + +// initNSLock - initialize name space lock map. +func initNSLock(isDist bool) { + nsMutex = &nsLockMap{ + isDist: isDist, + lockMap: make(map[nsParam]*nsLock), + } + if globalDebugLock { + // lock Debugging enabed, initialize nsLockMap with entry for debugging information. + // entries of -> stateInfo of locks, for instrumentation purpose. + nsMutex.debugLockMap = make(map[nsParam]*debugLockInfoPerVolumePath) + } +} + +func (n *nsLockMap) initLockInfoForVolumePath(param nsParam) { + n.debugLockMap[param] = newDebugLockInfoPerVolumePath() +} + +// RWLocker - interface that any read-write locking library should implement. +type RWLocker interface { + sync.Locker + RLock() + RUnlock() +} + // nsParam - carries name space resource. type nsParam struct { volume string @@ -29,98 +86,189 @@ type nsParam struct { // nsLock - provides primitives for locking critical namespace regions. type nsLock struct { - sync.RWMutex - ref uint + writer RWLocker + readerArray []RWLocker + ref uint } // nsLockMap - namespace lock map, provides primitives to Lock, // Unlock, RLock and RUnlock. type nsLockMap struct { - lockMap map[nsParam]*nsLock - mutex sync.Mutex -} + // lock counter used for lock debugging. + globalLockCounter int64 //total locks held. + blockedCounter int64 // total operations blocked waiting for locks. + runningLockCounter int64 // total locks held but not released yet. + debugLockMap map[nsParam]*debugLockInfoPerVolumePath // info for instrumentation on locks. -// Global name space lock. -var nsMutex *nsLockMap - -// initNSLock - initialize name space lock map. -func initNSLock() { - nsMutex = &nsLockMap{ - lockMap: make(map[nsParam]*nsLock), - } + isDist bool // indicates whether the locking service is part of a distributed setup or not. + lockMap map[nsParam]*nsLock + lockMapMutex sync.Mutex } // Lock the namespace resource. -func (n *nsLockMap) lock(volume, path string, readLock bool) { - n.mutex.Lock() +func (n *nsLockMap) lock(volume, path string, lockOrigin, opsID string, readLock bool) { + var nsLk *nsLock + n.lockMapMutex.Lock() param := nsParam{volume, path} nsLk, found := n.lockMap[param] if !found { nsLk = &nsLock{ + writer: func() RWLocker { + if n.isDist { + return dsync.NewDRWMutex(pathutil.Join(volume, path)) + } + return &sync.RWMutex{} + }(), ref: 0, } n.lockMap[param] = nsLk } nsLk.ref++ // Update ref count here to avoid multiple races. + rwlock := nsLk.writer + if readLock && n.isDist { + rwlock = dsync.NewDRWMutex(pathutil.Join(volume, path)) + } + + if globalDebugLock { + // change the state of the lock to be blocked for the given pair of and till the lock unblocks. + // The lock for accessing `nsMutex` is held inside the function itself. + err := n.statusNoneToBlocked(param, lockOrigin, opsID, readLock) + if err != nil { + errorIf(err, "Failed to set lock state to blocked.") + } + } // Unlock map before Locking NS which might block. - n.mutex.Unlock() + n.lockMapMutex.Unlock() // Locking here can block. if readLock { - nsLk.RLock() + rwlock.RLock() + + if n.isDist { + // Only add (for reader case) to array after RLock() succeeds + // (so that we know for sure that element in [0] can be RUnlocked()) + n.lockMapMutex.Lock() + if len(nsLk.readerArray) == 0 { + nsLk.readerArray = []RWLocker{rwlock} + } else { + nsLk.readerArray = append(nsLk.readerArray, rwlock) + } + n.lockMapMutex.Unlock() + } } else { - nsLk.Lock() + rwlock.Lock() + } + + // check if lock debugging enabled. + if globalDebugLock { + // Changing the status of the operation from blocked to running. + // change the state of the lock to be running (from blocked) for the given pair of and . + err := n.statusBlockedToRunning(param, lockOrigin, opsID, readLock) + if err != nil { + errorIf(err, "Failed to set the lock state to running.") + } } } // Unlock the namespace resource. -func (n *nsLockMap) unlock(volume, path string, readLock bool) { +func (n *nsLockMap) unlock(volume, path, opsID string, readLock bool) { // nsLk.Unlock() will not block, hence locking the map for the entire function is fine. - n.mutex.Lock() - defer n.mutex.Unlock() + n.lockMapMutex.Lock() + defer n.lockMapMutex.Unlock() param := nsParam{volume, path} if nsLk, found := n.lockMap[param]; found { if readLock { - nsLk.RUnlock() + if n.isDist { + if len(nsLk.readerArray) == 0 { + errorIf(errors.New("Length of reader lock array cannot be 0."), "Invalid reader lock array length detected.") + } + // Release first lock first (FIFO) + nsLk.readerArray[0].RUnlock() + // And discard first element + nsLk.readerArray = nsLk.readerArray[1:] + } else { + nsLk.writer.RUnlock() + } } else { - nsLk.Unlock() + nsLk.writer.Unlock() } if nsLk.ref == 0 { errorIf(errors.New("Namespace reference count cannot be 0."), "Invalid reference count detected.") } if nsLk.ref != 0 { nsLk.ref-- + // locking debug enabled, delete the lock state entry for given operation ID. + if globalDebugLock { + err := n.deleteLockInfoEntryForOps(param, opsID) + if err != nil { + errorIf(err, "Failed to delete lock info entry.") + } + } } if nsLk.ref == 0 { + if len(nsLk.readerArray) != 0 && n.isDist { + errorIf(errors.New("Length of reader lock array should be 0 upon deleting map entry."), "Invalid reader lock array length detected.") + } + // Remove from the map if there are no more references. delete(n.lockMap, param) + + // locking debug enabled, delete the lock state entry for given pair. + if globalDebugLock { + err := n.deleteLockInfoEntryForVolumePath(param) + if err != nil { + errorIf(err, "Failed to delete lock info entry.") + } + } } } } // Lock - locks the given resource for writes, using a previously // allocated name space lock or initializing a new one. -func (n *nsLockMap) Lock(volume, path string) { +func (n *nsLockMap) Lock(volume, path, opsID string) { + var lockOrigin string + // lock debugging enabled. The caller information of the lock held has be obtained here before calling any other function. + if globalDebugLock { + // fetching the package, function name and the line number of the caller from the runtime. + // here is an example https://play.golang.org/p/perrmNRI9_ . + pc, fn, line, success := runtime.Caller(1) + if !success { + errorIf(errors.New("Couldn't get caller info."), "Fetching caller info form runtime failed.") + } + lockOrigin = fmt.Sprintf("[lock held] in %s[%s:%d]", runtime.FuncForPC(pc).Name(), fn, line) + } readLock := false - n.lock(volume, path, readLock) + n.lock(volume, path, lockOrigin, opsID, readLock) } // Unlock - unlocks any previously acquired write locks. -func (n *nsLockMap) Unlock(volume, path string) { +func (n *nsLockMap) Unlock(volume, path, opsID string) { readLock := false - n.unlock(volume, path, readLock) + n.unlock(volume, path, opsID, readLock) } // RLock - locks any previously acquired read locks. -func (n *nsLockMap) RLock(volume, path string) { +func (n *nsLockMap) RLock(volume, path, opsID string) { + var lockOrigin string readLock := true - n.lock(volume, path, readLock) + // lock debugging enabled. The caller information of the lock held has be obtained here before calling any other function. + if globalDebugLock { + // fetching the package, function name and the line number of the caller from the runtime. + // here is an example https://play.golang.org/p/perrmNRI9_ . + pc, fn, line, success := runtime.Caller(1) + if !success { + errorIf(errors.New("Couldn't get caller info."), "Fetching caller info form runtime failed.") + } + lockOrigin = fmt.Sprintf("[lock held] in %s[%s:%d]", runtime.FuncForPC(pc).Name(), fn, line) + } + n.lock(volume, path, lockOrigin, opsID, readLock) } // RUnlock - unlocks any previously acquired read locks. -func (n *nsLockMap) RUnlock(volume, path string) { +func (n *nsLockMap) RUnlock(volume, path, opsID string) { readLock := true - n.unlock(volume, path, readLock) + n.unlock(volume, path, opsID, readLock) } diff --git a/cmd/namespace-lock_test.go b/cmd/namespace-lock_test.go index be2945df8..b2e359659 100644 --- a/cmd/namespace-lock_test.go +++ b/cmd/namespace-lock_test.go @@ -16,16 +16,21 @@ package cmd -import "testing" +import ( + "strconv" + "sync" + "testing" + "time" +) // Tests functionality provided by namespace lock. func TestNamespaceLockTest(t *testing.T) { // List of test cases. testCases := []struct { - lk func(s1, s2 string) - unlk func(s1, s2 string) - rlk func(s1, s2 string) - runlk func(s1, s2 string) + lk func(s1, s2, s3 string) + unlk func(s1, s2, s3 string) + rlk func(s1, s2, s3 string) + runlk func(s1, s2, s3 string) lkCount int lockedRefCount uint unlockedRefCount uint @@ -58,7 +63,7 @@ func TestNamespaceLockTest(t *testing.T) { // Write lock tests. testCase := testCases[0] - testCase.lk("a", "b") // lock once. + testCase.lk("a", "b", "c") // lock once. nsLk, ok := nsMutex.lockMap[nsParam{"a", "b"}] if !ok && testCase.shouldPass { t.Errorf("Lock in map missing.") @@ -67,7 +72,7 @@ func TestNamespaceLockTest(t *testing.T) { if testCase.lockedRefCount != nsLk.ref && testCase.shouldPass { t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 1, testCase.lockedRefCount, nsLk.ref) } - testCase.unlk("a", "b") // unlock once. + testCase.unlk("a", "b", "c") // unlock once. if testCase.unlockedRefCount != nsLk.ref && testCase.shouldPass { t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 1, testCase.unlockedRefCount, nsLk.ref) } @@ -78,10 +83,10 @@ func TestNamespaceLockTest(t *testing.T) { // Read lock tests. testCase = testCases[1] - testCase.rlk("a", "b") // lock once. - testCase.rlk("a", "b") // lock second time. - testCase.rlk("a", "b") // lock third time. - testCase.rlk("a", "b") // lock fourth time. + testCase.rlk("a", "b", "c") // lock once. + testCase.rlk("a", "b", "c") // lock second time. + testCase.rlk("a", "b", "c") // lock third time. + testCase.rlk("a", "b", "c") // lock fourth time. nsLk, ok = nsMutex.lockMap[nsParam{"a", "b"}] if !ok && testCase.shouldPass { t.Errorf("Lock in map missing.") @@ -90,8 +95,9 @@ func TestNamespaceLockTest(t *testing.T) { if testCase.lockedRefCount != nsLk.ref && testCase.shouldPass { t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 1, testCase.lockedRefCount, nsLk.ref) } - testCase.runlk("a", "b") // unlock once. - testCase.runlk("a", "b") // unlock second time. + + testCase.runlk("a", "b", "c") // unlock once. + testCase.runlk("a", "b", "c") // unlock second time. if testCase.unlockedRefCount != nsLk.ref && testCase.shouldPass { t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 2, testCase.unlockedRefCount, nsLk.ref) } @@ -102,7 +108,7 @@ func TestNamespaceLockTest(t *testing.T) { // Read lock 0 ref count. testCase = testCases[2] - testCase.rlk("a", "c") // lock once. + testCase.rlk("a", "c", "d") // lock once. nsLk, ok = nsMutex.lockMap[nsParam{"a", "c"}] if !ok && testCase.shouldPass { @@ -112,7 +118,7 @@ func TestNamespaceLockTest(t *testing.T) { if testCase.lockedRefCount != nsLk.ref && testCase.shouldPass { t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 3, testCase.lockedRefCount, nsLk.ref) } - testCase.runlk("a", "c") // unlock once. + testCase.runlk("a", "c", "d") // unlock once. if testCase.unlockedRefCount != nsLk.ref && testCase.shouldPass { t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 3, testCase.unlockedRefCount, nsLk.ref) } @@ -121,3 +127,266 @@ func TestNamespaceLockTest(t *testing.T) { t.Errorf("Lock map not found.") } } + +func TestLockStats(t *testing.T) { + + expectedResult := []lockStateCase{ + // Test case - 1. + // Case where 10 read locks are held. + // Entry for any of the 10 reads locks has to be found. + // Since they held in a loop, Lock origin for first 10 read locks (opsID 0-9) should be the same. + { + + volume: "my-bucket", + path: "my-object", + opsID: "0", + readLock: true, + lockOrigin: "[lock held] in github.com/minio/minio/cmd.TestLockStats[/Users/hackintoshrao/mycode/go/src/github.com/minio/minio/cmd/namespace-lock_test.go:298]", + // expected metrics. + expectedErr: nil, + expectedLockStatus: "Running", + + expectedGlobalLockCount: 10, + expectedRunningLockCount: 10, + expectedBlockedLockCount: 0, + + expectedVolPathLockCount: 10, + expectedVolPathRunningCount: 10, + expectedVolPathBlockCount: 0, + }, + // Test case - 2. + // Case where the first 5 read locks are released. + // Entry for any of the 6-10th "Running" reads lock has to be found. + { + volume: "my-bucket", + path: "my-object", + opsID: "6", + readLock: true, + lockOrigin: "[lock held] in github.com/minio/minio/cmd.TestLockStats[/Users/hackintoshrao/mycode/go/src/github.com/minio/minio/cmd/namespace-lock_test.go:298]", + // expected metrics. + expectedErr: nil, + expectedLockStatus: "Running", + + expectedGlobalLockCount: 5, + expectedRunningLockCount: 5, + expectedBlockedLockCount: 0, + + expectedVolPathLockCount: 5, + expectedVolPathRunningCount: 5, + expectedVolPathBlockCount: 0, + }, + // Test case - 3. + { + + volume: "my-bucket", + path: "my-object", + opsID: "10", + readLock: false, + lockOrigin: "[lock held] in github.com/minio/minio/cmd.TestLockStats[/Users/hackintoshrao/mycode/go/src/github.com/minio/minio/cmd/namespace-lock_test.go:298]", + // expected metrics. + expectedErr: nil, + expectedLockStatus: "Running", + + expectedGlobalLockCount: 2, + expectedRunningLockCount: 1, + expectedBlockedLockCount: 1, + + expectedVolPathLockCount: 2, + expectedVolPathRunningCount: 1, + expectedVolPathBlockCount: 1, + }, + // Test case - 4. + { + + volume: "my-bucket", + path: "my-object", + // expected metrics. + expectedErr: nil, + expectedLockStatus: "Blocked", + + expectedGlobalLockCount: 1, + expectedRunningLockCount: 0, + expectedBlockedLockCount: 1, + + expectedVolPathLockCount: 1, + expectedVolPathRunningCount: 0, + expectedVolPathBlockCount: 1, + }, + // Test case - 5. + { + + volume: "my-bucket", + path: "my-object", + opsID: "11", + readLock: false, + lockOrigin: "[lock held] in github.com/minio/minio/cmd.TestLockStats[/Users/hackintoshrao/mycode/go/src/github.com/minio/minio/cmd/namespace-lock_test.go:298]", + // expected metrics. + expectedErr: nil, + expectedLockStatus: "Running", + + expectedGlobalLockCount: 1, + expectedRunningLockCount: 1, + expectedBlockedLockCount: 0, + + expectedVolPathLockCount: 1, + expectedVolPathRunningCount: 1, + expectedVolPathBlockCount: 0, + }, + // Test case - 6. + // Case where in the first 5 read locks are released, but 2 write locks are + // blocked waiting for the remaining 5 read locks locks to be released (10 read locks were held initially). + // We check the entry for the first blocked write call here. + { + + volume: "my-bucket", + path: "my-object", + opsID: "10", + readLock: false, + // write lock is held at line 318. + // this confirms that we are looking the right write lock. + lockOrigin: "[lock held] in github.com/minio/minio/cmd.TestLockStats.func2[/Users/hackintoshrao/mycode/go/src/github.com/minio/minio/cmd/namespace-lock_test.go:318]", + // expected metrics. + expectedErr: nil, + expectedLockStatus: "Blocked", + + // count of held(running) + blocked locks. + expectedGlobalLockCount: 7, + // count of acquired locks. + expectedRunningLockCount: 5, + // 2 write calls are blocked, waiting for the remaining 5 read locks. + expectedBlockedLockCount: 2, + + expectedVolPathLockCount: 7, + expectedVolPathRunningCount: 5, + expectedVolPathBlockCount: 2, + }, + // Test case - 7. + // Case where in 9 out of 10 read locks are released. + // Since there's one more pending read lock, the 2 write locks are still blocked. + // Testing the entry for the last read lock. + {volume: "my-bucket", + path: "my-object", + opsID: "9", + readLock: true, + lockOrigin: "[lock held] in github.com/minio/minio/cmd.TestLockStats.func2[/Users/hackintoshrao/mycode/go/src/github.com/minio/minio/cmd/namespace-lock_test.go:318]", + // expected metrics. + expectedErr: nil, + expectedLockStatus: "Running", + + // Total running + blocked locks. + // 2 blocked write lock. + expectedGlobalLockCount: 3, + expectedRunningLockCount: 1, + expectedBlockedLockCount: 2, + + expectedVolPathLockCount: 3, + expectedVolPathRunningCount: 1, + expectedVolPathBlockCount: 2, + }, + // Test case - 8. + { + + volume: "my-bucket", + path: "my-object", + // expected metrics. + expectedErr: nil, + expectedLockStatus: "Blocked", + + expectedGlobalLockCount: 0, + expectedRunningLockCount: 0, + expectedBlockedLockCount: 0, + }, + } + var wg sync.WaitGroup + // enabling lock instrumentation. + globalDebugLock = true + // initializing the locks. + initNSLock(false) + + // set debug lock info to `nil` so that the next tests have to initialize them again. + defer func() { + globalDebugLock = false + nsMutex.debugLockMap = nil + }() + + // hold 10 read locks. + for i := 0; i < 10; i++ { + nsMutex.RLock("my-bucket", "my-object", strconv.Itoa(i)) + } + // expected lock info. + expectedLockStats := expectedResult[0] + // verify the actual lock info with the expected one. + verifyLockState(expectedLockStats, t, 1) + // unlock 5 readlock. + for i := 0; i < 5; i++ { + nsMutex.RUnlock("my-bucket", "my-object", strconv.Itoa(i)) + } + + expectedLockStats = expectedResult[1] + // verify the actual lock info with the expected one. + verifyLockState(expectedLockStats, t, 2) + + syncChan := make(chan struct{}, 1) + wg.Add(1) + go func() { + defer wg.Done() + // blocks till all read locks are released. + nsMutex.Lock("my-bucket", "my-object", strconv.Itoa(10)) + // Once the above attempt to lock is unblocked/acquired, we verify the stats and release the lock. + expectedWLockStats := expectedResult[2] + // Since the write lock acquired here, the number of blocked locks should reduce by 1 and + // count of running locks should increase by 1. + verifyLockState(expectedWLockStats, t, 3) + // release the write lock. + nsMutex.Unlock("my-bucket", "my-object", strconv.Itoa(10)) + // The number of running locks should decrease by 1. + // expectedWLockStats = expectedResult[3] + // verifyLockState(expectedWLockStats, t, 4) + // Take the lock stats after the first write lock is unlocked. + // Only then unlock then second write lock. + syncChan <- struct{}{} + }() + // waiting so that the write locks in the above go routines are held. + // sleeping so that we can predict the order of the write locks held. + time.Sleep(100 * time.Millisecond) + + // since there are 5 more readlocks still held on <"my-bucket","my-object">, + // an attempt to hold write locks blocks. So its run in a new go routine. + wg.Add(1) + go func() { + defer wg.Done() + // blocks till all read locks are released. + nsMutex.Lock("my-bucket", "my-object", strconv.Itoa(11)) + // Once the above attempt to lock is unblocked/acquired, we release the lock. + // Unlock the second write lock only after lock stats for first write lock release is taken. + <-syncChan + // The number of running locks should decrease by 1. + expectedWLockStats := expectedResult[4] + verifyLockState(expectedWLockStats, t, 5) + nsMutex.Unlock("my-bucket", "my-object", strconv.Itoa(11)) + }() + + expectedLockStats = expectedResult[5] + + time.Sleep(1 * time.Second) + // verify the actual lock info with the expected one. + verifyLockState(expectedLockStats, t, 6) + + // unlock 4 out of remaining 5 read locks. + for i := 0; i < 4; i++ { + nsMutex.RUnlock("my-bucket", "my-object", strconv.Itoa(i+5)) + } + + // verify the entry for one remaining read lock and count of blocked write locks. + expectedLockStats = expectedResult[6] + // verify the actual lock info with the expected one. + verifyLockState(expectedLockStats, t, 7) + + // Releasing the last read lock. + nsMutex.RUnlock("my-bucket", "my-object", strconv.Itoa(9)) + wg.Wait() + expectedLockStats = expectedResult[7] + // verify the actual lock info with the expected one. + verifyGlobalLockStats(expectedLockStats, t, 8) + +} diff --git a/cmd/naughty-disk_test.go b/cmd/naughty-disk_test.go index 088c40a4f..8060d0de0 100644 --- a/cmd/naughty-disk_test.go +++ b/cmd/naughty-disk_test.go @@ -16,6 +16,8 @@ package cmd +import "github.com/minio/minio/pkg/disk" + // naughtyDisk wraps a POSIX disk and returns programmed errors // specified by the developer. The purpose is to simulate errors // that are hard to simulate in practise like DiskNotFound. @@ -46,6 +48,13 @@ func (d *naughtyDisk) calcError() (err error) { return nil } +func (d *naughtyDisk) DiskInfo() (info disk.Info, err error) { + if err := d.calcError(); err != nil { + return info, err + } + return d.disk.DiskInfo() +} + func (d *naughtyDisk) MakeVol(volume string) (err error) { if err := d.calcError(); err != nil { return err diff --git a/cmd/net-rpc-client.go b/cmd/net-rpc-client.go new file mode 100644 index 000000000..14ff2d2f1 --- /dev/null +++ b/cmd/net-rpc-client.go @@ -0,0 +1,125 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "errors" + "net/rpc" + "sync" +) + +// RPCClient is a wrapper type for rpc.Client which provides reconnect on first failure. +type RPCClient struct { + mu sync.Mutex + rpcPrivate *rpc.Client + node string + rpcPath string +} + +// newClient constructs a RPCClient object with node and rpcPath initialized. +// It _doesn't_ connect to the remote endpoint. See Call method to see when the +// connect happens. +func newClient(node, rpcPath string) *RPCClient { + return &RPCClient{ + node: node, + rpcPath: rpcPath, + } +} + +// clearRPCClient clears the pointer to the rpc.Client object in a safe manner +func (rpcClient *RPCClient) clearRPCClient() { + rpcClient.mu.Lock() + rpcClient.rpcPrivate = nil + rpcClient.mu.Unlock() +} + +// getRPCClient gets the pointer to the rpc.Client object in a safe manner +func (rpcClient *RPCClient) getRPCClient() *rpc.Client { + rpcClient.mu.Lock() + rpcLocalStack := rpcClient.rpcPrivate + rpcClient.mu.Unlock() + return rpcLocalStack +} + +// dialRPCClient tries to establish a connection to the server in a safe manner +func (rpcClient *RPCClient) dialRPCClient() (*rpc.Client, error) { + rpcClient.mu.Lock() + defer rpcClient.mu.Unlock() + // After acquiring lock, check whether another thread may not have already dialed and established connection + if rpcClient.rpcPrivate != nil { + return rpcClient.rpcPrivate, nil + } + rpc, err := rpc.DialHTTPPath("tcp", rpcClient.node, rpcClient.rpcPath) + if err != nil { + return nil, err + } else if rpc == nil { + return nil, errors.New("No valid RPC Client created after dial") + } + rpcClient.rpcPrivate = rpc + return rpcClient.rpcPrivate, nil +} + +// Call makes a RPC call to the remote endpoint using the default codec, namely encoding/gob. +func (rpcClient *RPCClient) Call(serviceMethod string, args interface{}, reply interface{}) error { + // Make a copy below so that we can safely (continue to) work with the rpc.Client. + // Even in the case the two threads would simultaneously find that the connection is not initialised, + // they would both attempt to dial and only one of them would succeed in doing so. + rpcLocalStack := rpcClient.getRPCClient() + + // If the rpc.Client is nil, we attempt to (re)connect with the remote endpoint. + if rpcLocalStack == nil { + var err error + rpcLocalStack, err = rpcClient.dialRPCClient() + if err != nil { + return err + } + } + + // If the RPC fails due to a network-related error, then we reset + // rpc.Client for a subsequent reconnect. + err := rpcLocalStack.Call(serviceMethod, args, reply) + if err != nil { + if err.Error() == rpc.ErrShutdown.Error() { + // Reset rpcClient.rpc to nil to trigger a reconnect in future + // and close the underlying connection. + rpcClient.clearRPCClient() + + // Close the underlying connection. + rpcLocalStack.Close() + + // Set rpc error as rpc.ErrShutdown type. + err = rpc.ErrShutdown + } + } + return err +} + +// Close closes the underlying socket file descriptor. +func (rpcClient *RPCClient) Close() error { + // See comment above for making a copy on local stack + rpcLocalStack := rpcClient.getRPCClient() + + // If rpc client has not connected yet there is nothing to close. + if rpcLocalStack == nil { + return nil + } + + // Reset rpcClient.rpc to allow for subsequent calls to use a new + // (socket) connection. + rpcClient.clearRPCClient() + return rpcLocalStack.Close() +} diff --git a/cmd/object-api-getobjectinfo_test.go b/cmd/object-api-getobjectinfo_test.go index b65a1190d..ced333ff8 100644 --- a/cmd/object-api-getobjectinfo_test.go +++ b/cmd/object-api-getobjectinfo_test.go @@ -57,19 +57,19 @@ func testGetObjectInfo(obj ObjectLayer, instanceType string, t TestErrHandler) { {"Test", "", ObjectInfo{}, BucketNameInvalid{Bucket: "Test"}, false}, {"---", "", ObjectInfo{}, BucketNameInvalid{Bucket: "---"}, false}, {"ad", "", ObjectInfo{}, BucketNameInvalid{Bucket: "ad"}, false}, - // Test cases with valid but non-existing bucket names (Test number 5-7). + // Test cases with valid but non-existing bucket names (Test number 5-6). {"abcdefgh", "abc", ObjectInfo{}, BucketNotFound{Bucket: "abcdefgh"}, false}, {"ijklmnop", "efg", ObjectInfo{}, BucketNotFound{Bucket: "ijklmnop"}, false}, - // Test cases with valid but non-existing bucket names and invalid object name (Test number 8-9). + // Test cases with valid but non-existing bucket names and invalid object name (Test number 7-8). {"test-getobjectinfo", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "test-getobjectinfo", Object: ""}, false}, {"test-getobjectinfo", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "test-getobjectinfo", Object: ""}, false}, - // Test cases with non-existing object name with existing bucket (Test number 10-12). + // Test cases with non-existing object name with existing bucket (Test number 9-11). {"test-getobjectinfo", "Africa", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Africa"}, false}, {"test-getobjectinfo", "Antartica", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Antartica"}, false}, {"test-getobjectinfo", "Asia/myfile", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia/myfile"}, false}, - // Test case with existing bucket but object name set to a directory (Test number 13). + // Test case with existing bucket but object name set to a directory (Test number 12). {"test-getobjectinfo", "Asia", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia"}, false}, - // Valid case with existing object (Test number 14). + // Valid case with existing object (Test number 13). {"test-getobjectinfo", "Asia/asiapics.jpg", resultCases[0], nil, true}, } for i, testCase := range testCases { diff --git a/cmd/object-api-multipart_test.go b/cmd/object-api-multipart_test.go index c2b7fe8d8..0014d89bc 100644 --- a/cmd/object-api-multipart_test.go +++ b/cmd/object-api-multipart_test.go @@ -38,7 +38,7 @@ func testObjectNewMultipartUpload(obj ObjectLayer, instanceType string, t TestEr errMsg := "Bucket not found: minio-bucket" // opearation expected to fail since the bucket on which NewMultipartUpload is being initiated doesn't exist. - uploadID, err := obj.NewMultipartUpload(bucket, object, nil) + _, err := obj.NewMultipartUpload(bucket, object, nil) if err == nil { t.Fatalf("%s: Expected to fail since the NewMultipartUpload is intialized on a non-existent bucket.", instanceType) } @@ -53,7 +53,7 @@ func testObjectNewMultipartUpload(obj ObjectLayer, instanceType string, t TestEr t.Fatalf("%s : %s", instanceType, err.Error()) } - uploadID, err = obj.NewMultipartUpload(bucket, object, nil) + uploadID, err := obj.NewMultipartUpload(bucket, object, nil) if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } @@ -92,6 +92,7 @@ func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t TestE } err = obj.AbortMultipartUpload(bucket, object, "abc") + err = errorCause(err) switch err.(type) { case InvalidUploadID: default: diff --git a/cmd/object-api-putobject_test.go b/cmd/object-api-putobject_test.go index 11e90aa06..31f83e8f4 100644 --- a/cmd/object-api-putobject_test.go +++ b/cmd/object-api-putobject_test.go @@ -151,7 +151,8 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl } for i, testCase := range testCases { - actualMd5Hex, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta) + objInfo, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta) + actualErr = errorCause(actualErr) if actualErr != nil && testCase.expectedError == nil { t.Errorf("Test %d: %s: Expected to pass, but failed with: error %s.", i+1, instanceType, actualErr.Error()) } @@ -159,14 +160,14 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but passed instead.", i+1, instanceType, testCase.expectedError.Error()) } // Failed as expected, but does it fail for the expected reason. - if actualErr != nil && testCase.expectedError != actualErr { + if actualErr != nil && actualErr != testCase.expectedError { t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, instanceType, testCase.expectedError.Error(), actualErr.Error()) } // Test passes as expected, but the output values are verified for correctness here. if actualErr == nil { // Asserting whether the md5 output is correct. - if expectedMD5, ok := testCase.inputMeta["md5Sum"]; ok && expectedMD5 != actualMd5Hex { - t.Errorf("Test %d: %s: Calculated Md5 different from the actual one %s.", i+1, instanceType, actualMd5Hex) + if expectedMD5, ok := testCase.inputMeta["md5Sum"]; ok && expectedMD5 != objInfo.MD5Sum { + t.Errorf("Test %d: %s: Calculated Md5 different from the actual one %s.", i+1, instanceType, objInfo.MD5Sum) } } } @@ -223,7 +224,8 @@ func testObjectAPIPutObjectDiskNotFOund(obj ObjectLayer, instanceType string, di } for i, testCase := range testCases { - actualMd5Hex, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta) + objInfo, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta) + actualErr = errorCause(err) if actualErr != nil && testCase.shouldPass { t.Errorf("Test %d: %s: Expected to pass, but failed with: %s.", i+1, instanceType, actualErr.Error()) } @@ -240,8 +242,8 @@ func testObjectAPIPutObjectDiskNotFOund(obj ObjectLayer, instanceType string, di // Test passes as expected, but the output values are verified for correctness here. if actualErr == nil && testCase.shouldPass { // Asserting whether the md5 output is correct. - if testCase.inputMeta["md5Sum"] != actualMd5Hex { - t.Errorf("Test %d: %s: Calculated Md5 different from the actual one %s.", i+1, instanceType, actualMd5Hex) + if testCase.inputMeta["md5Sum"] != objInfo.MD5Sum { + t.Errorf("Test %d: %s: Calculated Md5 different from the actual one %s.", i+1, instanceType, objInfo.MD5Sum) } } } @@ -272,6 +274,7 @@ func testObjectAPIPutObjectDiskNotFOund(obj ObjectLayer, instanceType string, di InsufficientWriteQuorum{}, } _, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta) + actualErr = errorCause(actualErr) if actualErr != nil && testCase.shouldPass { t.Errorf("Test %d: %s: Expected to pass, but failed with: %s.", len(testCases)+1, instanceType, actualErr.Error()) } diff --git a/cmd/object-common.go b/cmd/object-common.go index d9395f1b3..c6243842f 100644 --- a/cmd/object-common.go +++ b/cmd/object-common.go @@ -17,7 +17,7 @@ package cmd import ( - "path/filepath" + "net" "strings" "sync" ) @@ -35,6 +35,7 @@ const ( // isErrIgnored should we ignore this error?, takes a list of errors which can be ignored. func isErrIgnored(err error, ignoredErrs []error) bool { + err = errorCause(err) for _, ignoredErr := range ignoredErrs { if ignoredErr == err { return true @@ -53,13 +54,62 @@ func fsHouseKeeping(storageDisk StorageAPI) error { return nil } +// Check if a network path is local to this node. +func isLocalStorage(networkPath string) bool { + if idx := strings.LastIndex(networkPath, ":"); idx != -1 { + // e.g 10.0.0.1:/mnt/networkPath + netAddr, _, err := splitNetPath(networkPath) + if err != nil { + errorIf(err, "Splitting into ip and path failed") + return false + } + // netAddr will only be set if this is not a local path. + if netAddr == "" { + return true + } + // Resolve host to address to check if the IP is loopback. + // If address resolution fails, assume it's a non-local host. + addrs, err := net.LookupHost(netAddr) + if err != nil { + errorIf(err, "Failed to lookup host") + return false + } + for _, addr := range addrs { + if ip := net.ParseIP(addr); ip.IsLoopback() { + return true + } + } + iaddrs, err := net.InterfaceAddrs() + if err != nil { + errorIf(err, "Unable to list interface addresses") + return false + } + for _, addr := range addrs { + for _, iaddr := range iaddrs { + ip, _, err := net.ParseCIDR(iaddr.String()) + if err != nil { + errorIf(err, "Unable to parse CIDR") + return false + } + if ip.String() == addr { + return true + } + + } + } + return false + } + return true +} + // Depending on the disk type network or local, initialize storage API. func newStorageAPI(disk string) (storage StorageAPI, err error) { - if !strings.ContainsRune(disk, ':') || filepath.VolumeName(disk) != "" { - // Initialize filesystem storage API. + if isLocalStorage(disk) { + if idx := strings.LastIndex(disk, ":"); idx != -1 { + return newPosix(disk[idx+1:]) + } return newPosix(disk) } - // Initialize rpc client storage API. return newRPCClient(disk) } @@ -84,7 +134,7 @@ func initMetaVolume(storageDisks []StorageAPI) error { // Indicate this wait group is done. defer wg.Done() - // Attempt to create `.minio`. + // Attempt to create `.minio.sys`. err := disk.MakeVol(minioMetaBucket) if err != nil { switch err { @@ -135,7 +185,11 @@ func xlHouseKeeping(storageDisks []StorageAPI) error { // Cleanup all temp entries upon start. err := cleanupDir(disk, minioMetaBucket, tmpMetaPrefix) if err != nil { - errs[index] = err + switch errorCause(err) { + case errDiskNotFound, errVolumeNotFound: + default: + errs[index] = err + } } }(index, disk) } @@ -171,7 +225,7 @@ func cleanupDir(storage StorageAPI, volume, dirPath string) error { if err == errFileNotFound { return nil } else if err != nil { // For any other errors fail. - return err + return traceError(err) } // else on success.. // Recurse and delete all other entries. diff --git a/cmd/object-errors.go b/cmd/object-errors.go index a080a52e0..e9ef85753 100644 --- a/cmd/object-errors.go +++ b/cmd/object-errors.go @@ -26,48 +26,57 @@ import ( // handle all cases where we have known types of errors returned by // underlying storage layer. func toObjectErr(err error, params ...string) error { + e, ok := err.(*Error) + if ok { + err = e.e + } + switch err { case errVolumeNotFound: if len(params) >= 1 { - return BucketNotFound{Bucket: params[0]} + err = BucketNotFound{Bucket: params[0]} } case errVolumeNotEmpty: if len(params) >= 1 { - return BucketNotEmpty{Bucket: params[0]} + err = BucketNotEmpty{Bucket: params[0]} } case errVolumeExists: if len(params) >= 1 { - return BucketExists{Bucket: params[0]} + err = BucketExists{Bucket: params[0]} } case errDiskFull: - return StorageFull{} + err = StorageFull{} case errIsNotRegular, errFileAccessDenied: if len(params) >= 2 { - return ObjectExistsAsDirectory{ + err = ObjectExistsAsDirectory{ Bucket: params[0], Object: params[1], } } case errFileNotFound: if len(params) >= 2 { - return ObjectNotFound{ + err = ObjectNotFound{ Bucket: params[0], Object: params[1], } } case errFileNameTooLong: if len(params) >= 2 { - return ObjectNameInvalid{ + err = ObjectNameInvalid{ Bucket: params[0], Object: params[1], } } case errXLReadQuorum: - return InsufficientReadQuorum{} + err = InsufficientReadQuorum{} case errXLWriteQuorum: - return InsufficientWriteQuorum{} + err = InsufficientWriteQuorum{} case io.ErrUnexpectedEOF, io.ErrShortWrite: - return IncompleteBody{} + err = IncompleteBody{} + } + if ok { + e.e = err + return e } return err } diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 90d0702f4..a9f12b983 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -27,10 +27,18 @@ import ( "sort" "strconv" "strings" + "sync" mux "github.com/gorilla/mux" ) +var objLayerMutex *sync.Mutex +var globalObjectAPI ObjectLayer + +func init() { + objLayerMutex = &sync.Mutex{} +} + // supportedGetReqParams - supported request parameters for GET presigned request. var supportedGetReqParams = map[string]string{ "response-expires": "Expires", @@ -84,6 +92,13 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req bucket = vars["bucket"] object = vars["object"] + // Fetch object stat info. + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + switch getRequestAuthType(r) { default: // For all unknown auth types return error. @@ -101,8 +116,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req return } } - // Fetch object stat info. - objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object) + objInfo, err := objectAPI.GetObjectInfo(bucket, object) if err != nil { errorIf(err, "Unable to fetch object info.") apiErr := toAPIErrorCode(err) @@ -161,7 +175,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req }) // Reads the object at startOffset and writes to mw. - if err := api.ObjectAPI.GetObject(bucket, object, startOffset, length, writer); err != nil { + if err := objectAPI.GetObject(bucket, object, startOffset, length, writer); err != nil { errorIf(err, "Unable to write to client.") if !dataWritten { // Error response only if no data has been written to client yet. i.e if @@ -190,6 +204,12 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re bucket = vars["bucket"] object = vars["object"] + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + switch getRequestAuthType(r) { default: // For all unknown auth types return error. @@ -208,7 +228,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re } } - objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object) + objInfo, err := objectAPI.GetObjectInfo(bucket, object) if err != nil { errorIf(err, "Unable to fetch object info.") apiErr := toAPIErrorCode(err) @@ -240,6 +260,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re bucket := vars["bucket"] object := vars["object"] + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + switch getRequestAuthType(r) { default: // For all unknown auth types return error. @@ -289,7 +315,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re return } - objInfo, err := api.ObjectAPI.GetObjectInfo(sourceBucket, sourceObject) + objInfo, err := objectAPI.GetObjectInfo(sourceBucket, sourceObject) if err != nil { errorIf(err, "Unable to fetch object info.") writeErrorResponse(w, r, toAPIErrorCode(err), objectSource) @@ -307,11 +333,14 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re return } + // Size of object. + size := objInfo.Size + pipeReader, pipeWriter := io.Pipe() go func() { startOffset := int64(0) // Read the whole file. // Get the object. - gErr := api.ObjectAPI.GetObject(sourceBucket, sourceObject, startOffset, objInfo.Size, pipeWriter) + gErr := objectAPI.GetObject(sourceBucket, sourceObject, startOffset, size, pipeWriter) if gErr != nil { errorIf(gErr, "Unable to read an object.") pipeWriter.CloseWithError(gErr) @@ -320,19 +349,14 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re pipeWriter.Close() // Close. }() - // Size of object. - size := objInfo.Size - - // Save metadata. - metadata := make(map[string]string) // Save other metadata if available. - metadata = objInfo.UserDefined + metadata := objInfo.UserDefined // Do not set `md5sum` as CopyObject will not keep the // same md5sum as the source. // Create the object. - md5Sum, err := api.ObjectAPI.PutObject(bucket, object, size, pipeReader, metadata) + objInfo, err = objectAPI.PutObject(bucket, object, size, pipeReader, metadata) if err != nil { // Close the this end of the pipe upon error in PutObject. pipeReader.CloseWithError(err) @@ -343,13 +367,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re // Explicitly close the reader, before fetching object info. pipeReader.Close() - objInfo, err = api.ObjectAPI.GetObjectInfo(bucket, object) - if err != nil { - errorIf(err, "Unable to fetch object info.") - writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) - return - } - + md5Sum := objInfo.MD5Sum response := generateCopyObjectResponse(md5Sum, objInfo.ModTime) encodedSuccessResponse := encodeResponse(response) // write headers @@ -374,6 +392,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re // ---------- // This implementation of the PUT operation adds an object to a bucket. func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Request) { + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + // If the matching failed, it means that the X-Amz-Copy-Source was // wrong, fail right here. if _, ok := r.Header["X-Amz-Copy-Source"]; ok { @@ -418,7 +442,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req // Make sure we hex encode md5sum here. metadata["md5Sum"] = hex.EncodeToString(md5Bytes) - var md5Sum string + var objInfo ObjectInfo switch rAuthType { default: // For all unknown auth types return error. @@ -431,7 +455,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req return } // Create anonymous object. - md5Sum, err = api.ObjectAPI.PutObject(bucket, object, size, r.Body, metadata) + objInfo, err = objectAPI.PutObject(bucket, object, size, r.Body, metadata) case authTypeStreamingSigned: // Initialize stream signature verifier. reader, s3Error := newSignV4ChunkedReader(r) @@ -439,31 +463,22 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req writeErrorResponse(w, r, s3Error, r.URL.Path) return } - md5Sum, err = api.ObjectAPI.PutObject(bucket, object, size, reader, metadata) + objInfo, err = objectAPI.PutObject(bucket, object, size, reader, metadata) case authTypePresigned, authTypeSigned: // Initialize signature verifier. reader := newSignVerify(r) // Create object. - md5Sum, err = api.ObjectAPI.PutObject(bucket, object, size, reader, metadata) + objInfo, err = objectAPI.PutObject(bucket, object, size, reader, metadata) } if err != nil { errorIf(err, "Unable to create an object.") writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) return } - if md5Sum != "" { - w.Header().Set("ETag", "\""+md5Sum+"\"") - } + w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"") writeSuccessResponse(w, nil) if globalEventNotifier.IsBucketNotificationSet(bucket) { - // Fetch object info for notifications. - objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object) - if err != nil { - errorIf(err, "Unable to fetch object info for \"%s\"", path.Join(bucket, object)) - return - } - // Notify object created event. eventNotify(eventData{ Type: ObjectCreatedPut, @@ -485,6 +500,12 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r bucket = vars["bucket"] object = vars["object"] + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + switch getRequestAuthType(r) { default: // For all unknown auth types return error. @@ -506,7 +527,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r // Extract metadata that needs to be saved. metadata := extractMetadataFromHeader(r.Header) - uploadID, err := api.ObjectAPI.NewMultipartUpload(bucket, object, metadata) + uploadID, err := objectAPI.NewMultipartUpload(bucket, object, metadata) if err != nil { errorIf(err, "Unable to initiate new multipart upload id.") writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) @@ -527,6 +548,12 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http bucket := vars["bucket"] object := vars["object"] + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + // get Content-Md5 sent by client and verify if valid md5Bytes, err := checkValidMD5(r.Header.Get("Content-Md5")) if err != nil { @@ -588,7 +615,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http return } // No need to verify signature, anonymous request access is already allowed. - partMD5, err = api.ObjectAPI.PutObjectPart(bucket, object, uploadID, partID, size, r.Body, incomingMD5) + partMD5, err = objectAPI.PutObjectPart(bucket, object, uploadID, partID, size, r.Body, incomingMD5) case authTypeStreamingSigned: // Initialize stream signature verifier. reader, s3Error := newSignV4ChunkedReader(r) @@ -596,11 +623,11 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http writeErrorResponse(w, r, s3Error, r.URL.Path) return } - partMD5, err = api.ObjectAPI.PutObjectPart(bucket, object, uploadID, partID, size, reader, incomingMD5) + partMD5, err = objectAPI.PutObjectPart(bucket, object, uploadID, partID, size, reader, incomingMD5) case authTypePresigned, authTypeSigned: // Initialize signature verifier. reader := newSignVerify(r) - partMD5, err = api.ObjectAPI.PutObjectPart(bucket, object, uploadID, partID, size, reader, incomingMD5) + partMD5, err = objectAPI.PutObjectPart(bucket, object, uploadID, partID, size, reader, incomingMD5) } if err != nil { errorIf(err, "Unable to create object part.") @@ -620,6 +647,12 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, bucket := vars["bucket"] object := vars["object"] + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + switch getRequestAuthType(r) { default: // For all unknown auth types return error. @@ -639,7 +672,7 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, } uploadID, _, _, _ := getObjectResources(r.URL.Query()) - if err := api.ObjectAPI.AbortMultipartUpload(bucket, object, uploadID); err != nil { + if err := objectAPI.AbortMultipartUpload(bucket, object, uploadID); err != nil { errorIf(err, "Unable to abort multipart upload.") writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) return @@ -653,6 +686,12 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht bucket := vars["bucket"] object := vars["object"] + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + switch getRequestAuthType(r) { default: // For all unknown auth types return error. @@ -680,7 +719,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht writeErrorResponse(w, r, ErrInvalidMaxParts, r.URL.Path) return } - listPartsInfo, err := api.ObjectAPI.ListObjectParts(bucket, object, uploadID, partNumberMarker, maxParts) + listPartsInfo, err := objectAPI.ListObjectParts(bucket, object, uploadID, partNumberMarker, maxParts) if err != nil { errorIf(err, "Unable to list uploaded parts.") writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path) @@ -700,6 +739,12 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite bucket := vars["bucket"] object := vars["object"] + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + // Get upload id. uploadID, _, _, _ := getObjectResources(r.URL.Query()) @@ -762,9 +807,10 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite } doneCh := make(chan struct{}) + // Signal that completeMultipartUpload is over via doneCh go func(doneCh chan<- struct{}) { - md5Sum, err = api.ObjectAPI.CompleteMultipartUpload(bucket, object, uploadID, completeParts) + md5Sum, err = objectAPI.CompleteMultipartUpload(bucket, object, uploadID, completeParts) doneCh <- struct{}{} }(doneCh) @@ -799,7 +845,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite if globalEventNotifier.IsBucketNotificationSet(bucket) { // Fetch object info for notifications. - objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object) + objInfo, err := objectAPI.GetObjectInfo(bucket, object) if err != nil { errorIf(err, "Unable to fetch object info for \"%s\"", path.Join(bucket, object)) return @@ -825,6 +871,12 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http. bucket := vars["bucket"] object := vars["object"] + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path) + return + } + switch getRequestAuthType(r) { default: // For all unknown auth types return error. @@ -845,7 +897,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http. /// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html /// Ignore delete object errors, since we are suppposed to reply /// only 204. - if err := api.ObjectAPI.DeleteObject(bucket, object); err != nil { + if err := objectAPI.DeleteObject(bucket, object); err != nil { writeSuccessNoContent(w) return } diff --git a/cmd/object-handlers_test.go b/cmd/object-handlers_test.go index 5e1676791..f00336ad6 100644 --- a/cmd/object-handlers_test.go +++ b/cmd/object-handlers_test.go @@ -27,34 +27,12 @@ import ( // Wrapper for calling GetObject API handler tests for both XL multiple disks and FS single drive setup. func TestAPIGetOjectHandler(t *testing.T) { - ExecObjectLayerTest(t, testAPIGetOjectHandler) + ExecObjectLayerAPITest(t, testAPIGetOjectHandler, []string{"GetObject"}) } -func testAPIGetOjectHandler(obj ObjectLayer, instanceType string, t TestErrHandler) { - - // get random bucket name. - bucketName := getRandomBucketName() +func testAPIGetOjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials credential, t TestErrHandler) { objectName := "test-object" - // Create bucket. - err := obj.MakeBucket(bucketName) - if err != nil { - // failed to create newbucket, abort. - t.Fatalf("%s : %s", instanceType, err) - } - // Register the API end points with XL/FS object layer. - // Registering only the GetObject handler. - apiRouter := initTestAPIEndPoints(obj, []string{"GetObject"}) - // initialize the server and obtain the credentials and root. - // credentials are necessary to sign the HTTP request. - rootPath, err := newTestConfig("us-east-1") - if err != nil { - t.Fatalf("Init Test config failed") - } - // remove the root folder after the test ends. - defer removeAll(rootPath) - - credentials := serverConfig.GetCredential() - // set of byte data for PutObject. // object has to be inserted before running tests for GetObject. // this is required even to assert the GetObject data, @@ -78,7 +56,7 @@ func testAPIGetOjectHandler(obj ObjectLayer, instanceType string, t TestErrHandl // iterate through the above set of inputs and upload the object. for i, input := range putObjectInputs { // uploading the object. - _, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData) + _, err := obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData) // if object upload fails stop the test. if err != nil { t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) @@ -174,40 +152,168 @@ func testAPIGetOjectHandler(obj ObjectLayer, instanceType string, t TestErrHandl } } -// Wrapper for calling Copy Object API handler tests for both XL multiple disks and single node setup. -func TestAPICopyObjectHandler(t *testing.T) { - ExecObjectLayerTest(t, testAPICopyObjectHandler) +// Wrapper for calling PutObject API handler tests using streaming signature v4 for both XL multiple disks and FS single drive setup. +func TestAPIPutObjectStreamSigV4Handler(t *testing.T) { + ExecObjectLayerAPITest(t, testAPIPutObjectStreamSigV4Handler, []string{"PutObject"}) } -func testAPICopyObjectHandler(obj ObjectLayer, instanceType string, t TestErrHandler) { - // get random bucket name. - bucketName := getRandomBucketName() - objectName := "test-object" - // Create bucket. - err := obj.MakeBucket(bucketName) - if err != nil { - // failed to create newbucket, abort. - t.Fatalf("%s : %s", instanceType, err) - } - // Register the API end points with XL/FS object layer. - // Registering only the Copy Object handler. - apiRouter := initTestAPIEndPoints(obj, []string{"CopyObject"}) - // initialize the server and obtain the credentials and root. - // credentials are necessary to sign the HTTP request. - rootPath, err := newTestConfig("us-east-1") - if err != nil { - t.Fatalf("Init Test config failed") - } - // remove the root folder after the test ends. - defer removeAll(rootPath) +func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials credential, t TestErrHandler) { - err = initEventNotifier(obj) + objectName := "test-object" + bytesDataLen := 65 * 1024 + bytesData := bytes.Repeat([]byte{'a'}, bytesDataLen) + + // byte data for PutObject. + // test cases with inputs and expected result for GetObject. + testCases := []struct { + bucketName string + objectName string + data []byte + dataLen int + // expected output. + expectedContent []byte // expected response body. + expectedRespStatus int // expected response status body. + }{ + // Test case - 1. + // Fetching the entire object and validating its contents. + { + bucketName: bucketName, + objectName: objectName, + data: bytesData, + dataLen: len(bytesData), + expectedContent: []byte{}, + expectedRespStatus: http.StatusOK, + }, + } + // Iterating over the cases, fetching the object validating the response. + for i, testCase := range testCases { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + // construct HTTP request for Put Object end point. + req, err := newTestStreamingSignedRequest("PUT", + getPutObjectURL("", testCase.bucketName, testCase.objectName), + int64(testCase.dataLen), 64*1024, bytes.NewReader(testCase.data), + credentials.AccessKeyID, credentials.SecretAccessKey) + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for Put Object: %v", i+1, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler,`func (api objectAPIHandlers) GetObjectHandler` handles the request. + apiRouter.ServeHTTP(rec, req) + // Assert the response code with the expected status. + if rec.Code != testCase.expectedRespStatus { + t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, rec.Code) + } + // read the response body. + actualContent, err := ioutil.ReadAll(rec.Body) + if err != nil { + t.Fatalf("Test %d: %s: Failed parsing response body: %v", i+1, instanceType, err) + } + // Verify whether the bucket obtained object is same as the one inserted. + if !bytes.Equal(testCase.expectedContent, actualContent) { + t.Errorf("Test %d: %s: Object content differs from expected value.: %s", i+1, instanceType, string(actualContent)) + } + + buffer := new(bytes.Buffer) + err = obj.GetObject(testCase.bucketName, testCase.objectName, 0, int64(bytesDataLen), buffer) + if err != nil { + t.Fatalf("Test %d: %s: Failed to fetch the copied object: %s", i+1, instanceType, err) + } + if !bytes.Equal(bytesData, buffer.Bytes()) { + t.Errorf("Test %d: %s: Data Mismatch: Data fetched back from the uploaded object doesn't match the original one.", i+1, instanceType) + } + buffer.Reset() + } +} + +// Wrapper for calling PutObject API handler tests for both XL multiple disks and FS single drive setup. +func TestAPIPutObjectHandler(t *testing.T) { + ExecObjectLayerAPITest(t, testAPIPutObjectHandler, []string{"PutObject"}) +} + +func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials credential, t TestErrHandler) { + + objectName := "test-object" + // byte data for PutObject. + bytesData := generateBytesData(6 * 1024 * 1024) + + // test cases with inputs and expected result for GetObject. + testCases := []struct { + bucketName string + objectName string + data []byte + dataLen int + // expected output. + expectedContent []byte // expected response body. + expectedRespStatus int // expected response status body. + }{ + // Test case - 1. + // Fetching the entire object and validating its contents. + { + bucketName: bucketName, + objectName: objectName, + data: bytesData, + dataLen: len(bytesData), + expectedContent: []byte{}, + expectedRespStatus: http.StatusOK, + }, + } + // Iterating over the cases, fetching the object validating the response. + for i, testCase := range testCases { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + // construct HTTP request for Get Object end point. + req, err := newTestSignedRequest("PUT", getPutObjectURL("", testCase.bucketName, testCase.objectName), + int64(testCase.dataLen), bytes.NewReader(testCase.data), credentials.AccessKeyID, credentials.SecretAccessKey) + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for Put Object: %v", i+1, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler,`func (api objectAPIHandlers) GetObjectHandler` handles the request. + apiRouter.ServeHTTP(rec, req) + // Assert the response code with the expected status. + if rec.Code != testCase.expectedRespStatus { + t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, rec.Code) + } + // read the response body. + actualContent, err := ioutil.ReadAll(rec.Body) + if err != nil { + t.Fatalf("Test %d: %s: Failed parsing response body: %v", i+1, instanceType, err) + } + // Verify whether the bucket obtained object is same as the one inserted. + if !bytes.Equal(testCase.expectedContent, actualContent) { + t.Errorf("Test %d: %s: Object content differs from expected value.: %s", i+1, instanceType, string(actualContent)) + } + + buffer := new(bytes.Buffer) + err = obj.GetObject(testCase.bucketName, testCase.objectName, 0, int64(len(bytesData)), buffer) + if err != nil { + t.Fatalf("Test %d: %s: Failed to fetch the copied object: %s", i+1, instanceType, err) + } + if !bytes.Equal(bytesData, buffer.Bytes()) { + t.Errorf("Test %d: %s: Data Mismatch: Data fetched back from the uploaded object doesn't match the original one.", i+1, instanceType) + } + buffer.Reset() + } +} + +// Wrapper for calling Copy Object API handler tests for both XL multiple disks and single node setup. +func TestAPICopyObjectHandler(t *testing.T) { + ExecObjectLayerAPITest(t, testAPICopyObjectHandler, []string{"CopyObject"}) +} + +func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials credential, t TestErrHandler) { + + objectName := "test-object" + // register event notifier. + err := initEventNotifier(obj) if err != nil { t.Fatalf("Initializing event notifiers failed") } - credentials := serverConfig.GetCredential() - // set of byte data for PutObject. // object has to be inserted before running tests for Copy Object. // this is required even to assert the copied object, diff --git a/cmd/object-interface.go b/cmd/object-interface.go index 8b868c9cb..943357667 100644 --- a/cmd/object-interface.go +++ b/cmd/object-interface.go @@ -22,6 +22,7 @@ import "io" type ObjectLayer interface { // Storage operations. Shutdown() error + HealDiskMetadata() error StorageInfo() StorageInfo // Bucket operations. @@ -35,7 +36,7 @@ type ObjectLayer interface { // Object operations. GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) GetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) - PutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string) (md5 string, err error) + PutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string) (objInto ObjectInfo, err error) DeleteObject(bucket, object string) error HealObject(bucket, object string) error diff --git a/cmd/object-multipart-common.go b/cmd/object-multipart-common.go index 482275f33..689a21dae 100644 --- a/cmd/object-multipart-common.go +++ b/cmd/object-multipart-common.go @@ -72,12 +72,12 @@ func readUploadsJSON(bucket, object string, disk StorageAPI) (uploadIDs uploadsV // Reads entire `uploads.json`. buf, err := disk.ReadAll(minioMetaBucket, uploadJSONPath) if err != nil { - return uploadsV1{}, err + return uploadsV1{}, traceError(err) } // Decode `uploads.json`. if err = json.Unmarshal(buf, &uploadIDs); err != nil { - return uploadsV1{}, err + return uploadsV1{}, traceError(err) } // Success. @@ -103,7 +103,7 @@ func cleanupUploadedParts(bucket, object, uploadID string, storageDisks ...Stora // Cleanup uploadID for all disks. for index, disk := range storageDisks { if disk == nil { - errs[index] = errDiskNotFound + errs[index] = traceError(errDiskNotFound) continue } wg.Add(1) diff --git a/cmd/object-utils.go b/cmd/object-utils.go index 420099a91..36e71e0c3 100644 --- a/cmd/object-utils.go +++ b/cmd/object-utils.go @@ -148,7 +148,7 @@ func completeMultipartMD5(parts ...completePart) (string, error) { for _, part := range parts { md5Bytes, err := hex.DecodeString(part.ETag) if err != nil { - return "", err + return "", traceError(err) } finalMD5Bytes = append(finalMD5Bytes, md5Bytes...) } diff --git a/cmd/object_api_suite_test.go b/cmd/object_api_suite_test.go index a82bf7734..e53a2151f 100644 --- a/cmd/object_api_suite_test.go +++ b/cmd/object_api_suite_test.go @@ -200,12 +200,12 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, c TestErrH objects[key] = []byte(randomString) metadata := make(map[string]string) metadata["md5Sum"] = expectedMD5Sumhex - var md5Sum string - md5Sum, err = obj.PutObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), metadata) + var objInfo ObjectInfo + objInfo, err = obj.PutObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), metadata) if err != nil { c.Fatalf("%s: %s", instanceType, err) } - if md5Sum != expectedMD5Sumhex { + if objInfo.MD5Sum != expectedMD5Sumhex { c.Errorf("Md5 Mismatch") } } @@ -625,6 +625,9 @@ func testListBuckets(obj ObjectLayer, instanceType string, c TestErrHandler) { // add three and test exists + prefix. err = obj.MakeBucket("bucket22") + if err != nil { + c.Fatalf("%s: %s", instanceType, err) + } buckets, err = obj.ListBuckets() if err != nil { @@ -707,6 +710,7 @@ func testNonExistantObjectInBucket(obj ObjectLayer, instanceType string, c TestE if err == nil { c.Fatalf("%s: Expected error but found nil", instanceType) } + err = errorCause(err) switch err := err.(type) { case ObjectNotFound: if err.Error() != "Object not found: bucket#dir1" { @@ -740,6 +744,7 @@ func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string, } _, err = obj.GetObjectInfo("bucket", "dir1") + err = errorCause(err) switch err := err.(type) { case ObjectNotFound: if err.Bucket != "bucket" { @@ -755,6 +760,7 @@ func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string, } _, err = obj.GetObjectInfo("bucket", "dir1/") + err = errorCause(err) switch err := err.(type) { case ObjectNameInvalid: if err.Bucket != "bucket" { diff --git a/cmd/posix-errors.go b/cmd/posix-errors.go new file mode 100644 index 000000000..0fc152822 --- /dev/null +++ b/cmd/posix-errors.go @@ -0,0 +1,89 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "os" + "runtime" + "syscall" +) + +// Check if the given error corresponds to ENOTDIR (is not a directory) +func isSysErrNotDir(err error) bool { + if pathErr, ok := err.(*os.PathError); ok { + switch pathErr.Err { + case syscall.ENOTDIR: + return true + } + } + return false +} + +// Check if the given error corresponds to EISDIR (is a directory) +func isSysErrIsDir(err error) bool { + if pathErr, ok := err.(*os.PathError); ok { + switch pathErr.Err { + case syscall.EISDIR: + return true + } + } + return false +} + +// Check if the given error corresponds to ENOTEMPTY for unix +// and ERROR_DIR_NOT_EMPTY for windows (directory not empty) +func isSysErrNotEmpty(err error) bool { + if pathErr, ok := err.(*os.PathError); ok { + if runtime.GOOS == "windows" { + if errno, _ok := pathErr.Err.(syscall.Errno); _ok && errno == 0x91 { + // ERROR_DIR_NOT_EMPTY + return true + } + } + switch pathErr.Err { + case syscall.ENOTEMPTY: + return true + } + } + return false +} + +// Check if the given error corresponds to the specific ERROR_PATH_NOT_FOUND for windows +func isSysErrPathNotFound(err error) bool { + if runtime.GOOS != "windows" { + return false + } + if pathErr, ok := err.(*os.PathError); ok { + if errno, _ok := pathErr.Err.(syscall.Errno); _ok && errno == 0x03 { + // ERROR_PATH_NOT_FOUND + return true + } + } + return false +} + +// Check if the given error corresponds to the specific ERROR_INVALID_HANDLE for windows +func isSysErrHandleInvalid(err error) bool { + if runtime.GOOS != "windows" { + return false + } + // Check if err contains ERROR_INVALID_HANDLE errno + if errno, ok := err.(syscall.Errno); ok && errno == 0x6 { + return true + } + return false +} diff --git a/cmd/posix-list-dir-nix.go b/cmd/posix-list-dir-nix.go index 1023ebf43..f6af62a08 100644 --- a/cmd/posix-list-dir-nix.go +++ b/cmd/posix-list-dir-nix.go @@ -89,7 +89,6 @@ func parseDirents(dirPath string, buf []byte) (entries []string, err error) { // Could happen if it was deleted in the middle while // this list was being performed. if os.IsNotExist(err) { - err = nil continue } return nil, err diff --git a/cmd/posix-utils_windows_test.go b/cmd/posix-utils_windows_test.go index d2c3c7ca6..c304d5339 100644 --- a/cmd/posix-utils_windows_test.go +++ b/cmd/posix-utils_windows_test.go @@ -41,7 +41,9 @@ func TestUNCPaths(t *testing.T) { // Instantiate posix object to manage a disk var err error err = os.Mkdir("c:\\testdisk", 0700) - + if err != nil { + t.Fatal(err) + } // Cleanup on exit of test defer os.RemoveAll("c:\\testdisk") @@ -74,7 +76,9 @@ func TestUNCPathENOTDIR(t *testing.T) { var err error // Instantiate posix object to manage a disk err = os.Mkdir("c:\\testdisk", 0700) - + if err != nil { + t.Fatal(err) + } // Cleanup on exit of test defer os.RemoveAll("c:\\testdisk") var fs StorageAPI diff --git a/cmd/posix.go b/cmd/posix.go index 29f8c93ce..8bf5bec3b 100644 --- a/cmd/posix.go +++ b/cmd/posix.go @@ -160,6 +160,12 @@ func checkDiskFree(diskPath string, minFreeDisk int64) (err error) { return nil } +// DiskInfo provides current information about disk space usage, +// total free inodes and underlying filesystem. +func (s *posix) DiskInfo() (info disk.Info, err error) { + return getDiskInfo(s.diskPath) +} + // getVolDir - will convert incoming volume names to // corresponding valid volume names on the backend in a platform // compatible way for all operating systems. If volume is not found @@ -333,12 +339,7 @@ func (s *posix) DeleteVol(volume string) (err error) { if err != nil { if os.IsNotExist(err) { return errVolumeNotFound - } else if strings.Contains(err.Error(), "directory is not empty") { - // On windows the string is slightly different, handle it here. - return errVolumeNotEmpty - } else if strings.Contains(err.Error(), "directory not empty") { - // Hopefully for all other operating systems, this is - // assumed to be consistent. + } else if isSysErrNotEmpty(err) { return errVolumeNotEmpty } return err @@ -433,7 +434,7 @@ func (s *posix) ReadAll(volume, path string) (buf []byte, err error) { case syscall.ENOTDIR, syscall.EISDIR: return nil, errFileNotFound default: - if strings.Contains(pathErr.Err.Error(), "The handle is invalid") { + if isSysErrHandleInvalid(pathErr.Err) { // This case is special and needs to be handled for windows. return nil, errFileNotFound } @@ -492,7 +493,7 @@ func (s *posix) ReadFile(volume string, path string, offset int64, buf []byte) ( return 0, errFileNotFound } else if os.IsPermission(err) { return 0, errFileAccessDenied - } else if strings.Contains(err.Error(), "not a directory") { + } else if isSysErrNotDir(err) { return 0, errFileAccessDenied } return 0, err @@ -569,9 +570,9 @@ func (s *posix) AppendFile(volume, path string, buf []byte) (err error) { // with mode 0777 mkdir honors system umask. if err = mkdirAll(filepath.Dir(filePath), 0777); err != nil { // File path cannot be verified since one of the parents is a file. - if strings.Contains(err.Error(), "not a directory") { + if isSysErrNotDir(err) { return errFileAccessDenied - } else if runtime.GOOS == "windows" && strings.Contains(err.Error(), "system cannot find the path specified") { + } else if isSysErrPathNotFound(err) { // Add specific case for windows. return errFileAccessDenied } @@ -583,7 +584,7 @@ func (s *posix) AppendFile(volume, path string, buf []byte) (err error) { w, err := os.OpenFile(preparePath(filePath), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666) if err != nil { // File path cannot be verified since one of the parents is a file. - if strings.Contains(err.Error(), "not a directory") { + if isSysErrNotDir(err) { return errFileAccessDenied } return err @@ -639,7 +640,7 @@ func (s *posix) StatFile(volume, path string) (file FileInfo, err error) { } // File path cannot be verified since one of the parents is a file. - if strings.Contains(err.Error(), "not a directory") { + if isSysErrNotDir(err) { return FileInfo{}, errFileNotFound } @@ -798,9 +799,9 @@ func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err e // Creates all the parent directories, with mode 0777 mkdir honors system umask. if err = mkdirAll(preparePath(slashpath.Dir(dstFilePath)), 0777); err != nil { // File path cannot be verified since one of the parents is a file. - if strings.Contains(err.Error(), "not a directory") { + if isSysErrNotDir(err) { return errFileAccessDenied - } else if strings.Contains(err.Error(), "The system cannot find the path specified.") && runtime.GOOS == "windows" { + } else if isSysErrPathNotFound(err) { // This is a special case should be handled only for // windows, because windows API does not return "not a // directory" error message. Handle this specifically here. diff --git a/cmd/posix_test.go b/cmd/posix_test.go index 25ac68b43..6bccb66cb 100644 --- a/cmd/posix_test.go +++ b/cmd/posix_test.go @@ -18,7 +18,6 @@ package cmd import ( "bytes" - "errors" "io" "io/ioutil" "os" @@ -901,7 +900,7 @@ func TestReadFile(t *testing.T) { return &os.PathError{ Op: "seek", Path: preparePath(slashpath.Join(path, "success-vol", "myobject")), - Err: errors.New("An attempt was made to move the file pointer before the beginning of the file."), + Err: syscall.Errno(0x83), // ERROR_NEGATIVE_SEEK } } return &os.PathError{ @@ -953,7 +952,24 @@ func TestReadFile(t *testing.T) { if err != nil && testCase.expectedErr != nil { // Validate if the type string of the errors are an exact match. if err.Error() != testCase.expectedErr.Error() { - t.Errorf("Case: %d %#v, expected: %s, got: %s", i+1, testCase, testCase.expectedErr, err) + if runtime.GOOS != "windows" { + t.Errorf("Case: %d %#v, expected: %s, got: %s", i+1, testCase, testCase.expectedErr, err) + } else { + var resultErrno, expectErrno uintptr + if pathErr, ok := err.(*os.PathError); ok { + if errno, pok := pathErr.Err.(syscall.Errno); pok { + resultErrno = uintptr(errno) + } + } + if pathErr, ok := testCase.expectedErr.(*os.PathError); ok { + if errno, pok := pathErr.Err.(syscall.Errno); pok { + expectErrno = uintptr(errno) + } + } + if !(expectErrno != 0 && resultErrno != 0 && expectErrno == resultErrno) { + t.Errorf("Case: %d %#v, expected: %s, got: %s", i+1, testCase, testCase.expectedErr, err) + } + } } // Err unexpected EOF special case, where we verify we have provided a larger // buffer than the data itself, but the results are in-fact valid. So we validate diff --git a/cmd/post-policy_test.go b/cmd/post-policy_test.go new file mode 100644 index 000000000..434f62863 --- /dev/null +++ b/cmd/post-policy_test.go @@ -0,0 +1,201 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "bytes" + "encoding/base64" + "fmt" + "mime/multipart" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +const ( + expirationDateFormat = "2006-01-02T15:04:05.999Z" + iso8601DateFormat = "20060102T150405Z" +) + +// newPostPolicyBytes - creates a bare bones postpolicy string with key and bucket matches. +func newPostPolicyBytes(credential, bucketName, objectKey string, expiration time.Time) []byte { + t := time.Now().UTC() + // Add the expiration date. + expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(expirationDateFormat)) + // Add the bucket condition, only accept buckets equal to the one passed. + bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName) + // Add the key condition, only accept keys equal to the one passed. + keyConditionStr := fmt.Sprintf(`["eq", "$key", "%s"]`, objectKey) + // Add the algorithm condition, only accept AWS SignV4 Sha256. + algorithmConditionStr := `["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"]` + // Add the date condition, only accept the current date. + dateConditionStr := fmt.Sprintf(`["eq", "$x-amz-date", "%s"]`, t.Format(iso8601DateFormat)) + // Add the credential string, only accept the credential passed. + credentialConditionStr := fmt.Sprintf(`["eq", "$x-amz-credential", "%s"]`, credential) + + // Combine all conditions into one string. + conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s]`, bucketConditionStr, keyConditionStr, algorithmConditionStr, dateConditionStr, credentialConditionStr) + retStr := "{" + retStr = retStr + expirationStr + "," + retStr = retStr + conditionStr + retStr = retStr + "}" + + return []byte(retStr) +} + +// Wrapper for calling TestPostPolicyHandlerHandler tests for both XL multiple disks and single node setup. +func TestPostPolicyHandler(t *testing.T) { + ExecObjectLayerTest(t, testPostPolicyHandler) +} + +// testPostPolicyHandler - Tests validate post policy handler uploading objects. +func testPostPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandler) { + // get random bucket name. + bucketName := getRandomBucketName() + + // Register the API end points with XL/FS object layer. + apiRouter := initTestAPIEndPoints(obj, []string{"PostPolicy"}) + + // initialize the server and obtain the credentials and root. + // credentials are necessary to sign the HTTP request. + rootPath, err := newTestConfig("us-east-1") + if err != nil { + t.Fatalf("Init Test config failed") + } + // remove the root folder after the test ends. + defer removeAll(rootPath) + + // bucketnames[0]. + // objectNames[0]. + // uploadIds [0]. + // Create bucket before initiating NewMultipartUpload. + err = obj.MakeBucket(bucketName) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + // Collection of non-exhaustive ListMultipartUploads test cases, valid errors + // and success responses. + testCases := []struct { + objectName string + data []byte + expectedRespStatus int + shouldPass bool + }{ + // Success case. + { + objectName: "test", + data: []byte("Hello, World"), + expectedRespStatus: http.StatusNoContent, + shouldPass: true, + }, + // Bad case. + { + objectName: "test", + data: []byte("Hello, World"), + expectedRespStatus: http.StatusBadRequest, + shouldPass: false, + }, + } + + for i, testCase := range testCases { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + req, perr := newPostRequest("", bucketName, testCase.objectName, testCase.data, testCase.shouldPass) + if perr != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for PostPolicyHandler: %v", i+1, instanceType, perr) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + if rec.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + } + } + +} + +// postPresignSignatureV4 - presigned signature for PostPolicy requests. +func postPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string { + // Get signining key. + signingkey := getSigningKey(secretAccessKey, t, location) + // Calculate signature. + signature := getSignature(signingkey, policyBase64) + return signature +} + +func newPostRequest(endPoint, bucketName, objectName string, objData []byte, shouldPass bool) (*http.Request, error) { + // Keep time. + t := time.Now().UTC() + // Expire the request five minutes from now. + expirationTime := t.Add(time.Minute * 5) + // Get the user credential. + credentials := serverConfig.GetCredential() + credStr := getCredential(credentials.AccessKeyID, serverConfig.GetRegion(), t) + // Create a new post policy. + policy := newPostPolicyBytes(credStr, bucketName, objectName, expirationTime) + // Only need the encoding. + encodedPolicy := base64.StdEncoding.EncodeToString(policy) + + formData := make(map[string]string) + if shouldPass { + // Presign with V4 signature based on the policy. + signature := postPresignSignatureV4(encodedPolicy, t, credentials.SecretAccessKey, serverConfig.GetRegion()) + + formData = map[string]string{ + "bucket": bucketName, + "key": objectName, + "x-amz-credential": credStr, + "policy": encodedPolicy, + "x-amz-signature": signature, + "x-amz-date": t.Format(iso8601DateFormat), + "x-amz-algorithm": "AWS4-HMAC-SHA256", + } + } + + // Create the multipart form. + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + + // Set the normal formData + for k, v := range formData { + w.WriteField(k, v) + } + // Set the File formData + writer, err := w.CreateFormFile("file", "s3verify/post/object") + if err != nil { + // return nil, err + return nil, err + } + writer.Write(objData) + // Close before creating the new request. + w.Close() + + // Set the body equal to the created policy. + reader := bytes.NewReader(buf.Bytes()) + + req, err := http.NewRequest("POST", makeTestTargetURL(endPoint, bucketName, objectName, nil), reader) + if err != nil { + return nil, err + } + + // Set form content-type. + req.Header.Set("Content-Type", w.FormDataContentType()) + return req, nil +} diff --git a/cmd/prepare-storage.go b/cmd/prepare-storage.go new file mode 100644 index 000000000..98ae06cbb --- /dev/null +++ b/cmd/prepare-storage.go @@ -0,0 +1,227 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "time" + + "github.com/minio/minio-go/pkg/set" +) + +// Channel where minioctl heal handler would notify if it were successful. This +// would be used by waitForFormattingDisks routine to check if it's worth +// retrying loadAllFormats. +var globalWakeupCh chan struct{} + +func init() { + globalWakeupCh = make(chan struct{}, 1) +} + +/* + + Following table lists different possible states the backend could be in. + + * In a single-node, multi-disk setup, "Online" would refer to disks' status. + + * In a multi-node setup, it could refer to disks' or network connectivity + between the nodes, or both. + + +----------+--------------------------+-----------------------+ + | Online | Format status | Course of action | + | | | | + -----------+--------------------------+-----------------------+ + | All | All Formatted | | + +----------+--------------------------+ initObjectLayer | + | Quorum | Quorum Formatted | | + +----------+--------------------------+-----------------------+ + | All | Quorum | Print message saying | + | | Formatted, | "Heal via minioctl" | + | | some unformatted | and initObjectLayer | + +----------+--------------------------+-----------------------+ + | All | None Formatted | FormatDisks | + | | | and initObjectLayer | + | | | | + +----------+--------------------------+-----------------------+ + | | | Wait for notify from | + | Quorum | | "Heal via minioctl" | + | | Quorum UnFormatted | | + +----------+--------------------------+-----------------------+ + | No | | Wait till enough | + | Quorum | _ | nodes are online and | + | | | one of the above | + | | | sections apply | + +----------+--------------------------+-----------------------+ + + N B A disk can be in one of the following states. + - Unformatted + - Formatted + - Corrupted + - Offline + +*/ + +// InitActions - a type synonym for enumerating initialization activities. +type InitActions int + +const ( + // FormatDisks - see above table for disk states where it is applicable. + FormatDisks InitActions = iota + + // WaitForHeal - Wait for disks to heal. + WaitForHeal + + // WaitForQuorum - Wait for quorum number of disks to be online. + WaitForQuorum + + // WaitForAll - Wait for all disks to be online. + WaitForAll + + // WaitForFormatting - Wait for formatting to be triggered from the '1st' server in the cluster. + WaitForFormatting + + // InitObjectLayer - Initialize object layer. + InitObjectLayer + + // Abort initialization of object layer since there aren't enough good + // copies of format.json to recover. + Abort +) + +func prepForInit(disks []string, sErrs []error, diskCount int) InitActions { + // Count errors by error value. + errMap := make(map[error]int) + for _, err := range sErrs { + errMap[err]++ + } + + quorum := diskCount/2 + 1 + disksOffline := errMap[errDiskNotFound] + disksFormatted := errMap[nil] + disksUnformatted := errMap[errUnformattedDisk] + disksCorrupted := errMap[errCorruptedFormat] + + // All disks are unformatted, proceed to formatting disks. + if disksUnformatted == diskCount { + // Only the first server formats an uninitialized setup, others wait for notification. + if isLocalStorage(disks[0]) { + return FormatDisks + } + return WaitForFormatting + } else if disksUnformatted >= quorum { + if disksUnformatted+disksOffline == diskCount { + return WaitForAll + } + // Some disks possibly corrupted. + return WaitForHeal + } + + // Already formatted, proceed to initialization of object layer. + if disksFormatted == diskCount { + return InitObjectLayer + } else if disksFormatted >= quorum { + if (disksFormatted+disksOffline == diskCount) || + (disksFormatted+disksUnformatted == diskCount) { + return InitObjectLayer + } + // Some disks possibly corrupted. + return WaitForHeal + } + + // No Quorum. + if disksOffline >= quorum { + return WaitForQuorum + } + + // There is quorum or more corrupted disks, there is not enough good + // disks to reconstruct format.json. + if disksCorrupted >= quorum { + return Abort + } + // Some of the formatted disks are possibly offline. + return WaitForHeal +} + +func retryFormattingDisks(disks []string, storageDisks []StorageAPI) ([]StorageAPI, error) { + nextBackoff := time.Duration(0) + var err error + done := false + for !done { + select { + case <-time.After(nextBackoff * time.Second): + // Attempt to load all `format.json`. + _, sErrs := loadAllFormats(storageDisks) + switch prepForInit(disks, sErrs, len(storageDisks)) { + case Abort: + err = errCorruptedFormat + done = true + case FormatDisks: + err = initFormatXL(storageDisks) + done = true + case InitObjectLayer: + err = nil + done = true + } + case <-globalWakeupCh: + // Reset nextBackoff to reduce the subsequent wait and re-read + // format.json from all disks again. + nextBackoff = 0 + } + } + if err != nil { + return nil, err + } + return storageDisks, nil +} + +func waitForFormattingDisks(disks, ignoredDisks []string) ([]StorageAPI, error) { + // FS Setup + if len(disks) == 1 { + storage, err := newStorageAPI(disks[0]) + if err != nil && err != errDiskNotFound { + return nil, err + } + return []StorageAPI{storage}, nil + } + + // XL Setup + if err := checkSufficientDisks(disks); err != nil { + return nil, err + } + + disksSet := set.NewStringSet() + if len(ignoredDisks) > 0 { + disksSet = set.CreateStringSet(ignoredDisks...) + } + // Bootstrap disks. + storageDisks := make([]StorageAPI, len(disks)) + for index, disk := range disks { + // Check if disk is ignored. + if disksSet.Contains(disk) { + storageDisks[index] = nil + continue + } + // Intentionally ignore disk not found errors. XL is designed + // to handle these errors internally. + storage, err := newStorageAPI(disk) + if err != nil && err != errDiskNotFound { + return nil, err + } + storageDisks[index] = storage + } + + return retryFormattingDisks(disks, storageDisks) +} diff --git a/cmd/prepare-storage_test.go b/cmd/prepare-storage_test.go new file mode 100644 index 000000000..bdeedd43b --- /dev/null +++ b/cmd/prepare-storage_test.go @@ -0,0 +1,153 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "runtime" + "testing" +) + +func (action InitActions) String() string { + switch action { + case InitObjectLayer: + return "InitObjectLayer" + case FormatDisks: + return "FormatDisks" + case WaitForFormatting: + return "WaitForFormatting" + case WaitForHeal: + return "WaitForHeal" + case WaitForAll: + return "WaitForAll" + case WaitForQuorum: + return "WaitForQuorum" + case Abort: + return "Abort" + default: + return "Unknown" + } +} +func TestPrepForInit(t *testing.T) { + var disks []string + if runtime.GOOS == "windows" { + disks = []string{ + `c:\mnt\disk1`, + `c:\mnt\disk2`, + `c:\mnt\disk3`, + `c:\mnt\disk4`, + `c:\mnt\disk5`, + `c:\mnt\disk6`, + `c:\mnt\disk7`, + `c:\mnt\disk8`, + } + } else { + disks = []string{ + "/mnt/disk1", + "/mnt/disk2", + "/mnt/disk3", + "/mnt/disk4", + "/mnt/disk5", + "/mnt/disk6", + "/mnt/disk7", + "/mnt/disk8", + } + } + // Building up disks that resolve to localhost and remote w.r.t isLocalStorage(). + var ( + disksLocal []string + disksRemote []string + ) + for i := range disks { + disksLocal = append(disksLocal, "localhost:"+disks[i]) + } + // Using 4.4.4.4 as a known non-local address. + for i := range disks { + disksRemote = append(disksRemote, "4.4.4.4:"+disks[i]) + } + // All disks are unformatted, a fresh setup. + allUnformatted := []error{ + errUnformattedDisk, errUnformattedDisk, errUnformattedDisk, errUnformattedDisk, + errUnformattedDisk, errUnformattedDisk, errUnformattedDisk, errUnformattedDisk, + } + // All disks are formatted, possible restart of a node in a formatted setup. + allFormatted := []error{ + nil, nil, nil, nil, + nil, nil, nil, nil, + } + // Quorum number of disks are formatted and rest are offline. + quorumFormatted := []error{ + nil, nil, nil, nil, + nil, errDiskNotFound, errDiskNotFound, errDiskNotFound, + } + // Minority disks are corrupted, can be healed. + minorityCorrupted := []error{ + errCorruptedFormat, errCorruptedFormat, errCorruptedFormat, nil, + nil, nil, nil, nil, + } + // Majority disks are corrupted, pretty bad setup. + majorityCorrupted := []error{ + errCorruptedFormat, errCorruptedFormat, errCorruptedFormat, errCorruptedFormat, + errCorruptedFormat, nil, nil, nil, + } + // Quorum disks are unformatted, remaining yet to come online. + quorumUnformatted := []error{ + errUnformattedDisk, errUnformattedDisk, errUnformattedDisk, errUnformattedDisk, + errUnformattedDisk, errDiskNotFound, errDiskNotFound, errDiskNotFound, + } + quorumUnformattedSomeCorrupted := []error{ + errUnformattedDisk, errUnformattedDisk, errUnformattedDisk, errUnformattedDisk, + errUnformattedDisk, errCorruptedFormat, errCorruptedFormat, errDiskNotFound, + } + // Quorum number of disks not online yet. + noQuourm := []error{ + errDiskNotFound, errDiskNotFound, errDiskNotFound, errDiskNotFound, + errDiskNotFound, nil, nil, nil, + } + + testCases := []struct { + // Params for prepForInit(). + disks []string + errs []error + diskCount int + action InitActions + }{ + // Local disks. + {disksLocal, allFormatted, 8, InitObjectLayer}, + {disksLocal, quorumFormatted, 8, InitObjectLayer}, + {disksLocal, allUnformatted, 8, FormatDisks}, + {disksLocal, quorumUnformatted, 8, WaitForAll}, + {disksLocal, quorumUnformattedSomeCorrupted, 8, WaitForHeal}, + {disksLocal, noQuourm, 8, WaitForQuorum}, + {disksLocal, minorityCorrupted, 8, WaitForHeal}, + {disksLocal, majorityCorrupted, 8, Abort}, + // Remote disks. + {disksRemote, allFormatted, 8, InitObjectLayer}, + {disksRemote, quorumFormatted, 8, InitObjectLayer}, + {disksRemote, allUnformatted, 8, WaitForFormatting}, + {disksRemote, quorumUnformatted, 8, WaitForAll}, + {disksRemote, quorumUnformattedSomeCorrupted, 8, WaitForHeal}, + {disksRemote, noQuourm, 8, WaitForQuorum}, + {disksRemote, minorityCorrupted, 8, WaitForHeal}, + {disksRemote, majorityCorrupted, 8, Abort}, + } + for i, test := range testCases { + actual := prepForInit(test.disks, test.errs, test.diskCount) + if actual != test.action { + t.Errorf("Test %d expected %s but receieved %s\n", i+1, test.action, actual) + } + } +} diff --git a/cmd/routers.go b/cmd/routers.go index 42503de70..7890657b6 100644 --- a/cmd/routers.go +++ b/cmd/routers.go @@ -17,7 +17,6 @@ package cmd import ( - "errors" "net/http" "os" "strings" @@ -25,86 +24,102 @@ import ( router "github.com/gorilla/mux" ) -// newObjectLayer - initialize any object layer depending on the number of disks. -func newObjectLayer(disks, ignoredDisks []string) (ObjectLayer, error) { - if len(disks) == 1 { - exportPath := disks[0] - // Initialize FS object layer. - return newFSObjects(exportPath) - } - // Initialize XL object layer. - objAPI, err := newXLObjects(disks, ignoredDisks) - if err == errXLWriteQuorum { - return objAPI, errors.New("Disks are different with last minio server run.") - } - return objAPI, err +func newObjectLayerFn() ObjectLayer { + objLayerMutex.Lock() + defer objLayerMutex.Unlock() + return globalObjectAPI } -// configureServer handler returns final handler for the http server. -func configureServerHandler(srvCmdConfig serverCmdConfig) http.Handler { - // Initialize name space lock. - initNSLock() - - objAPI, err := newObjectLayer(srvCmdConfig.disks, srvCmdConfig.ignoredDisks) - fatalIf(err, "Unable to intialize object layer.") +// newObjectLayer - initialize any object layer depending on the number of disks. +func newObjectLayer(disks, ignoredDisks []string) (ObjectLayer, error) { + var objAPI ObjectLayer + var err error + if len(disks) == 1 { + // Initialize FS object layer. + objAPI, err = newFSObjects(disks[0]) + } else { + // Initialize XL object layer. + objAPI, err = newXLObjects(disks, ignoredDisks) + } + if err != nil { + return nil, err + } // Migrate bucket policy from configDir to .minio.sys/buckets/ err = migrateBucketPolicyConfig(objAPI) - fatalIf(err, "Unable to migrate bucket policy from config directory") + if err != nil { + errorIf(err, "Unable to migrate bucket policy from config directory") + return nil, err + } err = cleanupOldBucketPolicyConfigs() - fatalIf(err, "Unable to clean up bucket policy from config directory.") - - // Initialize storage rpc server. - storageRPC, err := newRPCServer(srvCmdConfig.disks[0]) // FIXME: should only have one path. - fatalIf(err, "Unable to initialize storage RPC server.") - - // Initialize API. - apiHandlers := objectAPIHandlers{ - ObjectAPI: objAPI, + if err != nil { + errorIf(err, "Unable to clean up bucket policy from config directory.") + return nil, err } - // Initialize Web. - webHandlers := &webAPIHandlers{ - ObjectAPI: objAPI, - } - - // Initialize Controller. - ctrlHandlers := &controllerAPIHandlers{ - ObjectAPI: objAPI, - } - - // Initialize and monitor shutdown signals. - err = initGracefulShutdown(os.Exit) - fatalIf(err, "Unable to initialize graceful shutdown operation") - // Register the callback that should be called when the process shuts down. globalShutdownCBs.AddObjectLayerCB(func() errCode { - if sErr := objAPI.Shutdown(); sErr != nil { - return exitFailure + if objAPI != nil { + if sErr := objAPI.Shutdown(); sErr != nil { + return exitFailure + } } return exitSuccess }) // Initialize a new event notifier. err = initEventNotifier(objAPI) - fatalIf(err, "Unable to initialize event notification queue") + if err != nil { + errorIf(err, "Unable to initialize event notification.") + } - // Initialize a new bucket policies. + // Initialize and load bucket policies. err = initBucketPolicies(objAPI) - fatalIf(err, "Unable to load all bucket policies") + if err != nil { + errorIf(err, "Unable to load all bucket policies.") + } + + // Success. + return objAPI, nil +} + +// configureServer handler returns final handler for the http server. +func configureServerHandler(srvCmdConfig serverCmdConfig) http.Handler { + // Initialize storage rpc servers for every disk that is hosted on this node. + storageRPCs, err := newRPCServer(srvCmdConfig) + fatalIf(err, "Unable to initialize storage RPC server.") + + // Initialize and monitor shutdown signals. + err = initGracefulShutdown(os.Exit) + fatalIf(err, "Unable to initialize graceful shutdown operation") + + // Initialize API. + apiHandlers := objectAPIHandlers{ + ObjectAPI: newObjectLayerFn, + } + + // Initialize Web. + webHandlers := &webAPIHandlers{ + ObjectAPI: newObjectLayerFn, + } + + // Initialize Controller. + controllerHandlers := &controllerAPIHandlers{ + ObjectAPI: newObjectLayerFn, + } // Initialize router. mux := router.NewRouter() // Register all routers. - registerStorageRPCRouter(mux, storageRPC) + registerStorageRPCRouters(mux, storageRPCs) - // FIXME: till net/rpc auth is brought in "minio control" can be enabled only though - // this env variable. - if os.Getenv("MINIO_CONTROL") != "" { - registerControlRPCRouter(mux, ctrlHandlers) - } + // Initialize distributed NS lock. + initDistributedNSLock(mux, srvCmdConfig) + + // Register controller rpc router. + registerControllerRPCRouter(mux, controllerHandlers) // set environmental variable MINIO_BROWSER=off to disable minio web browser. // By default minio web browser is enabled. @@ -112,11 +127,10 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) http.Handler { registerWebRouter(mux, webHandlers) } - registerAPIRouter(mux, apiHandlers) // Add new routers here. + registerAPIRouter(mux, apiHandlers) - // List of some generic handlers which are applied for all - // incoming requests. + // List of some generic handlers which are applied for all incoming requests. var handlerFns = []HandlerFunc{ // Limits the number of concurrent http requests. setRateLimitHandler, diff --git a/cmd/rpc-server.go b/cmd/rpc-server.go deleted file mode 100644 index 2a32705d1..000000000 --- a/cmd/rpc-server.go +++ /dev/null @@ -1,124 +0,0 @@ -package cmd - -import ( - "net/rpc" - - router "github.com/gorilla/mux" -) - -// Storage server implements rpc primitives to facilitate exporting a -// disk over a network. -type storageServer struct { - storage StorageAPI -} - -/// Volume operations handlers - -// MakeVolHandler - make vol handler is rpc wrapper for MakeVol operation. -func (s *storageServer) MakeVolHandler(arg *string, reply *GenericReply) error { - return s.storage.MakeVol(*arg) -} - -// ListVolsHandler - list vols handler is rpc wrapper for ListVols operation. -func (s *storageServer) ListVolsHandler(arg *string, reply *ListVolsReply) error { - vols, err := s.storage.ListVols() - if err != nil { - return err - } - reply.Vols = vols - return nil -} - -// StatVolHandler - stat vol handler is a rpc wrapper for StatVol operation. -func (s *storageServer) StatVolHandler(arg *string, reply *VolInfo) error { - volInfo, err := s.storage.StatVol(*arg) - if err != nil { - return err - } - *reply = volInfo - return nil -} - -// DeleteVolHandler - delete vol handler is a rpc wrapper for -// DeleteVol operation. -func (s *storageServer) DeleteVolHandler(arg *string, reply *GenericReply) error { - return s.storage.DeleteVol(*arg) -} - -/// File operations - -// StatFileHandler - stat file handler is rpc wrapper to stat file. -func (s *storageServer) StatFileHandler(arg *StatFileArgs, reply *FileInfo) error { - fileInfo, err := s.storage.StatFile(arg.Vol, arg.Path) - if err != nil { - return err - } - *reply = fileInfo - return nil -} - -// ListDirHandler - list directory handler is rpc wrapper to list dir. -func (s *storageServer) ListDirHandler(arg *ListDirArgs, reply *[]string) error { - entries, err := s.storage.ListDir(arg.Vol, arg.Path) - if err != nil { - return err - } - *reply = entries - return nil -} - -// ReadAllHandler - read all handler is rpc wrapper to read all storage API. -func (s *storageServer) ReadAllHandler(arg *ReadFileArgs, reply *[]byte) error { - buf, err := s.storage.ReadAll(arg.Vol, arg.Path) - if err != nil { - return err - } - reply = &buf - return nil -} - -// ReadFileHandler - read file handler is rpc wrapper to read file. -func (s *storageServer) ReadFileHandler(arg *ReadFileArgs, reply *int64) error { - n, err := s.storage.ReadFile(arg.Vol, arg.Path, arg.Offset, arg.Buffer) - if err != nil { - return err - } - reply = &n - return nil -} - -// AppendFileHandler - append file handler is rpc wrapper to append file. -func (s *storageServer) AppendFileHandler(arg *AppendFileArgs, reply *GenericReply) error { - return s.storage.AppendFile(arg.Vol, arg.Path, arg.Buffer) -} - -// DeleteFileHandler - delete file handler is rpc wrapper to delete file. -func (s *storageServer) DeleteFileHandler(arg *DeleteFileArgs, reply *GenericReply) error { - return s.storage.DeleteFile(arg.Vol, arg.Path) -} - -// RenameFileHandler - rename file handler is rpc wrapper to rename file. -func (s *storageServer) RenameFileHandler(arg *RenameFileArgs, reply *GenericReply) error { - return s.storage.RenameFile(arg.SrcVol, arg.SrcPath, arg.DstVol, arg.DstPath) -} - -// Initialize new storage rpc. -func newRPCServer(exportPath string) (*storageServer, error) { - // Initialize posix storage API. - storage, err := newPosix(exportPath) - if err != nil && err != errDiskNotFound { - return nil, err - } - return &storageServer{ - storage: storage, - }, nil -} - -// registerStorageRPCRouter - register storage rpc router. -func registerStorageRPCRouter(mux *router.Router, stServer *storageServer) { - storageRPCServer := rpc.NewServer() - storageRPCServer.RegisterName("Storage", stServer) - storageRouter := mux.NewRoute().PathPrefix(reservedBucket).Subrouter() - // Add minio storage routes. - storageRouter.Path("/storage").Handler(storageRPCServer) -} diff --git a/cmd/server-main.go b/cmd/server-main.go index a0ed0307a..4ff9088dc 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -28,28 +28,32 @@ import ( "github.com/minio/cli" ) -var serverCmd = cli.Command{ - Name: "server", - Usage: "Start object storage server.", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "address", - Value: ":9000", - Usage: "Specify custom server \"ADDRESS:PORT\", defaults to \":9000\".", - }, - cli.StringFlag{ - Name: "ignore-disks", - Usage: "Specify comma separated list of disks that are offline.", - }, +var srvConfig serverCmdConfig + +var serverFlags = []cli.Flag{ + cli.StringFlag{ + Name: "address", + Value: ":9000", + Usage: "Specify custom server \"ADDRESS:PORT\", defaults to \":9000\".", }, + cli.StringFlag{ + Name: "ignore-disks", + Usage: "Specify comma separated list of disks that are offline.", + }, +} + +var serverCmd = cli.Command{ + Name: "server", + Usage: "Start object storage server.", + Flags: append(serverFlags, globalFlags...), Action: serverMain, CustomHelpTemplate: `NAME: minio {{.Name}} - {{.Usage}} USAGE: - minio {{.Name}} [OPTIONS] PATH [PATH...] + minio {{.Name}} [FLAGS] PATH [PATH...] -OPTIONS: +FLAGS: {{range .Flags}}{{.}} {{end}} ENVIRONMENT VARIABLES: @@ -72,15 +76,21 @@ EXAMPLES: $ minio {{.Name}} C:\MyShare 4. Start minio server on 12 disks to enable erasure coded layer with 6 data and 6 parity. - $ minio {{.Name}} /mnt/export1/backend /mnt/export2/backend /mnt/export3/backend /mnt/export4/backend \ - /mnt/export5/backend /mnt/export6/backend /mnt/export7/backend /mnt/export8/backend /mnt/export9/backend \ - /mnt/export10/backend /mnt/export11/backend /mnt/export12/backend + $ minio {{.Name}} /mnt/export1/ /mnt/export2/ /mnt/export3/ /mnt/export4/ \ + /mnt/export5/ /mnt/export6/ /mnt/export7/ /mnt/export8/ /mnt/export9/ \ + /mnt/export10/ /mnt/export11/ /mnt/export12/ 5. Start minio server on 12 disks while ignoring two disks for initialization. - $ minio {{.Name}} --ignore-disks=/mnt/export1/backend,/mnt/export2/backend /mnt/export1/backend \ - /mnt/export2/backend /mnt/export3/backend /mnt/export4/backend /mnt/export5/backend /mnt/export6/backend \ - /mnt/export7/backend /mnt/export8/backend /mnt/export9/backend /mnt/export10/backend /mnt/export11/backend \ - /mnt/export12/backend + $ minio {{.Name}} --ignore-disks=/mnt/export1/ /mnt/export1/ /mnt/export2/ \ + /mnt/export3/ /mnt/export4/ /mnt/export5/ /mnt/export6/ /mnt/export7/ \ + /mnt/export8/ /mnt/export9/ /mnt/export10/ /mnt/export11/ /mnt/export12/ + + 6. Start minio server on a 4 node distributed setup. Type the following command on all the 4 nodes. + $ export MINIO_ACCESS_KEY=minio + $ export MINIO_SECRET_KEY=miniostorage + $ minio {{.Name}} 192.168.1.11:/mnt/export/ 192.168.1.12:/mnt/export/ \ + 192.168.1.13:/mnt/export/ 192.168.1.14:/mnt/export/ + `, } @@ -194,16 +204,70 @@ func initServerConfig(c *cli.Context) { // Do not fail if this is not allowed, lower limits are fine as well. } +// Validate if input disks are sufficient for initializing XL. +func checkSufficientDisks(disks []string) error { + // Verify total number of disks. + totalDisks := len(disks) + if totalDisks > maxErasureBlocks { + return errXLMaxDisks + } + if totalDisks < minErasureBlocks { + return errXLMinDisks + } + + // isEven function to verify if a given number if even. + isEven := func(number int) bool { + return number%2 == 0 + } + + // Verify if we have even number of disks. + // only combination of 4, 6, 8, 10, 12, 14, 16 are supported. + if !isEven(totalDisks) { + return errXLNumDisks + } + + // Success. + return nil +} + +// Validates if disks are of supported format, invalid arguments are rejected. +func checkNamingDisks(disks []string) error { + for _, disk := range disks { + _, _, err := splitNetPath(disk) + if err != nil { + return err + } + } + return nil +} + // Check server arguments. func checkServerSyntax(c *cli.Context) { if !c.Args().Present() || c.Args().First() == "help" { cli.ShowCommandHelpAndExit(c, "server", 1) } + disks := c.Args() + if len(disks) > 1 { + // Validate if input disks have duplicates in them. + err := checkDuplicates(disks) + fatalIf(err, "Invalid disk arguments for server.") + + // Validate if input disks are sufficient for erasure coded setup. + err = checkSufficientDisks(disks) + fatalIf(err, "Invalid disk arguments for server.") + + // Validate if input disks are properly named in accordance with either + // - /mnt/disk1 + // - ip:/mnt/disk1 + err = checkNamingDisks(disks) + fatalIf(err, "Invalid disk arguments for server.") + } } // Extract port number from address address should be of the form host:port. func getPort(address string) int { _, portStr, _ := net.SplitHostPort(address) + // If port empty, default to port '80' if portStr == "" { portStr = "80" @@ -219,6 +283,51 @@ func getPort(address string) int { return portInt } +// Returns if slice of disks is a distributed setup. +func isDistributedSetup(disks []string) (isDist bool) { + // Port to connect to for the lock servers in a distributed setup. + for _, disk := range disks { + if !isLocalStorage(disk) { + // One or more disks supplied as arguments are not + // attached to the local node. + isDist = true + } + } + return isDist +} + +// Format disks before initialization object layer. +func formatDisks(disks, ignoredDisks []string) error { + storageDisks, err := waitForFormattingDisks(disks, ignoredDisks) + for _, storage := range storageDisks { + if storage == nil { + continue + } + switch store := storage.(type) { + // Closing associated TCP connections since + // []StorageAPI is garbage collected eventually. + case networkStorage: + store.rpcClient.Close() + } + } + if err != nil { + return err + } + if isLocalStorage(disks[0]) { + // notify every one else that they can try init again. + for _, storage := range storageDisks { + switch store := storage.(type) { + // Closing associated TCP connections since + // []StorageAPI is garage collected eventually. + case networkStorage: + var reply GenericReply + _ = store.rpcClient.Call("Storage.TryInitHandler", &GenericArgs{}, &reply) + } + } + } + return nil +} + // serverMain handler called for 'minio server' command. func serverMain(c *cli.Context) { // Check 'server' cli arguments. @@ -244,12 +353,29 @@ func serverMain(c *cli.Context) { // Disks to be used in server init. disks := c.Args() + isDist := isDistributedSetup(disks) + // Set nodes for dsync for distributed setup. + if isDist { + err = initDsyncNodes(disks, port) + fatalIf(err, "Unable to initialize distributed locking") + } + + // Initialize name space lock. + initNSLock(isDist) + // Configure server. - handler := configureServerHandler(serverCmdConfig{ + srvConfig = serverCmdConfig{ serverAddr: serverAddress, disks: disks, ignoredDisks: ignoredDisks, - }) + } + + // Initialize and monitor shutdown signals. + err = initGracefulShutdown(os.Exit) + fatalIf(err, "Unable to initialize graceful shutdown operation") + + // Configure server. + handler := configureServerHandler(srvConfig) apiServer := NewServerMux(serverAddress, handler) @@ -267,12 +393,36 @@ func serverMain(c *cli.Context) { // Start server. // Configure TLS if certs are available. - if tls { - err = apiServer.ListenAndServeTLS(mustGetCertFile(), mustGetKeyFile()) - } else { - // Fallback to http. - err = apiServer.ListenAndServe() + wait := make(chan struct{}, 1) + go func(tls bool, wait chan<- struct{}) { + if tls { + err = apiServer.ListenAndServeTLS(mustGetCertFile(), mustGetKeyFile()) + } else { + // Fallback to http. + err = apiServer.ListenAndServe() + } + wait <- struct{}{} + + }(tls, wait) + err = formatDisks(disks, ignoredDisks) + if err != nil { + // FIXME: call graceful exit + errorIf(err, "formatting storage disks failed") + return } + newObject, err := newObjectLayer(disks, ignoredDisks) + if err != nil { + // FIXME: call graceful exit + errorIf(err, "intializing object layer failed") + return + } + + printEventNotifiers() + + objLayerMutex.Lock() + globalObjectAPI = newObject + objLayerMutex.Unlock() + <-wait fatalIf(err, "Failed to start minio server.") } diff --git a/cmd/server-startup-msg.go b/cmd/server-startup-msg.go index d71158bef..b01d94b49 100644 --- a/cmd/server-startup-msg.go +++ b/cmd/server-startup-msg.go @@ -60,14 +60,25 @@ func printServerCommonMsg(endPoints []string) { console.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKeyID))) console.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretAccessKey))) console.Println(colorBlue("Region: ") + colorBold(fmt.Sprintf(getFormatStr(len(region), 3), region))) - arnMsg := colorBlue("SqsARNs: ") + + console.Println(colorBlue("\nBrowser Access:")) + console.Println(fmt.Sprintf(getFormatStr(len(endPointStr), 3), endPointStr)) +} + +// Prints bucket notification configurations. +func printEventNotifiers() { + if globalEventNotifier == nil { + // In case initEventNotifier() was not done or failed. + return + } + arnMsg := colorBlue("\nSQS ARNs: ") + if len(globalEventNotifier.queueTargets) == 0 { + arnMsg += colorBold(fmt.Sprintf(getFormatStr(len(""), 2), "")) + } for queueArn := range globalEventNotifier.queueTargets { arnMsg += colorBold(fmt.Sprintf(getFormatStr(len(queueArn), 2), queueArn)) } console.Println(arnMsg) - - console.Println(colorBlue("\nBrowser Access:")) - console.Println(fmt.Sprintf(getFormatStr(len(endPointStr), 3), endPointStr)) } // Prints startup message for command line access. Prints link to our documentation diff --git a/cmd/server_test.go b/cmd/server_test.go index f7ac00b00..158c28d8b 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -309,6 +309,7 @@ func (s *TestSuiteCommon) TestDeleteBucketNotEmpty(c *C) { } +// Test deletes multple objects and verifies server resonse. func (s *TestSuiteCommon) TestDeleteMultipleObjects(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() @@ -347,18 +348,11 @@ func (s *TestSuiteCommon) TestDeleteMultipleObjects(c *C) { ObjectName: objName, }) } - // Append a non-existent object for which the response should be marked - // as deleted. - delObjReq.Objects = append(delObjReq.Objects, ObjectIdentifier{ - ObjectName: fmt.Sprintf("%d/%s", 10, objectName), - }) - // Marshal delete request. deleteReqBytes, err := xml.Marshal(delObjReq) c.Assert(err, IsNil) - // object name was "prefix/myobject", an attempt to delelte "prefix" - // Should not delete "prefix/myobject" + // Delete list of objects. request, err = newTestSignedRequest("POST", getMultiDeleteObjectURL(s.endPoint, bucketName), int64(len(deleteReqBytes)), bytes.NewReader(deleteReqBytes), s.accessKey, s.secretKey) c.Assert(err, IsNil) @@ -372,11 +366,31 @@ func (s *TestSuiteCommon) TestDeleteMultipleObjects(c *C) { c.Assert(err, IsNil) err = xml.Unmarshal(delRespBytes, &deleteResp) c.Assert(err, IsNil) - for i := 0; i <= 10; i++ { + for i := 0; i < 10; i++ { // All the objects should be under deleted list (including non-existent object) c.Assert(deleteResp.DeletedObjects[i], DeepEquals, delObjReq.Objects[i]) } c.Assert(len(deleteResp.Errors), Equals, 0) + + // Attempt second time results should be same, NoSuchKey for objects not found + // shouldn't be set. + request, err = newTestSignedRequest("POST", getMultiDeleteObjectURL(s.endPoint, bucketName), + int64(len(deleteReqBytes)), bytes.NewReader(deleteReqBytes), s.accessKey, s.secretKey) + c.Assert(err, IsNil) + client = http.Client{} + response, err = client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + + deleteResp = DeleteObjectsResponse{} + delRespBytes, err = ioutil.ReadAll(response.Body) + c.Assert(err, IsNil) + err = xml.Unmarshal(delRespBytes, &deleteResp) + c.Assert(err, IsNil) + for i := 0; i < 10; i++ { + c.Assert(deleteResp.DeletedObjects[i], DeepEquals, delObjReq.Objects[i]) + } + c.Assert(len(deleteResp.Errors), Equals, 0) } // Tests delete object responses and success. @@ -1364,6 +1378,7 @@ func (s *TestSuiteCommon) TestListObjectsHandler(c *C) { c.Assert(response.StatusCode, Equals, http.StatusOK) getContent, err := ioutil.ReadAll(response.Body) + c.Assert(err, IsNil) c.Assert(strings.Contains(string(getContent), "bar"), Equals, true) // create listObjectsV2 request with valid parameters @@ -1377,6 +1392,7 @@ func (s *TestSuiteCommon) TestListObjectsHandler(c *C) { c.Assert(response.StatusCode, Equals, http.StatusOK) getContent, err = ioutil.ReadAll(response.Body) + c.Assert(err, IsNil) c.Assert(strings.Contains(string(getContent), "bar"), Equals, true) c.Assert(strings.Contains(string(getContent), ""), Equals, true) @@ -1960,6 +1976,7 @@ func (s *TestSuiteCommon) TestObjectMultipartAbort(c *C) { // execute the HTTP request initiating the new multipart upload. response, err = client.Do(request) + c.Assert(err, IsNil) c.Assert(response.StatusCode, Equals, http.StatusOK) // parse the response body and obtain the new upload ID. @@ -1977,6 +1994,7 @@ func (s *TestSuiteCommon) TestObjectMultipartAbort(c *C) { // execute the HTTP request initiating the new multipart upload. response, err = client.Do(request) + c.Assert(err, IsNil) c.Assert(response.StatusCode, Equals, http.StatusOK) // parse the response body and obtain the new upload ID. @@ -2193,6 +2211,7 @@ func (s *TestSuiteCommon) TestObjectMultipartListError(c *C) { c.Assert(err, IsNil) // execute the HTTP request initiating the new multipart upload. response, err = client.Do(request) + c.Assert(err, IsNil) c.Assert(response.StatusCode, Equals, http.StatusOK) // parse the response body and obtain the new upload ID. decoder := xml.NewDecoder(response.Body) diff --git a/cmd/server_utils_test.go b/cmd/server_utils_test.go index 32eb208b6..b33cb361c 100644 --- a/cmd/server_utils_test.go +++ b/cmd/server_utils_test.go @@ -18,6 +18,7 @@ package cmd import ( "encoding/xml" + "fmt" "io/ioutil" "net" "net/http" @@ -65,6 +66,40 @@ var ignoredHeaders = map[string]bool{ "User-Agent": true, } +// Headers to ignore in streaming v4 +var ignoredStreamingHeaders = map[string]bool{ + "Authorization": true, + "Content-Type": true, + "Content-Md5": true, + "User-Agent": true, +} + +// calculateSignedChunkLength - calculates the length of chunk metadata +func calculateSignedChunkLength(chunkDataSize int64) int64 { + return int64(len(fmt.Sprintf("%x", chunkDataSize))) + + 17 + // ";chunk-signature=" + 64 + // e.g. "f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2" + 2 + // CRLF + chunkDataSize + + 2 // CRLF +} + +// calculateSignedChunkLength - calculates the length of the overall stream (data + metadata) +func calculateStreamContentLength(dataLen, chunkSize int64) int64 { + if dataLen <= 0 { + return 0 + } + chunksCount := int64(dataLen / chunkSize) + remainingBytes := int64(dataLen % chunkSize) + streamLen := int64(0) + streamLen += chunksCount * calculateSignedChunkLength(chunkSize) + if remainingBytes > 0 { + streamLen += calculateSignedChunkLength(remainingBytes) + } + streamLen += calculateSignedChunkLength(0) + return streamLen +} + // Ask the kernel for a free open port. func getFreePort() int { addr, err := net.ResolveTCPAddr("tcp", "localhost:0") diff --git a/cmd/signature-jwt.go b/cmd/signature-jwt.go index 107c02efd..bcf99fc31 100644 --- a/cmd/signature-jwt.go +++ b/cmd/signature-jwt.go @@ -17,7 +17,7 @@ package cmd import ( - "fmt" + "errors" "strings" "time" @@ -32,24 +32,24 @@ type JWT struct { credential } -// Default - each token expires in 10hrs. +// Default each token expires in 100yrs. const ( - tokenExpires time.Duration = 10 + defaultTokenExpiry time.Duration = time.Hour * 876000 // 100yrs. ) // newJWT - returns new JWT object. -func newJWT() (*JWT, error) { +func newJWT(expiry time.Duration) (*JWT, error) { if serverConfig == nil { - return nil, fmt.Errorf("server not initialzed") + return nil, errors.New("Server not initialzed") } // Save access, secret keys. cred := serverConfig.GetCredential() if !isValidAccessKey.MatchString(cred.AccessKeyID) { - return nil, fmt.Errorf("Invalid access key") + return nil, errors.New("Invalid access key") } if !isValidSecretKey.MatchString(cred.SecretAccessKey) { - return nil, fmt.Errorf("Invalid secret key") + return nil, errors.New("Invalid secret key") } return &JWT{cred}, nil @@ -61,13 +61,13 @@ func (jwt *JWT) GenerateToken(accessKey string) (string, error) { accessKey = strings.TrimSpace(accessKey) if !isValidAccessKey.MatchString(accessKey) { - return "", fmt.Errorf("Invalid access key") + return "", errors.New("Invalid access key") } tUTCNow := time.Now().UTC() token := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, jwtgo.MapClaims{ // Token expires in 10hrs. - "exp": tUTCNow.Add(time.Hour * tokenExpires).Unix(), + "exp": tUTCNow.Add(defaultTokenExpiry).Unix(), "iat": tUTCNow.Unix(), "sub": accessKey, }) @@ -80,20 +80,20 @@ func (jwt *JWT) Authenticate(accessKey, secretKey string) error { accessKey = strings.TrimSpace(accessKey) if !isValidAccessKey.MatchString(accessKey) { - return fmt.Errorf("Invalid access key") + return errors.New("Invalid access key") } if !isValidSecretKey.MatchString(secretKey) { - return fmt.Errorf("Invalid secret key") + return errors.New("Invalid secret key") } if accessKey != jwt.AccessKeyID { - return fmt.Errorf("Access key does not match") + return errors.New("Access key does not match") } hashedSecretKey, _ := bcrypt.GenerateFromPassword([]byte(jwt.SecretAccessKey), bcrypt.DefaultCost) if bcrypt.CompareHashAndPassword(hashedSecretKey, []byte(secretKey)) != nil { - return fmt.Errorf("Authentication failed") + return errors.New("Authentication failed") } // Success. diff --git a/cmd/signature-jwt_test.go b/cmd/signature-jwt_test.go index 8137dad3c..0405f7435 100644 --- a/cmd/signature-jwt_test.go +++ b/cmd/signature-jwt_test.go @@ -72,11 +72,11 @@ func TestNewJWT(t *testing.T) { expectedErr error }{ // Test non-existent config directory. - {path.Join(path1, "non-existent-dir"), false, nil, fmt.Errorf("server not initialzed")}, + {path.Join(path1, "non-existent-dir"), false, nil, fmt.Errorf("Server not initialzed")}, // Test empty config directory. - {path2, false, nil, fmt.Errorf("server not initialzed")}, + {path2, false, nil, fmt.Errorf("Server not initialzed")}, // Test empty config file. - {path3, false, nil, fmt.Errorf("server not initialzed")}, + {path3, false, nil, fmt.Errorf("Server not initialzed")}, // Test initialized config file. {path4, true, nil, nil}, // Test to read already created config file. @@ -108,7 +108,7 @@ func TestNewJWT(t *testing.T) { serverConfig.SetCredential(*testCase.cred) } - _, err := newJWT() + _, err := newJWT(defaultWebTokenExpiry) if testCase.expectedErr != nil { if err == nil { @@ -132,7 +132,7 @@ func TestGenerateToken(t *testing.T) { } defer removeAll(testPath) - jwt, err := newJWT() + jwt, err := newJWT(defaultWebTokenExpiry) if err != nil { t.Fatalf("unable get new JWT, %s", err) } @@ -179,7 +179,7 @@ func TestAuthenticate(t *testing.T) { } defer removeAll(testPath) - jwt, err := newJWT() + jwt, err := newJWT(defaultWebTokenExpiry) if err != nil { t.Fatalf("unable get new JWT, %s", err) } diff --git a/cmd/storage-errors.go b/cmd/storage-errors.go index 787febf25..8e6fb65f0 100644 --- a/cmd/storage-errors.go +++ b/cmd/storage-errors.go @@ -59,3 +59,6 @@ var errVolumeAccessDenied = errors.New("volume access denied") // errVolumeAccessDenied - cannot access file, insufficient permissions. var errFileAccessDenied = errors.New("file access denied") + +// errVolumeBusy - remote disk is not connected to yet. +var errVolumeBusy = errors.New("volume is busy") diff --git a/cmd/storage-interface.go b/cmd/storage-interface.go index db1380186..566ebbf5c 100644 --- a/cmd/storage-interface.go +++ b/cmd/storage-interface.go @@ -16,8 +16,13 @@ package cmd +import "github.com/minio/minio/pkg/disk" + // StorageAPI interface. type StorageAPI interface { + // Storage operations. + DiskInfo() (info disk.Info, err error) + // Volume operations. MakeVol(volume string) (err error) ListVols() (vols []VolInfo, err error) diff --git a/cmd/rpc-client.go b/cmd/storage-rpc-client.go similarity index 62% rename from cmd/rpc-client.go rename to cmd/storage-rpc-client.go index 391ee2477..09251d1eb 100644 --- a/cmd/rpc-client.go +++ b/cmd/storage-rpc-client.go @@ -17,37 +17,48 @@ package cmd import ( - "net/http" + "io" + "net" "net/rpc" + "path" + "strconv" "strings" - "time" + + "github.com/minio/minio/pkg/disk" ) type networkStorage struct { - netScheme string - netAddr string - netPath string - rpcClient *rpc.Client - httpClient *http.Client + netAddr string + netPath string + rpcClient *AuthRPCClient } const ( storageRPCPath = reservedBucket + "/storage" ) -// splits network path into its components Address and Path. -func splitNetPath(networkPath string) (netAddr, netPath string) { - index := strings.LastIndex(networkPath, ":") - netAddr = networkPath[:index] - netPath = networkPath[index+1:] - return netAddr, netPath -} - // Converts rpc.ServerError to underlying error. This function is // written so that the storageAPI errors are consistent across network // disks as well. func toStorageErr(err error) error { + if err == nil { + return nil + } + + switch err.(type) { + case *net.OpError: + return errDiskNotFound + } + switch err.Error() { + case io.EOF.Error(): + return io.EOF + case io.ErrUnexpectedEOF.Error(): + return io.ErrUnexpectedEOF + case rpc.ErrShutdown.Error(): + return errDiskNotFound + case errUnexpected.Error(): + return errUnexpected case errDiskFull.Error(): return errDiskFull case errVolumeNotFound.Error(): @@ -56,14 +67,20 @@ func toStorageErr(err error) error { return errVolumeExists case errFileNotFound.Error(): return errFileNotFound + case errFileNameTooLong.Error(): + return errFileNameTooLong + case errFileAccessDenied.Error(): + return errFileAccessDenied case errIsNotRegular.Error(): return errIsNotRegular case errVolumeNotEmpty.Error(): return errVolumeNotEmpty - case errFileAccessDenied.Error(): - return errFileAccessDenied case errVolumeAccessDenied.Error(): return errVolumeAccessDenied + case errCorruptedFormat.Error(): + return errCorruptedFormat + case errUnformattedDisk.Error(): + return errUnformattedDisk } return err } @@ -75,50 +92,59 @@ func newRPCClient(networkPath string) (StorageAPI, error) { return nil, errInvalidArgument } - // TODO validate netAddr and netPath. - netAddr, netPath := splitNetPath(networkPath) - - // Dial minio rpc storage http path. - rpcClient, err := rpc.DialHTTPPath("tcp", netAddr, storageRPCPath) + // Split network path into its components. + netAddr, netPath, err := splitNetPath(networkPath) if err != nil { return nil, err } - // Initialize http client. - httpClient := &http.Client{ - // Setting a sensible time out of 6minutes to wait for - // response headers. Request is pro-actively cancelled - // after 6minutes if no response was received from server. - Timeout: 6 * time.Minute, - Transport: http.DefaultTransport, - } - + // Dial minio rpc storage http path. + rpcPath := path.Join(storageRPCPath, netPath) + port := getPort(srvConfig.serverAddr) + rpcAddr := netAddr + ":" + strconv.Itoa(port) + // Initialize rpc client with network address and rpc path. + cred := serverConfig.GetCredential() + rpcClient := newAuthClient(&authConfig{ + accessKey: cred.AccessKeyID, + secretKey: cred.SecretAccessKey, + address: rpcAddr, + path: rpcPath, + loginMethod: "Storage.LoginHandler", + }) // Initialize network storage. ndisk := &networkStorage{ - netScheme: "http", // TODO: fix for ssl rpc support. - netAddr: netAddr, - netPath: netPath, - rpcClient: rpcClient, - httpClient: httpClient, + netAddr: netAddr, + netPath: netPath, + rpcClient: rpcClient, } // Returns successfully here. return ndisk, nil } -// MakeVol - make a volume. +// DiskInfo - fetch disk information for a remote disk. +func (n networkStorage) DiskInfo() (info disk.Info, err error) { + args := GenericArgs{} + if err = n.rpcClient.Call("Storage.DiskInfoHandler", &args, &info); err != nil { + return disk.Info{}, err + } + return info, nil +} + +// MakeVol - create a volume on a remote disk. func (n networkStorage) MakeVol(volume string) error { reply := GenericReply{} - if err := n.rpcClient.Call("Storage.MakeVolHandler", volume, &reply); err != nil { + args := GenericVolArgs{Vol: volume} + if err := n.rpcClient.Call("Storage.MakeVolHandler", &args, &reply); err != nil { return toStorageErr(err) } return nil } -// ListVols - List all volumes. +// ListVols - List all volumes on a remote disk. func (n networkStorage) ListVols() (vols []VolInfo, err error) { ListVols := ListVolsReply{} - err = n.rpcClient.Call("Storage.ListVolsHandler", "", &ListVols) + err = n.rpcClient.Call("Storage.ListVolsHandler", &GenericArgs{}, &ListVols) if err != nil { return nil, err } @@ -127,7 +153,8 @@ func (n networkStorage) ListVols() (vols []VolInfo, err error) { // StatVol - get current Stat volume info. func (n networkStorage) StatVol(volume string) (volInfo VolInfo, err error) { - if err = n.rpcClient.Call("Storage.StatVolHandler", volume, &volInfo); err != nil { + args := GenericVolArgs{Vol: volume} + if err = n.rpcClient.Call("Storage.StatVolHandler", &args, &volInfo); err != nil { return VolInfo{}, toStorageErr(err) } return volInfo, nil @@ -136,7 +163,8 @@ func (n networkStorage) StatVol(volume string) (volInfo VolInfo, err error) { // DeleteVol - Delete a volume. func (n networkStorage) DeleteVol(volume string) error { reply := GenericReply{} - if err := n.rpcClient.Call("Storage.DeleteVolHandler", volume, &reply); err != nil { + args := GenericVolArgs{Vol: volume} + if err := n.rpcClient.Call("Storage.DeleteVolHandler", &args, &reply); err != nil { return toStorageErr(err) } return nil @@ -147,7 +175,7 @@ func (n networkStorage) DeleteVol(volume string) error { // CreateFile - create file. func (n networkStorage) AppendFile(volume, path string, buffer []byte) (err error) { reply := GenericReply{} - if err = n.rpcClient.Call("Storage.AppendFileHandler", AppendFileArgs{ + if err = n.rpcClient.Call("Storage.AppendFileHandler", &AppendFileArgs{ Vol: volume, Path: path, Buffer: buffer, @@ -159,7 +187,7 @@ func (n networkStorage) AppendFile(volume, path string, buffer []byte) (err erro // StatFile - get latest Stat information for a file at path. func (n networkStorage) StatFile(volume, path string) (fileInfo FileInfo, err error) { - if err = n.rpcClient.Call("Storage.StatFileHandler", StatFileArgs{ + if err = n.rpcClient.Call("Storage.StatFileHandler", &StatFileArgs{ Vol: volume, Path: path, }, &fileInfo); err != nil { @@ -173,7 +201,7 @@ func (n networkStorage) StatFile(volume, path string) (fileInfo FileInfo, err er // This API is meant to be used on files which have small memory footprint, do // not use this on large files as it would cause server to crash. func (n networkStorage) ReadAll(volume, path string) (buf []byte, err error) { - if err = n.rpcClient.Call("Storage.ReadAllHandler", ReadAllArgs{ + if err = n.rpcClient.Call("Storage.ReadAllHandler", &ReadAllArgs{ Vol: volume, Path: path, }, &buf); err != nil { @@ -184,20 +212,22 @@ func (n networkStorage) ReadAll(volume, path string) (buf []byte, err error) { // ReadFile - reads a file. func (n networkStorage) ReadFile(volume string, path string, offset int64, buffer []byte) (m int64, err error) { - if err = n.rpcClient.Call("Storage.ReadFileHandler", ReadFileArgs{ + var result []byte + err = n.rpcClient.Call("Storage.ReadFileHandler", &ReadFileArgs{ Vol: volume, Path: path, Offset: offset, - Buffer: buffer, - }, &m); err != nil { - return 0, toStorageErr(err) - } - return m, nil + Size: len(buffer), + }, &result) + // Copy results to buffer. + copy(buffer, result) + // Return length of result, err if any. + return int64(len(result)), toStorageErr(err) } // ListDir - list all entries at prefix. func (n networkStorage) ListDir(volume, path string) (entries []string, err error) { - if err = n.rpcClient.Call("Storage.ListDirHandler", ListDirArgs{ + if err = n.rpcClient.Call("Storage.ListDirHandler", &ListDirArgs{ Vol: volume, Path: path, }, &entries); err != nil { @@ -210,7 +240,7 @@ func (n networkStorage) ListDir(volume, path string) (entries []string, err erro // DeleteFile - Delete a file at path. func (n networkStorage) DeleteFile(volume, path string) (err error) { reply := GenericReply{} - if err = n.rpcClient.Call("Storage.DeleteFileHandler", DeleteFileArgs{ + if err = n.rpcClient.Call("Storage.DeleteFileHandler", &DeleteFileArgs{ Vol: volume, Path: path, }, &reply); err != nil { @@ -222,7 +252,7 @@ func (n networkStorage) DeleteFile(volume, path string) (err error) { // RenameFile - Rename file. func (n networkStorage) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) { reply := GenericReply{} - if err = n.rpcClient.Call("Storage.RenameFileHandler", RenameFileArgs{ + if err = n.rpcClient.Call("Storage.RenameFileHandler", &RenameFileArgs{ SrcVol: srcVolume, SrcPath: srcPath, DstVol: dstVolume, diff --git a/cmd/rpc-server-datatypes.go b/cmd/storage-rpc-server-datatypes.go similarity index 76% rename from cmd/rpc-server-datatypes.go rename to cmd/storage-rpc-server-datatypes.go index f036c1fb7..58fa36581 100644 --- a/cmd/rpc-server-datatypes.go +++ b/cmd/storage-rpc-server-datatypes.go @@ -16,11 +16,14 @@ package cmd -// GenericReply represents any generic RPC reply. -type GenericReply struct{} +// GenericVolArgs - generic volume args. +type GenericVolArgs struct { + // Authentication token generated by Login. + GenericArgs -// GenericArgs represents any generic RPC arguments. -type GenericArgs struct{} + // Name of the volume. + Vol string +} // ListVolsReply represents list of vols RPC reply. type ListVolsReply struct { @@ -30,6 +33,9 @@ type ListVolsReply struct { // ReadAllArgs represents read all RPC arguments. type ReadAllArgs struct { + // Authentication token generated by Login. + GenericArgs + // Name of the volume. Vol string @@ -39,6 +45,9 @@ type ReadAllArgs struct { // ReadFileArgs represents read file RPC arguments. type ReadFileArgs struct { + // Authentication token generated by Login. + GenericArgs + // Name of the volume. Vol string @@ -48,12 +57,15 @@ type ReadFileArgs struct { // Starting offset to start reading into Buffer. Offset int64 - // Data buffer read from the path at offset. - Buffer []byte + // Data size read from the path at offset. + Size int } // AppendFileArgs represents append file RPC arguments. type AppendFileArgs struct { + // Authentication token generated by Login. + GenericArgs + // Name of the volume. Vol string @@ -66,6 +78,9 @@ type AppendFileArgs struct { // StatFileArgs represents stat file RPC arguments. type StatFileArgs struct { + // Authentication token generated by Login. + GenericArgs + // Name of the volume. Vol string @@ -75,6 +90,9 @@ type StatFileArgs struct { // DeleteFileArgs represents delete file RPC arguments. type DeleteFileArgs struct { + // Authentication token generated by Login. + GenericArgs + // Name of the volume. Vol string @@ -84,6 +102,9 @@ type DeleteFileArgs struct { // ListDirArgs represents list contents RPC arguments. type ListDirArgs struct { + // Authentication token generated by Login. + GenericArgs + // Name of the volume. Vol string @@ -93,6 +114,9 @@ type ListDirArgs struct { // RenameFileArgs represents rename file RPC arguments. type RenameFileArgs struct { + // Authentication token generated by Login. + GenericArgs + // Name of source volume. SrcVol string diff --git a/cmd/storage-rpc-server.go b/cmd/storage-rpc-server.go new file mode 100644 index 000000000..8b733a2e5 --- /dev/null +++ b/cmd/storage-rpc-server.go @@ -0,0 +1,252 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "bytes" + "io" + "net/rpc" + "path" + "strings" + + router "github.com/gorilla/mux" + "github.com/minio/minio/pkg/disk" +) + +// Storage server implements rpc primitives to facilitate exporting a +// disk over a network. +type storageServer struct { + storage StorageAPI + path string +} + +/// Auth operations + +// Login - login handler. +func (s *storageServer) LoginHandler(args *RPCLoginArgs, reply *RPCLoginReply) error { + jwt, err := newJWT(defaultTokenExpiry) + if err != nil { + return err + } + if err = jwt.Authenticate(args.Username, args.Password); err != nil { + return err + } + token, err := jwt.GenerateToken(args.Username) + if err != nil { + return err + } + reply.Token = token + reply.ServerVersion = Version + return nil +} + +/// Storage operations handlers. + +// DiskInfoHandler - disk info handler is rpc wrapper for DiskInfo operation. +func (s *storageServer) DiskInfoHandler(args *GenericArgs, reply *disk.Info) error { + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + info, err := s.storage.DiskInfo() + *reply = info + return err +} + +/// Volume operations handlers. + +// MakeVolHandler - make vol handler is rpc wrapper for MakeVol operation. +func (s *storageServer) MakeVolHandler(args *GenericVolArgs, reply *GenericReply) error { + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + return s.storage.MakeVol(args.Vol) +} + +// ListVolsHandler - list vols handler is rpc wrapper for ListVols operation. +func (s *storageServer) ListVolsHandler(args *GenericArgs, reply *ListVolsReply) error { + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + vols, err := s.storage.ListVols() + if err != nil { + return err + } + reply.Vols = vols + return nil +} + +// StatVolHandler - stat vol handler is a rpc wrapper for StatVol operation. +func (s *storageServer) StatVolHandler(args *GenericVolArgs, reply *VolInfo) error { + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + volInfo, err := s.storage.StatVol(args.Vol) + if err != nil { + return err + } + *reply = volInfo + return nil +} + +// DeleteVolHandler - delete vol handler is a rpc wrapper for +// DeleteVol operation. +func (s *storageServer) DeleteVolHandler(args *GenericVolArgs, reply *GenericReply) error { + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + return s.storage.DeleteVol(args.Vol) +} + +/// File operations + +// StatFileHandler - stat file handler is rpc wrapper to stat file. +func (s *storageServer) StatFileHandler(args *StatFileArgs, reply *FileInfo) error { + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + fileInfo, err := s.storage.StatFile(args.Vol, args.Path) + if err != nil { + return err + } + *reply = fileInfo + return nil +} + +// ListDirHandler - list directory handler is rpc wrapper to list dir. +func (s *storageServer) ListDirHandler(args *ListDirArgs, reply *[]string) error { + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + entries, err := s.storage.ListDir(args.Vol, args.Path) + if err != nil { + return err + } + *reply = entries + return nil +} + +// ReadAllHandler - read all handler is rpc wrapper to read all storage API. +func (s *storageServer) ReadAllHandler(args *ReadFileArgs, reply *[]byte) error { + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + buf, err := s.storage.ReadAll(args.Vol, args.Path) + if err != nil { + return err + } + *reply = buf + return nil +} + +// ReadFileHandler - read file handler is rpc wrapper to read file. +func (s *storageServer) ReadFileHandler(args *ReadFileArgs, reply *[]byte) (err error) { + defer func() { + if r := recover(); r != nil { + // Recover any panic and return ErrCacheFull. + err = bytes.ErrTooLarge + } + }() // Do not crash the server. + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + // Allocate the requested buffer from the client. + *reply = make([]byte, args.Size) + var n int64 + n, err = s.storage.ReadFile(args.Vol, args.Path, args.Offset, *reply) + // Sending an error over the rpc layer, would cause unmarshalling to fail. In situations + // when we have short read i.e `io.ErrUnexpectedEOF` treat it as good condition and copy + // the buffer properly. + if err == io.ErrUnexpectedEOF { + // Reset to nil as good condition. + err = nil + } + *reply = (*reply)[0:n] + return err +} + +// AppendFileHandler - append file handler is rpc wrapper to append file. +func (s *storageServer) AppendFileHandler(args *AppendFileArgs, reply *GenericReply) error { + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + return s.storage.AppendFile(args.Vol, args.Path, args.Buffer) +} + +// DeleteFileHandler - delete file handler is rpc wrapper to delete file. +func (s *storageServer) DeleteFileHandler(args *DeleteFileArgs, reply *GenericReply) error { + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + return s.storage.DeleteFile(args.Vol, args.Path) +} + +// RenameFileHandler - rename file handler is rpc wrapper to rename file. +func (s *storageServer) RenameFileHandler(args *RenameFileArgs, reply *GenericReply) error { + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + return s.storage.RenameFile(args.SrcVol, args.SrcPath, args.DstVol, args.DstPath) +} + +// Initialize new storage rpc. +func newRPCServer(serverConfig serverCmdConfig) (servers []*storageServer, err error) { + // Initialize posix storage API. + exports := serverConfig.disks + ignoredExports := serverConfig.ignoredDisks + + // Save ignored disks in a map + skipDisks := make(map[string]bool) + for _, ignoredExport := range ignoredExports { + skipDisks[ignoredExport] = true + } + for _, export := range exports { + if skipDisks[export] { + continue + } + // e.g server:/mnt/disk1 + if isLocalStorage(export) { + if idx := strings.LastIndex(export, ":"); idx != -1 { + export = export[idx+1:] + } + var storage StorageAPI + storage, err = newPosix(export) + if err != nil && err != errDiskNotFound { + return nil, err + } + if idx := strings.LastIndex(export, ":"); idx != -1 { + export = export[idx+1:] + } + servers = append(servers, &storageServer{ + storage: storage, + path: export, + }) + } + } + return servers, err +} + +// registerStorageRPCRouter - register storage rpc router. +func registerStorageRPCRouters(mux *router.Router, stServers []*storageServer) { + // Create a unique route for each disk exported from this node. + for _, stServer := range stServers { + storageRPCServer := rpc.NewServer() + storageRPCServer.RegisterName("Storage", stServer) + // Add minio storage routes. + storageRouter := mux.PathPrefix(reservedBucket).Subrouter() + storageRouter.Path(path.Join("/storage", stServer.path)).Handler(storageRPCServer) + } +} diff --git a/cmd/strconv-bytes_test.go b/cmd/strconv-bytes_test.go index fd0206d16..1393db43d 100644 --- a/cmd/strconv-bytes_test.go +++ b/cmd/strconv-bytes_test.go @@ -91,7 +91,7 @@ func TestByteErrors(t *testing.T) { t.Errorf("Expected error, got %v", got) } // Empty string. - got, err = strconvBytes("") + _, err = strconvBytes("") if err == nil { t.Errorf("Expected error parsing nothing") } diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index c538fd908..b39a887dc 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -45,7 +45,35 @@ import ( // Tests should initNSLock only once. func init() { // Initialize name space lock. - initNSLock() + isDist := false + initNSLock(isDist) +} + +func prepareFS() (ObjectLayer, string, error) { + fsDirs, err := getRandomDisks(1) + if err != nil { + return nil, "", err + } + obj, err := getSingleNodeObjectLayer(fsDirs[0]) + if err != nil { + removeRoots(fsDirs) + return nil, "", err + } + return obj, fsDirs[0], nil +} + +func prepareXL() (ObjectLayer, []string, error) { + nDisks := 16 + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + return nil, nil, err + } + obj, err := getXLObjectLayer(fsDirs) + if err != nil { + removeRoots(fsDirs) + return nil, nil, err + } + return obj, fsDirs, nil } // TestErrHandler - Golang Testing.T and Testing.B, and gocheck.C satisfy this interface. @@ -109,6 +137,7 @@ type TestServer struct { AccessKey string SecretKey string Server *httptest.Server + Obj ObjectLayer } // Starts the test server and returns the TestServer instance. @@ -116,7 +145,67 @@ func StartTestServer(t TestErrHandler, instanceType string) TestServer { // create an instance of TestServer. testServer := TestServer{} // create temporary backend for the test server. - _, erasureDisks, err := makeTestBackend(instanceType) + nDisks := 16 + disks, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal("Failed to create disks for the backend") + } + + root, err := newTestConfig("us-east-1") + if err != nil { + t.Fatalf("%s", err) + } + + // Test Server needs to start before formatting of disks. + // Get credential. + credentials := serverConfig.GetCredential() + + testServer.Root = root + testServer.Disks = disks + testServer.AccessKey = credentials.AccessKeyID + testServer.SecretKey = credentials.SecretAccessKey + // Run TestServer. + testServer.Server = httptest.NewServer(configureServerHandler(serverCmdConfig{disks: disks})) + + objLayer, err := makeTestBackend(disks, instanceType) + if err != nil { + t.Fatalf("Failed obtaining Temp Backend: %s", err) + } + testServer.Obj = objLayer + objLayerMutex.Lock() + globalObjectAPI = objLayer + objLayerMutex.Unlock() + return testServer +} + +// Initializes control RPC end points. +// The object Layer will be a temp back used for testing purpose. +func initTestControlRPCEndPoint(objectLayer ObjectLayer) http.Handler { + // Initialize Web. + + controllerHandlers := &controllerAPIHandlers{ + ObjectAPI: func() ObjectLayer { return objectLayer }, + } + + // Initialize router. + muxRouter := router.NewRouter() + registerControllerRPCRouter(muxRouter, controllerHandlers) + return muxRouter +} + +// StartTestRPCServer - Creates a temp XL/FS backend and initializes control RPC end points, +// then starts a test server with those control RPC end points registered. +func StartTestRPCServer(t TestErrHandler, instanceType string) TestServer { + // create temporary backend for the test server. + nDisks := 16 + disks, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal("Failed to create disks for the backend") + } + // create an instance of TestServer. + testRPCServer := TestServer{} + // create temporary backend for the test server. + objLayer, err := makeTestBackend(disks, instanceType) if err != nil { t.Fatalf("Failed obtaining Temp Backend: %s", err) @@ -130,14 +219,15 @@ func StartTestServer(t TestErrHandler, instanceType string) TestServer { // Get credential. credentials := serverConfig.GetCredential() - testServer.Root = root - testServer.Disks = erasureDisks - testServer.AccessKey = credentials.AccessKeyID - testServer.SecretKey = credentials.SecretAccessKey + testRPCServer.Root = root + testRPCServer.Disks = disks + testRPCServer.AccessKey = credentials.AccessKeyID + testRPCServer.SecretKey = credentials.SecretAccessKey + testRPCServer.Obj = objLayer // Run TestServer. - testServer.Server = httptest.NewServer(configureServerHandler(serverCmdConfig{disks: erasureDisks})) + testRPCServer.Server = httptest.NewServer(initTestControlRPCEndPoint(objLayer)) - return testServer + return testRPCServer } // Configure the server for the test run. @@ -177,6 +267,208 @@ func (testServer TestServer) Stop() { testServer.Server.Close() } +// Sign given request using Signature V4. +func signStreamingRequest(req *http.Request, accessKey, secretKey string) (string, error) { + // Get hashed payload. + hashedPayload := req.Header.Get("x-amz-content-sha256") + if hashedPayload == "" { + return "", fmt.Errorf("Invalid hashed payload.") + } + + currTime := time.Now().UTC() + // Set x-amz-date. + req.Header.Set("x-amz-date", currTime.Format(iso8601Format)) + + // Get header map. + headerMap := make(map[string][]string) + for k, vv := range req.Header { + // If request header key is not in ignored headers, then add it. + if _, ok := ignoredStreamingHeaders[http.CanonicalHeaderKey(k)]; !ok { + headerMap[strings.ToLower(k)] = vv + } + } + + // Get header keys. + headers := []string{"host"} + for k := range headerMap { + headers = append(headers, k) + } + sort.Strings(headers) + + // Get canonical headers. + var buf bytes.Buffer + for _, k := range headers { + buf.WriteString(k) + buf.WriteByte(':') + switch { + case k == "host": + buf.WriteString(req.URL.Host) + fallthrough + default: + for idx, v := range headerMap[k] { + if idx > 0 { + buf.WriteByte(',') + } + buf.WriteString(v) + } + buf.WriteByte('\n') + } + } + canonicalHeaders := buf.String() + + // Get signed headers. + signedHeaders := strings.Join(headers, ";") + + // Get canonical query string. + req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1) + + // Get canonical URI. + canonicalURI := getURLEncodedName(req.URL.Path) + + // Get canonical request. + // canonicalRequest = + // \n + // \n + // \n + // \n + // \n + // + // + canonicalRequest := strings.Join([]string{ + req.Method, + canonicalURI, + req.URL.RawQuery, + canonicalHeaders, + signedHeaders, + hashedPayload, + }, "\n") + + // Get scope. + scope := strings.Join([]string{ + currTime.Format(yyyymmdd), + "us-east-1", + "s3", + "aws4_request", + }, "/") + + stringToSign := "AWS4-HMAC-SHA256" + "\n" + currTime.Format(iso8601Format) + "\n" + stringToSign = stringToSign + scope + "\n" + stringToSign = stringToSign + hex.EncodeToString(sum256([]byte(canonicalRequest))) + + date := sumHMAC([]byte("AWS4"+secretKey), []byte(currTime.Format(yyyymmdd))) + region := sumHMAC(date, []byte("us-east-1")) + service := sumHMAC(region, []byte("s3")) + signingKey := sumHMAC(service, []byte("aws4_request")) + + signature := hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) + + // final Authorization header + parts := []string{ + "AWS4-HMAC-SHA256" + " Credential=" + accessKey + "/" + scope, + "SignedHeaders=" + signedHeaders, + "Signature=" + signature, + } + auth := strings.Join(parts, ", ") + req.Header.Set("Authorization", auth) + + return signature, nil +} + +// Returns new HTTP request object. +func newTestStreamingRequest(method, urlStr string, dataLength, chunkSize int64, body io.ReadSeeker) (*http.Request, error) { + if method == "" { + method = "POST" + } + + req, err := http.NewRequest(method, urlStr, nil) + if err != nil { + return nil, err + } + + if body == nil { + // this is added to avoid panic during ioutil.ReadAll(req.Body). + // th stack trace can be found here https://github.com/minio/minio/pull/2074 . + // This is very similar to https://github.com/golang/go/issues/7527. + req.Body = ioutil.NopCloser(bytes.NewReader([]byte(""))) + } + + contentLength := calculateStreamContentLength(dataLength, chunkSize) + + req.Header.Set("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD") + req.Header.Set("content-encoding", "aws-chunked") + req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") + + req.Header.Set("x-amz-decoded-content-length", strconv.FormatInt(dataLength, 10)) + req.Header.Set("content-length", strconv.FormatInt(contentLength, 10)) + + // Seek back to beginning. + body.Seek(0, 0) + // Add body + req.Body = ioutil.NopCloser(body) + req.ContentLength = contentLength + + return req, nil +} + +// Returns new HTTP request object signed with streaming signature v4. +func newTestStreamingSignedRequest(method, urlStr string, contentLength, chunkSize int64, body io.ReadSeeker, accessKey, secretKey string) (*http.Request, error) { + req, err := newTestStreamingRequest(method, urlStr, contentLength, chunkSize, body) + if err != nil { + return nil, err + } + + signature, err := signStreamingRequest(req, accessKey, secretKey) + if err != nil { + return nil, err + } + + var stream []byte + var buffer []byte + body.Seek(0, 0) + for { + buffer = make([]byte, chunkSize) + n, err := body.Read(buffer) + if err != nil && err != io.EOF { + return nil, err + } + + currTime := time.Now().UTC() + // Get scope. + scope := strings.Join([]string{ + currTime.Format(yyyymmdd), + "us-east-1", + "s3", + "aws4_request", + }, "/") + + stringToSign := "AWS4-HMAC-SHA256-PAYLOAD" + "\n" + stringToSign = stringToSign + currTime.Format(iso8601Format) + "\n" + stringToSign = stringToSign + scope + "\n" + stringToSign = stringToSign + signature + "\n" + stringToSign = stringToSign + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + "\n" // hex(sum256("")) + stringToSign = stringToSign + hex.EncodeToString(sum256(buffer[:n])) + + date := sumHMAC([]byte("AWS4"+secretKey), []byte(currTime.Format(yyyymmdd))) + region := sumHMAC(date, []byte("us-east-1")) + service := sumHMAC(region, []byte("s3")) + signingKey := sumHMAC(service, []byte("aws4_request")) + + signature = hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) + + stream = append(stream, []byte(fmt.Sprintf("%x", n)+";chunk-signature="+signature+"\r\n")...) + stream = append(stream, buffer[:n]...) + stream = append(stream, []byte("\r\n")...) + + if n <= 0 { + break + } + + } + + req.Body = ioutil.NopCloser(bytes.NewReader(stream)) + return req, nil +} + // Sign given request using Signature V4. func signRequest(req *http.Request, accessKey, secretKey string) error { // Get hashed payload. @@ -285,6 +577,11 @@ func signRequest(req *http.Request, accessKey, secretKey string) error { return nil } +// getCredential generate a credential string. +func getCredential(accessKeyID, location string, t time.Time) string { + return accessKeyID + "/" + getScope(t, location) +} + // Returns new HTTP request object. func newTestRequest(method, urlStr string, contentLength int64, body io.ReadSeeker) (*http.Request, error) { if method == "" { @@ -336,6 +633,11 @@ func newTestSignedRequest(method, urlStr string, contentLength int64, body io.Re return nil, err } + // Anonymous request return quickly. + if accessKey == "" || secretKey == "" { + return req, nil + } + err = signRequest(req, accessKey, secretKey) if err != nil { return nil, err @@ -378,6 +680,9 @@ func newTestWebRPCRequest(rpcMethod string, authorization string, data interface } encapsulatedData := genericJSON{JSONRPC: "2.0", ID: "1", Method: rpcMethod, Params: data} jsonData, err := json.Marshal(encapsulatedData) + if err != nil { + return nil, err + } req, err := newWebRPCRequest(rpcMethod, authorization, bytes.NewReader(jsonData)) if err != nil { return nil, err @@ -416,24 +721,24 @@ func getTestWebRPCResponse(resp *httptest.ResponseRecorder, data interface{}) er // if the option is // FS: Returns a temp single disk setup initializes FS Backend. // XL: Returns a 16 temp single disk setup and initializse XL Backend. -func makeTestBackend(instanceType string) (ObjectLayer, []string, error) { +func makeTestBackend(disks []string, instanceType string) (ObjectLayer, error) { switch instanceType { case "FS": - objLayer, fsroot, err := getSingleNodeObjectLayer() + objLayer, err := getSingleNodeObjectLayer(disks[0]) if err != nil { - return nil, []string{}, err + return nil, err } - return objLayer, []string{fsroot}, err + return objLayer, err case "XL": - objectLayer, erasureDisks, err := getXLObjectLayer() + objectLayer, err := getXLObjectLayer(disks) if err != nil { - return nil, []string{}, err + return nil, err } - return objectLayer, erasureDisks, err + return objectLayer, err default: errMsg := "Invalid instance type, Only FS and XL are valid options" - return nil, []string{}, fmt.Errorf("Failed obtaining Temp XL layer: %s", errMsg) + return nil, fmt.Errorf("Failed obtaining Temp XL layer: %s", errMsg) } } @@ -693,7 +998,13 @@ func getHEADBucketURL(endPoint, bucketName string) string { // return URL for deleting the bucket. func getDeleteBucketURL(endPoint, bucketName string) string { return makeTestTargetURL(endPoint, bucketName, "", url.Values{}) +} +// return URL For fetching location of the bucket. +func getBucketLocationURL(endPoint, bucketName string) string { + queryValue := url.Values{} + queryValue.Set("location", "") + return makeTestTargetURL(endPoint, bucketName, "", queryValue) } // return URL for listing objects in the bucket with V1 legacy API. @@ -740,14 +1051,26 @@ func getAbortMultipartUploadURL(endPoint, bucketName, objectName, uploadID strin return makeTestTargetURL(endPoint, bucketName, objectName, queryValue) } -// return URL for a new multipart upload. +// return URL for a listing pending multipart uploads. func getListMultipartURL(endPoint, bucketName string) string { queryValue := url.Values{} queryValue.Set("uploads", "") return makeTestTargetURL(endPoint, bucketName, "", queryValue) } -// return URL for a new multipart upload. +// return URL for listing pending multipart uploads with parameters. +func getListMultipartUploadsURLWithParams(endPoint, bucketName, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads string) string { + queryValue := url.Values{} + queryValue.Set("uploads", "") + queryValue.Set("prefix", prefix) + queryValue.Set("delimiter", delimiter) + queryValue.Set("key-marker", keyMarker) + queryValue.Set("upload-id-marker", uploadIDMarker) + queryValue.Set("max-uploads", maxUploads) + return makeTestTargetURL(endPoint, bucketName, "", queryValue) +} + +// return URL for a listing parts on a given upload id. func getListMultipartURLWithParams(endPoint, bucketName, objectName, uploadID, maxParts string) string { queryValues := url.Values{} queryValues.Set("uploadId", uploadID) @@ -768,21 +1091,30 @@ func getTestRoot() (string, error) { return ioutil.TempDir(os.TempDir(), "api-") } -// getXLObjectLayer - Instantiates XL object layer and returns it. -func getXLObjectLayer() (ObjectLayer, []string, error) { - var nDisks = 16 // Maximum disks. +// getRandomDisks - Creates a slice of N random disks, each of the form - minio-XXX +func getRandomDisks(N int) ([]string, error) { var erasureDisks []string - for i := 0; i < nDisks; i++ { + for i := 0; i < N; i++ { path, err := ioutil.TempDir(os.TempDir(), "minio-") if err != nil { - return nil, nil, err + // Remove directories created so far. + removeRoots(erasureDisks) + return nil, err } erasureDisks = append(erasureDisks, path) } + return erasureDisks, nil +} +// getXLObjectLayer - Instantiates XL object layer and returns it. +func getXLObjectLayer(erasureDisks []string) (ObjectLayer, error) { + err := formatDisks(erasureDisks, nil) + if err != nil { + return nil, err + } objLayer, err := newXLObjects(erasureDisks, nil) if err != nil { - return nil, nil, err + return nil, err } // Disabling the cache for integration tests. // Should use the object layer tests for validating cache. @@ -790,23 +1122,17 @@ func getXLObjectLayer() (ObjectLayer, []string, error) { xl.objCacheEnabled = false } - return objLayer, erasureDisks, nil + return objLayer, nil } // getSingleNodeObjectLayer - Instantiates single node object layer and returns it. -func getSingleNodeObjectLayer() (ObjectLayer, string, error) { - // Make a temporary directory to use as the obj. - fsDir, err := ioutil.TempDir("", "minio-") +func getSingleNodeObjectLayer(disk string) (ObjectLayer, error) { + // Create the object layer. + objLayer, err := newFSObjects(disk) if err != nil { - return nil, "", err + return nil, err } - - // Create the obj. - objLayer, err := newFSObjects(fsDir) - if err != nil { - return nil, "", err - } - return objLayer, fsDir, nil + return objLayer, nil } // removeRoots - Cleans up initialized directories during tests. @@ -826,6 +1152,66 @@ func removeDiskN(disks []string, n int) { } } +// creates a bucket for the tests and returns the bucket name. +// initializes the specified API endpoints for the tests. +// initialies the root and returns its path. +// return credentials. +func initAPIHandlerTest(obj ObjectLayer, endPoints []string) (bucketName, rootPath string, apiRouter http.Handler, err error) { + // get random bucket name. + bucketName = getRandomBucketName() + + // Create bucket. + err = obj.MakeBucket(bucketName) + if err != nil { + // failed to create newbucket, return err. + return "", "", nil, err + } + // Register the API end points with XL/FS object layer. + // Registering only the GetObject handler. + apiRouter = initTestAPIEndPoints(obj, endPoints) + // initialize the server and obtain the credentials and root. + // credentials are necessary to sign the HTTP request. + rootPath, err = newTestConfig("us-east-1") + if err != nil { + return "", "", nil, err + } + + return bucketName, rootPath, apiRouter, nil +} + +// ExecObjectLayerAPITest - executes object layer API tests. +// Creates single node and XL ObjectLayer instance, registers the specified API end points and runs test for both the layers. +func ExecObjectLayerAPITest(t TestErrHandler, objAPITest objAPITestType, endPoints []string) { + objLayer, fsDir, err := prepareFS() + if err != nil { + t.Fatalf("Initialization of object layer failed for single node setup: %s", err) + } + bucketFS, fsRoot, fsAPIRouter, err := initAPIHandlerTest(objLayer, endPoints) + if err != nil { + t.Fatalf("Initialzation of API handler tests failed: %s", err) + } + credentials := serverConfig.GetCredential() + // Executing the object layer tests for single node setup. + objAPITest(objLayer, singleNodeTestStr, bucketFS, fsAPIRouter, credentials, t) + + objLayer, xlDisks, err := prepareXL() + if err != nil { + t.Fatalf("Initialization of object layer failed for XL setup: %s", err) + } + bucketXL, xlRoot, xlAPIRouter, err := initAPIHandlerTest(objLayer, endPoints) + if err != nil { + t.Fatalf("Initialzation of API handler tests failed: %s", err) + } + credentials = serverConfig.GetCredential() + // Executing the object layer tests for XL. + objAPITest(objLayer, xLTestStr, bucketXL, xlAPIRouter, credentials, t) + defer removeRoots(append(xlDisks, fsDir, fsRoot, xlRoot)) +} + +// function to be passed to ExecObjectLayerAPITest, for executing object layr API handler tests. +type objAPITestType func(obj ObjectLayer, instanceType string, bucketName string, + apiRouter http.Handler, credentials credential, t TestErrHandler) + // Regular object test type. type objTestType func(obj ObjectLayer, instanceType string, t TestErrHandler) @@ -835,14 +1221,14 @@ type objTestDiskNotFoundType func(obj ObjectLayer, instanceType string, dirs []s // ExecObjectLayerTest - executes object layer tests. // Creates single node and XL ObjectLayer instance and runs test for both the layers. func ExecObjectLayerTest(t TestErrHandler, objTest objTestType) { - objLayer, fsDir, err := getSingleNodeObjectLayer() + objLayer, fsDir, err := prepareFS() if err != nil { t.Fatalf("Initialization of object layer failed for single node setup: %s", err) } // Executing the object layer tests for single node setup. objTest(objLayer, singleNodeTestStr, t) - objLayer, fsDirs, err := getXLObjectLayer() + objLayer, fsDirs, err := prepareXL() if err != nil { t.Fatalf("Initialization of object layer failed for XL setup: %s", err) } @@ -854,7 +1240,7 @@ func ExecObjectLayerTest(t TestErrHandler, objTest objTestType) { // ExecObjectLayerDiskNotFoundTest - executes object layer tests while deleting // disks in between tests. Creates XL ObjectLayer instance and runs test for XL layer. func ExecObjectLayerDiskNotFoundTest(t *testing.T, objTest objTestDiskNotFoundType) { - objLayer, fsDirs, err := getXLObjectLayer() + objLayer, fsDirs, err := prepareXL() if err != nil { t.Fatalf("Initialization of object layer failed for XL setup: %s", err) } @@ -869,13 +1255,18 @@ type objTestStaleFilesType func(obj ObjectLayer, instanceType string, dirs []str // ExecObjectLayerStaleFilesTest - executes object layer tests those leaves stale // files/directories under .minio/tmp. Creates XL ObjectLayer instance and runs test for XL layer. func ExecObjectLayerStaleFilesTest(t *testing.T, objTest objTestStaleFilesType) { - objLayer, fsDirs, err := getXLObjectLayer() + nDisks := 16 + erasureDisks, err := getRandomDisks(nDisks) + if err != nil { + t.Fatalf("Initialization of disks for XL setup: %s", err) + } + objLayer, err := getXLObjectLayer(erasureDisks) if err != nil { t.Fatalf("Initialization of object layer failed for XL setup: %s", err) } // Executing the object layer tests for XL. - objTest(objLayer, xLTestStr, fsDirs, t) - defer removeRoots(fsDirs) + objTest(objLayer, xLTestStr, erasureDisks, t) + defer removeRoots(erasureDisks) } // Takes in XL/FS object layer, and the list of API end points to be tested/required, registers the API end points and returns the HTTP handler. @@ -888,7 +1279,7 @@ func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Hand // All object storage operations are registered as HTTP handlers on `objectAPIHandlers`. // When the handlers get a HTTP request they use the underlyting ObjectLayer to perform operations. api := objectAPIHandlers{ - ObjectAPI: objLayer, + ObjectAPI: func() ObjectLayer { return objLayer }, } // API Router. apiRouter := muxRouter.NewRoute().PathPrefix("/").Subrouter() @@ -903,24 +1294,25 @@ func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Hand // Register GetObject handler. case "CopyObject`": bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectHandler) - // Register PutBucket Policy handler. case "PutBucketPolicy": bucket.Methods("PUT").HandlerFunc(api.PutBucketPolicyHandler).Queries("policy", "") - // Register Delete bucket HTTP policy handler. case "DeleteBucketPolicy": bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketPolicyHandler).Queries("policy", "") - - // Register Get Bucket policy HTTP Handler. + // Register Get Bucket policy HTTP Handler. case "GetBucketPolicy": bucket.Methods("GET").HandlerFunc(api.GetBucketPolicyHandler).Queries("policy", "") - - // Register Post Bucket policy function. - case "PostBucketPolicy": - bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(api.PostPolicyBucketHandler) - - // Register all api endpoints by default. + // Register GetBucketLocation handler. + case "GetBucketLocation": + bucket.Methods("GET").HandlerFunc(api.GetBucketLocationHandler).Queries("location", "") + // Register HeadBucket handler. + case "HeadBucket": + bucket.Methods("HEAD").HandlerFunc(api.HeadBucketHandler) + // Register ListMultipartUploads handler. + case "ListMultipartUploads": + bucket.Methods("GET").HandlerFunc(api.ListMultipartUploadsHandler).Queries("uploads", "") + // Register all api endpoints by default. default: registerAPIRouter(muxRouter, api) // No need to register any more end points, all the end points are registered. @@ -930,10 +1322,11 @@ func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Hand return muxRouter } +// Initialize Web RPC Handlers for testing func initTestWebRPCEndPoint(objLayer ObjectLayer) http.Handler { // Initialize Web. webHandlers := &webAPIHandlers{ - ObjectAPI: objLayer, + ObjectAPI: func() ObjectLayer { return objLayer }, } // Initialize router. diff --git a/cmd/tree-walk.go b/cmd/tree-walk.go index d3ca940ad..83b436b94 100644 --- a/cmd/tree-walk.go +++ b/cmd/tree-walk.go @@ -148,7 +148,7 @@ func listDirFactory(isLeaf isLeafFunc, disks ...StorageAPI) listDirFunc { break } // Return error at the end. - return nil, false, err + return nil, false, traceError(err) } return listDir } @@ -173,7 +173,7 @@ func doTreeWalk(bucket, prefixDir, entryPrefixMatch, marker string, recursive bo if err != nil { select { case <-endWalkCh: - return errWalkAbort + return traceError(errWalkAbort) case resultCh <- treeWalkResult{err: err}: return err } @@ -235,7 +235,7 @@ func doTreeWalk(bucket, prefixDir, entryPrefixMatch, marker string, recursive bo isEOF := ((i == len(entries)-1) && isEnd) select { case <-endWalkCh: - return errWalkAbort + return traceError(errWalkAbort) case resultCh <- treeWalkResult{entry: pathJoin(prefixDir, entry), end: isEOF}: } } diff --git a/cmd/tree-walk_test.go b/cmd/tree-walk_test.go index b4e6064df..372a1ebb8 100644 --- a/cmd/tree-walk_test.go +++ b/cmd/tree-walk_test.go @@ -337,7 +337,7 @@ func TestListDir(t *testing.T) { } // None of the disks are available, should get errDiskNotFound. _, _, err = listDir(volume, "", "") - if err != errDiskNotFound { + if errorCause(err) != errDiskNotFound { t.Error("expected errDiskNotFound error.") } } diff --git a/cmd/typed-errors.go b/cmd/typed-errors.go index 84cde5c73..6d6225376 100644 --- a/cmd/typed-errors.go +++ b/cmd/typed-errors.go @@ -38,6 +38,9 @@ var errSignatureMismatch = errors.New("Signature does not match") // used when token used for authentication by the MinioBrowser has expired var errInvalidToken = errors.New("Invalid token") +// used when cached timestamp do not match with what client remembers. +var errInvalidTimestamp = errors.New("Timestamps don't match, server may have restarted.") + // If x-amz-content-sha256 header value mismatches with what we calculate. var errContentSHA256Mismatch = errors.New("sha256 mismatch") diff --git a/cmd/update-main.go b/cmd/update-main.go index a6a8885fc..8aaae1790 100644 --- a/cmd/update-main.go +++ b/cmd/update-main.go @@ -33,10 +33,6 @@ import ( // command specific flags. var ( updateFlags = []cli.Flag{ - cli.BoolFlag{ - Name: "help, h", - Usage: "Help for update.", - }, cli.BoolFlag{ Name: "experimental, E", Usage: "Check experimental update.", @@ -49,7 +45,7 @@ var updateCmd = cli.Command{ Name: "update", Usage: "Check for a new software update.", Action: mainUpdate, - Flags: updateFlags, + Flags: append(updateFlags, globalFlags...), CustomHelpTemplate: `Name: minio {{.Name}} - {{.Usage}} @@ -133,7 +129,7 @@ func parseReleaseData(data string) (time.Time, error) { } // verify updates for releases. -func getReleaseUpdate(updateURL string, noError bool) updateMessage { +func getReleaseUpdate(updateURL string) (updateMsg updateMessage, errMsg string, err error) { // Construct a new update url. newUpdateURLPrefix := updateURL + "/" + runtime.GOOS + "-" + runtime.GOARCH newUpdateURL := newUpdateURLPrefix + "/minio.shasum" @@ -150,7 +146,7 @@ func getReleaseUpdate(updateURL string, noError bool) updateMessage { } // Initialize update message. - updateMsg := updateMessage{ + updateMsg = updateMessage{ Download: downloadURL, Version: Version, } @@ -160,61 +156,54 @@ func getReleaseUpdate(updateURL string, noError bool) updateMessage { Timeout: 3 * time.Second, } - // Fetch new update. - data, err := client.Get(newUpdateURL) - if err != nil && noError { - return updateMsg - } - fatalIf((err), "Unable to read from update URL ‘"+newUpdateURL+"’.") - - // Error out if 'update' command is issued for development based builds. - if Version == "DEVELOPMENT.GOGET" && !noError { - fatalIf((errors.New("")), - "Update mechanism is not supported for ‘go get’ based binary builds. Please download official releases from https://minio.io/#minio") - } - // Parse current minio version into RFC3339. current, err := time.Parse(time.RFC3339, Version) - if err != nil && noError { - return updateMsg + if err != nil { + errMsg = "Unable to parse version string as time." + return } - fatalIf((err), "Unable to parse version string as time.") // Verify if current minio version is zero. - if current.IsZero() && !noError { - fatalIf((errors.New("")), - "Updates mechanism is not supported for custom builds. Please download official releases from https://minio.io/#minio") + if current.IsZero() { + err = errors.New("date should not be zero") + errMsg = "Updates mechanism is not supported for custom builds. Please download official releases from https://minio.io/#minio" + return + } + + // Fetch new update. + data, err := client.Get(newUpdateURL) + if err != nil { + return } // Verify if we have a valid http response i.e http.StatusOK. if data != nil { if data.StatusCode != http.StatusOK { - // Return quickly if noError is set. - if noError { - return updateMsg - } - fatalIf((errors.New("")), "Failed to retrieve update notice. "+data.Status) + errMsg = "Failed to retrieve update notice." + err = errors.New("http status : " + data.Status) + return } } // Read the response body. updateBody, err := ioutil.ReadAll(data.Body) - if err != nil && noError { - return updateMsg + if err != nil { + errMsg = "Failed to retrieve update notice. Please try again later." + return } - fatalIf((err), "Failed to retrieve update notice. Please try again later.") + + errMsg = "Failed to retrieve update notice. Please try again later. Please report this issue at https://github.com/minio/minio/issues" // Parse the date if its valid. latest, err := parseReleaseData(string(updateBody)) - if err != nil && noError { - return updateMsg + if err != nil { + return } - errMsg := "Failed to retrieve update notice. Please try again later. Please report this issue at https://github.com/minio/minio/issues" - fatalIf(err, errMsg) // Verify if the date is not zero. - if latest.IsZero() && !noError { - fatalIf((errors.New("")), errMsg) + if latest.IsZero() { + err = errors.New("date should not be zero") + return } // Is the update latest?. @@ -223,18 +212,25 @@ func getReleaseUpdate(updateURL string, noError bool) updateMessage { } // Return update message. - return updateMsg + return updateMsg, "", nil } // main entry point for update command. func mainUpdate(ctx *cli.Context) { - // Print all errors as they occur. - noError := false + // Error out if 'update' command is issued for development based builds. + if Version == "DEVELOPMENT.GOGET" { + fatalIf(errors.New(""), "Update mechanism is not supported for ‘go get’ based binary builds. Please download official releases from https://minio.io/#minio") + } // Check for update. + var updateMsg updateMessage + var errMsg string + var err error if ctx.Bool("experimental") { - console.Println(getReleaseUpdate(minioUpdateExperimentalURL, noError)) + updateMsg, errMsg, err = getReleaseUpdate(minioUpdateExperimentalURL) } else { - console.Println(getReleaseUpdate(minioUpdateStableURL, noError)) + updateMsg, errMsg, err = getReleaseUpdate(minioUpdateStableURL) } + fatalIf(err, errMsg) + console.Println(updateMsg) } diff --git a/cmd/utils.go b/cmd/utils.go index 29897906c..f861625aa 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -20,10 +20,14 @@ import ( "encoding/base64" "encoding/xml" "errors" + "fmt" "io" + "net" "net/http" "os" "os/exec" + "path/filepath" + "runtime" "strings" "sync" "syscall" @@ -43,6 +47,54 @@ func cloneHeader(h http.Header) http.Header { return h2 } +// checkDuplicates - function to validate if there are duplicates in a slice of strings. +func checkDuplicates(list []string) error { + // Empty lists are not allowed. + if len(list) == 0 { + return errInvalidArgument + } + // Empty keys are not allowed. + for _, key := range list { + if key == "" { + return errInvalidArgument + } + } + listMaps := make(map[string]int) + // Navigate through each configs and count the entries. + for _, key := range list { + listMaps[key]++ + } + // Validate if there are any duplicate counts. + for key, count := range listMaps { + if count != 1 { + return fmt.Errorf("Duplicate key: \"%s\" found of count: \"%d\"", key, count) + } + } + // No duplicates. + return nil +} + +// splits network path into its components Address and Path. +func splitNetPath(networkPath string) (netAddr, netPath string, err error) { + if runtime.GOOS == "windows" { + if volumeName := filepath.VolumeName(networkPath); volumeName != "" { + return "", networkPath, nil + } + } + networkParts := strings.SplitN(networkPath, ":", 2) + if len(networkParts) == 1 { + return "", networkPath, nil + } + if networkParts[1] == "" { + return "", "", &net.AddrError{Err: "Missing path in network path", Addr: networkPath} + } else if networkParts[0] == "" { + return "", "", &net.AddrError{Err: "Missing address in network path", Addr: networkPath} + } else if !filepath.IsAbs(networkParts[1]) { + return "", "", &net.AddrError{Err: "Network path should be absolute", Addr: networkPath} + } + return networkParts[0], networkParts[1], nil +} + // xmlDecoder provide decoded value in xml. func xmlDecoder(body io.Reader, v interface{}, size int64) error { var lbody io.Reader diff --git a/cmd/utils_nix_test.go b/cmd/utils_nix_test.go new file mode 100644 index 000000000..215e4b5b1 --- /dev/null +++ b/cmd/utils_nix_test.go @@ -0,0 +1,61 @@ +// +build !windows + +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "net" + "testing" +) + +// Test for splitNetPath +func TestSplitNetPath(t *testing.T) { + testCases := []struct { + networkPath string + netAddr string + netPath string + err error + }{ + // Invalid cases 1-5. + {"10.1.10.1:", "", "", &net.AddrError{Err: "Missing path in network path", Addr: "10.1.10.1:"}}, + {"10.1.10.1:../1", "", "", &net.AddrError{Err: "Network path should be absolute", Addr: "10.1.10.1:../1"}}, + {":/tmp/1", "", "", &net.AddrError{Err: "Missing address in network path", Addr: ":/tmp/1"}}, + {"10.1.10.1:disk/1", "", "", &net.AddrError{Err: "Network path should be absolute", Addr: "10.1.10.1:disk/1"}}, + {"10.1.10.1:\\path\\test", "", "", &net.AddrError{Err: "Network path should be absolute", Addr: "10.1.10.1:\\path\\test"}}, + + // Valid cases 6-8 + {"10.1.10.1", "", "10.1.10.1", nil}, + {"10.1.10.1://", "10.1.10.1", "//", nil}, + {"10.1.10.1:/disk/1", "10.1.10.1", "/disk/1", nil}, + } + + for i, test := range testCases { + receivedAddr, receivedPath, receivedErr := splitNetPath(test.networkPath) + if receivedAddr != test.netAddr { + t.Errorf("Test case %d: Expected: %s, Received: %s", i+1, test.netAddr, receivedAddr) + } + if receivedPath != test.netPath { + t.Errorf("Test case %d: Expected: %s, Received: %s", i+1, test.netPath, receivedPath) + } + if test.err != nil { + if receivedErr == nil || receivedErr.Error() != test.err.Error() { + t.Errorf("Test case %d: Expected: %v, Received: %v", i+1, test.err, receivedErr) + } + } + } +} diff --git a/cmd/utils_test.go b/cmd/utils_test.go index 3b6215378..800bb73af 100644 --- a/cmd/utils_test.go +++ b/cmd/utils_test.go @@ -17,6 +17,7 @@ package cmd import ( + "fmt" "net/http" "reflect" "testing" @@ -46,6 +47,57 @@ func TestCloneHeader(t *testing.T) { } } +// Tests check duplicates function. +func TestCheckDuplicates(t *testing.T) { + tests := []struct { + list []string + err error + shouldPass bool + }{ + // Test 1 - for '/tmp/1' repeated twice. + { + list: []string{"/tmp/1", "/tmp/1", "/tmp/2", "/tmp/3"}, + err: fmt.Errorf("Duplicate key: \"/tmp/1\" found of count: \"2\""), + shouldPass: false, + }, + // Test 2 - for '/tmp/1' repeated thrice. + { + list: []string{"/tmp/1", "/tmp/1", "/tmp/1", "/tmp/3"}, + err: fmt.Errorf("Duplicate key: \"/tmp/1\" found of count: \"3\""), + shouldPass: false, + }, + // Test 3 - empty string. + { + list: []string{""}, + err: errInvalidArgument, + shouldPass: false, + }, + // Test 4 - empty string. + { + list: nil, + err: errInvalidArgument, + shouldPass: false, + }, + // Test 5 - non repeated strings. + { + list: []string{"/tmp/1", "/tmp/2", "/tmp/3"}, + err: nil, + shouldPass: true, + }, + } + + // Validate if function runs as expected. + for i, test := range tests { + err := checkDuplicates(test.list) + if test.shouldPass && err != test.err { + t.Errorf("Test: %d, Expected %s got %s", i+1, test.err, err) + } + if !test.shouldPass && err.Error() != test.err.Error() { + t.Errorf("Test: %d, Expected %s got %s", i+1, test.err, err) + } + } +} + // Tests maximum object size. func TestMaxObjectSize(t *testing.T) { sizes := []struct { diff --git a/cmd/utils_windows_test.go b/cmd/utils_windows_test.go new file mode 100644 index 000000000..b175c4a42 --- /dev/null +++ b/cmd/utils_windows_test.go @@ -0,0 +1,70 @@ +// +build windows + +/* + * Minio Cloud Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "net" + "testing" +) + +// Test for splitNetPath +func TestSplitNetPath(t *testing.T) { + testCases := []struct { + networkPath string + netAddr string + netPath string + err error + }{ + // Invalid cases 1-8. + {":C:", "", "", &net.AddrError{Err: "Missing address in network path", Addr: ":C:"}}, + {"10.1.10.1:", "", "", &net.AddrError{Err: "Missing path in network path", Addr: "10.1.10.1:"}}, + {"10.1.10.1:C", "", "", &net.AddrError{Err: "Network path should be absolute", Addr: "10.1.10.1:C"}}, + {"10.1.10.1:C:", "", "", &net.AddrError{Err: "Network path should be absolute", Addr: "10.1.10.1:C:"}}, + {"10.1.10.1:C:../path", "", "", &net.AddrError{Err: "Network path should be absolute", Addr: "10.1.10.1:C:../path"}}, + {"10.1.10.1:C:tmp/1", "", "", &net.AddrError{Err: "Network path should be absolute", Addr: "10.1.10.1:C:tmp/1"}}, + {"10.1.10.1::C:\\path\\test", "", "", &net.AddrError{ + Err: "Network path should be absolute", + Addr: "10.1.10.1::C:\\path\\test", + }}, + {"10.1.10.1:\\path\\test", "", "", &net.AddrError{ + Err: "Network path should be absolute", + Addr: "10.1.10.1:\\path\\test", + }}, + + // Valid cases 9-11. + {"10.1.10.1:C:\\path\\test", "10.1.10.1", "C:\\path\\test", nil}, + {"C:\\path\\test", "", "C:\\path\\test", nil}, + {`10.1.10.1:\\?\UNC\path\test`, "10.1.10.1", `\\?\UNC\path\test`, nil}, + } + + for i, test := range testCases { + receivedAddr, receivedPath, receivedErr := splitNetPath(test.networkPath) + if receivedAddr != test.netAddr { + t.Errorf("Test case %d: Expected: %s, Received: %s", i+1, test.netAddr, receivedAddr) + } + if receivedPath != test.netPath { + t.Errorf("Test case %d: Expected: %s, Received: %s", i+1, test.netPath, receivedPath) + } + if test.err != nil { + if receivedErr == nil || receivedErr.Error() != test.err.Error() { + t.Errorf("Test case %d: Expected: %v, Received: %v", i+1, test.err, receivedErr) + } + } + } +} diff --git a/cmd/version-main.go b/cmd/version-main.go index 8b0bb8d03..1d621a8fd 100644 --- a/cmd/version-main.go +++ b/cmd/version-main.go @@ -25,15 +25,25 @@ var versionCmd = cli.Command{ Name: "version", Usage: "Print version.", Action: mainVersion, + Flags: globalFlags, CustomHelpTemplate: `NAME: minio {{.Name}} - {{.Usage}} USAGE: - minio {{.Name}} {{if .Description}} + minio {{.Name}} + +FLAGS: + {{range .Flags}}{{.}} + {{end}} + `, } func mainVersion(ctx *cli.Context) { + if len(ctx.Args()) != 0 { + cli.ShowCommandHelpAndExit(ctx, "version", 1) + } + console.Println("Version: " + Version) console.Println("Release-Tag: " + ReleaseTag) console.Println("Commit-ID: " + CommitID) diff --git a/cmd/web-handlers.go b/cmd/web-handlers.go index add91dd9a..5f1762a2f 100644 --- a/cmd/web-handlers.go +++ b/cmd/web-handlers.go @@ -19,6 +19,7 @@ package cmd import ( "bytes" "encoding/json" + "errors" "fmt" "io/ioutil" "net/http" @@ -40,7 +41,7 @@ import ( // isJWTReqAuthenticated validates if any incoming request to be a // valid JWT authenticated request. func isJWTReqAuthenticated(req *http.Request) bool { - jwt, err := newJWT() + jwt, err := newJWT(defaultWebTokenExpiry) if err != nil { errorIf(err, "unable to initialize a new JWT") return false @@ -124,7 +125,11 @@ func (web *webAPIHandlers) StorageInfo(r *http.Request, args *GenericArgs, reply return &json2.Error{Message: "Unauthorized request"} } reply.UIVersion = miniobrowser.UIVersion - reply.StorageInfo = web.ObjectAPI.StorageInfo() + objectAPI := web.ObjectAPI() + if objectAPI == nil { + return &json2.Error{Message: "Volume not found"} + } + reply.StorageInfo = objectAPI.StorageInfo() return nil } @@ -139,7 +144,11 @@ func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, rep return &json2.Error{Message: "Unauthorized request"} } reply.UIVersion = miniobrowser.UIVersion - if err := web.ObjectAPI.MakeBucket(args.BucketName); err != nil { + objectAPI := web.ObjectAPI() + if objectAPI == nil { + return &json2.Error{Message: "Volume not found"} + } + if err := objectAPI.MakeBucket(args.BucketName); err != nil { return &json2.Error{Message: err.Error()} } return nil @@ -164,7 +173,11 @@ func (web *webAPIHandlers) ListBuckets(r *http.Request, args *WebGenericArgs, re if !isJWTReqAuthenticated(r) { return &json2.Error{Message: "Unauthorized request"} } - buckets, err := web.ObjectAPI.ListBuckets() + objectAPI := web.ObjectAPI() + if objectAPI == nil { + return &json2.Error{Message: "Volume not found"} + } + buckets, err := objectAPI.ListBuckets() if err != nil { return &json2.Error{Message: err.Error()} } @@ -212,7 +225,11 @@ func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, r return &json2.Error{Message: "Unauthorized request"} } for { - lo, err := web.ObjectAPI.ListObjects(args.BucketName, args.Prefix, marker, "/", 1000) + objectAPI := web.ObjectAPI() + if objectAPI == nil { + return &json2.Error{Message: "Volume not found"} + } + lo, err := objectAPI.ListObjects(args.BucketName, args.Prefix, marker, "/", 1000) if err != nil { return &json2.Error{Message: err.Error()} } @@ -250,7 +267,11 @@ func (web *webAPIHandlers) RemoveObject(r *http.Request, args *RemoveObjectArgs, return &json2.Error{Message: "Unauthorized request"} } reply.UIVersion = miniobrowser.UIVersion - if err := web.ObjectAPI.DeleteObject(args.BucketName, args.ObjectName); err != nil { + objectAPI := web.ObjectAPI() + if objectAPI == nil { + return &json2.Error{Message: "Volume not found"} + } + if err := objectAPI.DeleteObject(args.BucketName, args.ObjectName); err != nil { return &json2.Error{Message: err.Error()} } return nil @@ -268,9 +289,14 @@ type LoginRep struct { UIVersion string `json:"uiVersion"` } +// Default JWT for minio browser expires in 24hrs. +const ( + defaultWebTokenExpiry time.Duration = time.Hour * 24 // 24Hrs. +) + // Login - user login handler. func (web *webAPIHandlers) Login(r *http.Request, args *LoginArgs, reply *LoginRep) error { - jwt, err := newJWT() + jwt, err := newJWT(defaultWebTokenExpiry) if err != nil { return &json2.Error{Message: err.Error()} } @@ -335,7 +361,7 @@ func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *Se return &json2.Error{Message: err.Error()} } - jwt, err := newJWT() + jwt, err := newJWT(defaultWebTokenExpiry) // JWT Expiry set to 24Hrs. if err != nil { return &json2.Error{Message: err.Error()} } @@ -384,13 +410,18 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) { // Extract incoming metadata if any. metadata := extractMetadataFromHeader(r.Header) - if _, err := web.ObjectAPI.PutObject(bucket, object, -1, r.Body, metadata); err != nil { + objectAPI := web.ObjectAPI() + if objectAPI == nil { + writeWebErrorResponse(w, errors.New("Volume not found")) + return + } + if _, err := objectAPI.PutObject(bucket, object, -1, r.Body, metadata); err != nil { writeWebErrorResponse(w, err) return } // Fetch object info for notifications. - objInfo, err := web.ObjectAPI.GetObjectInfo(bucket, object) + objInfo, err := objectAPI.GetObjectInfo(bucket, object) if err != nil { errorIf(err, "Unable to fetch object info for \"%s\"", path.Join(bucket, object)) return @@ -416,7 +447,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) { object := vars["object"] tokenStr := r.URL.Query().Get("token") - jwt, err := newJWT() + jwt, err := newJWT(defaultWebTokenExpiry) // Expiry set to 24Hrs. if err != nil { errorIf(err, "error in getting new JWT") return @@ -435,13 +466,18 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) { // Add content disposition. w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", path.Base(object))) - objInfo, err := web.ObjectAPI.GetObjectInfo(bucket, object) + objectAPI := web.ObjectAPI() + if objectAPI == nil { + writeWebErrorResponse(w, errors.New("Volume not found")) + return + } + objInfo, err := objectAPI.GetObjectInfo(bucket, object) if err != nil { writeWebErrorResponse(w, err) return } offset := int64(0) - err = web.ObjectAPI.GetObject(bucket, object, offset, objInfo.Size, w) + err = objectAPI.GetObject(bucket, object, offset, objInfo.Size, w) if err != nil { /// No need to print error, response writer already written to. return @@ -529,7 +565,11 @@ func (web *webAPIHandlers) GetBucketPolicy(r *http.Request, args *GetBucketPolic return &json2.Error{Message: "Unauthorized request"} } - policyInfo, err := readBucketAccessPolicy(web.ObjectAPI, args.BucketName) + objectAPI := web.ObjectAPI() + if objectAPI == nil { + return &json2.Error{Message: "Server not initialized"} + } + policyInfo, err := readBucketAccessPolicy(objectAPI, args.BucketName) if err != nil { return &json2.Error{Message: err.Error()} } @@ -560,7 +600,11 @@ func (web *webAPIHandlers) SetBucketPolicy(r *http.Request, args *SetBucketPolic return &json2.Error{Message: "Invalid policy " + args.Policy} } - policyInfo, err := readBucketAccessPolicy(web.ObjectAPI, args.BucketName) + objectAPI := web.ObjectAPI() + if objectAPI == nil { + return &json2.Error{Message: "Server not initialized"} + } + policyInfo, err := readBucketAccessPolicy(objectAPI, args.BucketName) if err != nil { return &json2.Error{Message: err.Error()} } @@ -573,7 +617,7 @@ func (web *webAPIHandlers) SetBucketPolicy(r *http.Request, args *SetBucketPolic } // TODO: update policy statements according to bucket name, prefix and policy arguments. - if err := writeBucketPolicy(args.BucketName, web.ObjectAPI, bytes.NewReader(data), int64(len(data))); err != nil { + if err := writeBucketPolicy(args.BucketName, objectAPI, bytes.NewReader(data), int64(len(data))); err != nil { return &json2.Error{Message: err.Error()} } diff --git a/cmd/web-router.go b/cmd/web-router.go index f2a5e18e4..e5bee0c48 100644 --- a/cmd/web-router.go +++ b/cmd/web-router.go @@ -30,7 +30,7 @@ import ( // webAPI container for Web API. type webAPIHandlers struct { - ObjectAPI ObjectLayer + ObjectAPI func() ObjectLayer } // indexHandler - Handler to serve index.html diff --git a/cmd/xl-v1-bucket.go b/cmd/xl-v1-bucket.go index 9e5501561..e3f767ec7 100644 --- a/cmd/xl-v1-bucket.go +++ b/cmd/xl-v1-bucket.go @@ -28,15 +28,15 @@ import ( func (xl xlObjects) MakeBucket(bucket string) error { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return BucketNameInvalid{Bucket: bucket} - } - // Verify if bucket is found. - if xl.isBucketExist(bucket) { - return toObjectErr(errVolumeExists, bucket) + return traceError(BucketNameInvalid{Bucket: bucket}) } - nsMutex.Lock(bucket, "") - defer nsMutex.Unlock(bucket, "") + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + + nsMutex.Lock(bucket, "", opsID) + defer nsMutex.Unlock(bucket, "", opsID) // Initialize sync waitgroup. var wg = &sync.WaitGroup{} @@ -47,7 +47,7 @@ func (xl xlObjects) MakeBucket(bucket string) error { // Make a volume entry on all underlying storage disks. for index, disk := range xl.storageDisks { if disk == nil { - dErrs[index] = errDiskNotFound + dErrs[index] = traceError(errDiskNotFound) continue } wg.Add(1) @@ -56,7 +56,7 @@ func (xl xlObjects) MakeBucket(bucket string) error { defer wg.Done() err := disk.MakeVol(bucket) if err != nil { - dErrs[index] = err + dErrs[index] = traceError(err) } }(index, disk) } @@ -68,7 +68,7 @@ func (xl xlObjects) MakeBucket(bucket string) error { if !isDiskQuorum(dErrs, xl.writeQuorum) { // Purge successfully created buckets if we don't have writeQuorum. xl.undoMakeBucket(bucket) - return toObjectErr(errXLWriteQuorum, bucket) + return toObjectErr(traceError(errXLWriteQuorum), bucket) } // Verify we have any other errors which should undo make bucket. @@ -146,6 +146,7 @@ func (xl xlObjects) getBucketInfo(bucketName string) (bucketInfo BucketInfo, err } return bucketInfo, nil } + err = traceError(err) // For any reason disk went offline continue and pick the next one. if isErrIgnored(err, bucketMetadataOpIgnoredErrs) { continue @@ -157,16 +158,12 @@ func (xl xlObjects) getBucketInfo(bucketName string) (bucketInfo BucketInfo, err // Checks whether bucket exists. func (xl xlObjects) isBucketExist(bucket string) bool { - nsMutex.RLock(bucket, "") - defer nsMutex.RUnlock(bucket, "") - // Check whether bucket exists. _, err := xl.getBucketInfo(bucket) if err != nil { if err == errVolumeNotFound { return false } - errorIf(err, "Stat failed on bucket "+bucket+".") return false } return true @@ -178,8 +175,12 @@ func (xl xlObjects) GetBucketInfo(bucket string) (BucketInfo, error) { if !IsValidBucketName(bucket) { return BucketInfo{}, BucketNameInvalid{Bucket: bucket} } - nsMutex.RLock(bucket, "") - defer nsMutex.RUnlock(bucket, "") + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + + nsMutex.RLock(bucket, "", opsID) + defer nsMutex.RUnlock(bucket, "", opsID) bucketInfo, err := xl.getBucketInfo(bucket) if err != nil { return BucketInfo{}, toObjectErr(err, bucket) @@ -249,13 +250,13 @@ func (xl xlObjects) DeleteBucket(bucket string) error { if !IsValidBucketName(bucket) { return BucketNameInvalid{Bucket: bucket} } - // Verify if bucket is found. - if !xl.isBucketExist(bucket) { - return BucketNotFound{Bucket: bucket} - } - nsMutex.Lock(bucket, "") - defer nsMutex.Unlock(bucket, "") + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + + nsMutex.Lock(bucket, "", opsID) + defer nsMutex.Unlock(bucket, "", opsID) // Collect if all disks report volume not found. var wg = &sync.WaitGroup{} @@ -264,7 +265,7 @@ func (xl xlObjects) DeleteBucket(bucket string) error { // Remove a volume entry on all underlying storage disks. for index, disk := range xl.storageDisks { if disk == nil { - dErrs[index] = errDiskNotFound + dErrs[index] = traceError(errDiskNotFound) continue } wg.Add(1) @@ -274,12 +275,15 @@ func (xl xlObjects) DeleteBucket(bucket string) error { // Attempt to delete bucket. err := disk.DeleteVol(bucket) if err != nil { - dErrs[index] = err + dErrs[index] = traceError(err) return } // Cleanup all the previously incomplete multiparts. err = cleanupDir(disk, path.Join(minioMetaBucket, mpartMetaPrefix), bucket) - if err != nil && err != errVolumeNotFound { + if err != nil { + if errorCause(err) == errVolumeNotFound { + return + } dErrs[index] = err } }(index, disk) @@ -290,7 +294,7 @@ func (xl xlObjects) DeleteBucket(bucket string) error { if !isDiskQuorum(dErrs, xl.writeQuorum) { xl.undoDeleteBucket(bucket) - return toObjectErr(errXLWriteQuorum, bucket) + return toObjectErr(traceError(errXLWriteQuorum), bucket) } if reducedErr := reduceErrs(dErrs, []error{ @@ -300,5 +304,7 @@ func (xl xlObjects) DeleteBucket(bucket string) error { }); reducedErr != nil { return toObjectErr(reducedErr, bucket) } + + // Success. return nil } diff --git a/cmd/xl-v1-errors.go b/cmd/xl-v1-errors.go index faa03e92f..582f80f82 100644 --- a/cmd/xl-v1-errors.go +++ b/cmd/xl-v1-errors.go @@ -27,6 +27,9 @@ var errXLMinDisks = errors.New("Minimum '4' disks are required to enable erasure // errXLNumDisks - returned for odd number of disks. var errXLNumDisks = errors.New("Total number of disks should be multiples of '2'") +// errXLDuplicateArguments - returned for duplicate disks. +var errXLDuplicateArguments = errors.New("Duplicate disks found.") + // errXLReadQuorum - did not meet read quorum. var errXLReadQuorum = errors.New("Read failed. Insufficient number of disks online") diff --git a/cmd/xl-v1-healing.go b/cmd/xl-v1-healing.go index a9c5b9d1d..73b0768c1 100644 --- a/cmd/xl-v1-healing.go +++ b/cmd/xl-v1-healing.go @@ -134,11 +134,11 @@ func xlLatestMetadata(partsMetadata []xlMetaV1, errs []error) (latestMeta xlMeta func xlShouldHeal(partsMetadata []xlMetaV1, errs []error) bool { modTime := commonTime(listObjectModtimes(partsMetadata, errs)) for index := range partsMetadata { - if errs[index] == errFileNotFound { - return true + if errs[index] == errDiskNotFound { + continue } if errs[index] != nil { - continue + return true } if modTime != partsMetadata[index].Stat.ModTime { return true diff --git a/cmd/xl-v1-list-objects-heal.go b/cmd/xl-v1-list-objects-heal.go index 577ad453f..1039faf8f 100644 --- a/cmd/xl-v1-list-objects-heal.go +++ b/cmd/xl-v1-list-objects-heal.go @@ -137,12 +137,23 @@ func (xl xlObjects) listObjectsHeal(bucket, prefix, marker, delimiter string, ma result.Prefixes = append(result.Prefixes, objInfo.Name) continue } - result.Objects = append(result.Objects, ObjectInfo{ - Name: objInfo.Name, - ModTime: objInfo.ModTime, - Size: objInfo.Size, - IsDir: false, - }) + + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + + // Check if the current object needs healing + nsMutex.RLock(bucket, objInfo.Name, opsID) + partsMetadata, errs := readAllXLMetadata(xl.storageDisks, bucket, objInfo.Name) + if xlShouldHeal(partsMetadata, errs) { + result.Objects = append(result.Objects, ObjectInfo{ + Name: objInfo.Name, + ModTime: objInfo.ModTime, + Size: objInfo.Size, + IsDir: false, + }) + } + nsMutex.RUnlock(bucket, objInfo.Name, opsID) } return result, nil } @@ -151,28 +162,28 @@ func (xl xlObjects) listObjectsHeal(bucket, prefix, marker, delimiter string, ma func (xl xlObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return ListObjectsInfo{}, BucketNameInvalid{Bucket: bucket} + return ListObjectsInfo{}, traceError(BucketNameInvalid{Bucket: bucket}) } // Verify if bucket exists. if !xl.isBucketExist(bucket) { - return ListObjectsInfo{}, BucketNotFound{Bucket: bucket} + return ListObjectsInfo{}, traceError(BucketNotFound{Bucket: bucket}) } if !IsValidObjectPrefix(prefix) { - return ListObjectsInfo{}, ObjectNameInvalid{Bucket: bucket, Object: prefix} + return ListObjectsInfo{}, traceError(ObjectNameInvalid{Bucket: bucket, Object: prefix}) } // Verify if delimiter is anything other than '/', which we do not support. if delimiter != "" && delimiter != slashSeparator { - return ListObjectsInfo{}, UnsupportedDelimiter{ + return ListObjectsInfo{}, traceError(UnsupportedDelimiter{ Delimiter: delimiter, - } + }) } // Verify if marker has prefix. if marker != "" { if !strings.HasPrefix(marker, prefix) { - return ListObjectsInfo{}, InvalidMarkerPrefixCombination{ + return ListObjectsInfo{}, traceError(InvalidMarkerPrefixCombination{ Marker: marker, Prefix: prefix, - } + }) } } diff --git a/cmd/xl-v1-list-objects.go b/cmd/xl-v1-list-objects.go index 6aa4bc7a2..29e33f932 100644 --- a/cmd/xl-v1-list-objects.go +++ b/cmd/xl-v1-list-objects.go @@ -48,7 +48,7 @@ func (xl xlObjects) listObjects(bucket, prefix, marker, delimiter string, maxKey // For any walk error return right away. if walkResult.err != nil { // File not found is a valid case. - if walkResult.err == errFileNotFound { + if errorCause(walkResult.err) == errFileNotFound { return ListObjectsInfo{}, nil } return ListObjectsInfo{}, toObjectErr(walkResult.err, bucket, prefix) @@ -66,8 +66,7 @@ func (xl xlObjects) listObjects(bucket, prefix, marker, delimiter string, maxKey objInfo, err = xl.getObjectInfo(bucket, entry) if err != nil { // Ignore errFileNotFound - if err == errFileNotFound { - errorIf(err, "Unable to get object info", bucket, entry) + if errorCause(err) == errFileNotFound { continue } return ListObjectsInfo{}, toObjectErr(err, bucket, prefix) @@ -109,28 +108,28 @@ func (xl xlObjects) listObjects(bucket, prefix, marker, delimiter string, maxKey func (xl xlObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return ListObjectsInfo{}, BucketNameInvalid{Bucket: bucket} + return ListObjectsInfo{}, traceError(BucketNameInvalid{Bucket: bucket}) } // Verify if bucket exists. if !xl.isBucketExist(bucket) { - return ListObjectsInfo{}, BucketNotFound{Bucket: bucket} + return ListObjectsInfo{}, traceError(BucketNotFound{Bucket: bucket}) } if !IsValidObjectPrefix(prefix) { - return ListObjectsInfo{}, ObjectNameInvalid{Bucket: bucket, Object: prefix} + return ListObjectsInfo{}, traceError(ObjectNameInvalid{Bucket: bucket, Object: prefix}) } // Verify if delimiter is anything other than '/', which we do not support. if delimiter != "" && delimiter != slashSeparator { - return ListObjectsInfo{}, UnsupportedDelimiter{ + return ListObjectsInfo{}, traceError(UnsupportedDelimiter{ Delimiter: delimiter, - } + }) } // Verify if marker has prefix. if marker != "" { if !strings.HasPrefix(marker, prefix) { - return ListObjectsInfo{}, InvalidMarkerPrefixCombination{ + return ListObjectsInfo{}, traceError(InvalidMarkerPrefixCombination{ Marker: marker, Prefix: prefix, - } + }) } } diff --git a/cmd/xl-v1-metadata.go b/cmd/xl-v1-metadata.go index 8e2b04c5e..0208a91ff 100644 --- a/cmd/xl-v1-metadata.go +++ b/cmd/xl-v1-metadata.go @@ -88,7 +88,7 @@ func (e erasureInfo) GetCheckSumInfo(partName string) (ckSum checkSumInfo, err e return sum, nil } } - return checkSumInfo{}, errUnexpected + return checkSumInfo{}, traceError(errUnexpected) } // statInfo - carries stat information of the object. @@ -136,9 +136,9 @@ func (m xlMetaV1) IsValid() bool { return m.Version == "1.0.0" && m.Format == "xl" } -// ObjectPartIndex - returns the index of matching object part number. -func (m xlMetaV1) ObjectPartIndex(partNumber int) int { - for i, part := range m.Parts { +// objectPartIndex - returns the index of matching object part number. +func objectPartIndex(parts []objectPartInfo, partNumber int) int { + for i, part := range parts { if partNumber == part.Number { return i } @@ -188,7 +188,7 @@ func (m xlMetaV1) ObjectToPartOffset(offset int64) (partIndex int, partOffset in partOffset -= part.Size } // Offset beyond the size of the object return InvalidRange. - return 0, 0, InvalidRange{} + return 0, 0, traceError(InvalidRange{}) } // pickValidXLMeta - picks one valid xlMeta content and returns from a @@ -214,16 +214,15 @@ var objMetadataOpIgnoredErrs = []error{ errFileNotFound, } -// readXLMetadata - returns the object metadata `xl.json` content from -// one of the disks picked at random. -func (xl xlObjects) readXLMetadata(bucket, object string) (xlMeta xlMetaV1, err error) { +// readXLMetaParts - returns the XL Metadata Parts from xl.json of one of the disks picked at random. +func (xl xlObjects) readXLMetaParts(bucket, object string) (xlMetaParts []objectPartInfo, err error) { for _, disk := range xl.getLoadBalancedDisks() { if disk == nil { continue } - xlMeta, err = readXLMeta(disk, bucket, object) + xlMetaParts, err = readXLMetaParts(disk, bucket, object) if err == nil { - return xlMeta, nil + return xlMetaParts, nil } // For any reason disk or bucket is not available continue // and read from other disks. @@ -233,13 +232,35 @@ func (xl xlObjects) readXLMetadata(bucket, object string) (xlMeta xlMetaV1, err break } // Return error here. - return xlMetaV1{}, err + return nil, err +} + +// readXLMetaStat - return xlMetaV1.Stat and xlMetaV1.Meta from one of the disks picked at random. +func (xl xlObjects) readXLMetaStat(bucket, object string) (xlStat statInfo, xlMeta map[string]string, err error) { + for _, disk := range xl.getLoadBalancedDisks() { + if disk == nil { + continue + } + // parses only xlMetaV1.Meta and xlMeta.Stat + xlStat, xlMeta, err = readXLMetaStat(disk, bucket, object) + if err == nil { + return xlStat, xlMeta, nil + } + // For any reason disk or bucket is not available continue + // and read from other disks. + if isErrIgnored(err, objMetadataOpIgnoredErrs) { + continue + } + break + } + // Return error here. + return statInfo{}, nil, err } // deleteXLMetadata - deletes `xl.json` on a single disk. func deleteXLMetdata(disk StorageAPI, bucket, prefix string) error { jsonFile := path.Join(prefix, xlMetaJSONFile) - return disk.DeleteFile(bucket, jsonFile) + return traceError(disk.DeleteFile(bucket, jsonFile)) } // writeXLMetadata - writes `xl.json` to a single disk. @@ -249,10 +270,10 @@ func writeXLMetadata(disk StorageAPI, bucket, prefix string, xlMeta xlMetaV1) er // Marshal json. metadataBytes, err := json.Marshal(&xlMeta) if err != nil { - return err + return traceError(err) } // Persist marshalled data. - return disk.AppendFile(bucket, jsonFile, metadataBytes) + return traceError(disk.AppendFile(bucket, jsonFile, metadataBytes)) } // deleteAllXLMetadata - deletes all partially written `xl.json` depending on errs. @@ -284,7 +305,7 @@ func writeUniqueXLMetadata(disks []StorageAPI, bucket, prefix string, xlMetas [] // Start writing `xl.json` to all disks in parallel. for index, disk := range disks { if disk == nil { - mErrs[index] = errDiskNotFound + mErrs[index] = traceError(errDiskNotFound) continue } wg.Add(1) @@ -310,7 +331,7 @@ func writeUniqueXLMetadata(disks []StorageAPI, bucket, prefix string, xlMetas [] if !isDiskQuorum(mErrs, quorum) { // Delete all `xl.json` successfully renamed. deleteAllXLMetadata(disks, bucket, prefix, mErrs) - return errXLWriteQuorum + return traceError(errXLWriteQuorum) } return reduceErrs(mErrs, []error{ @@ -328,7 +349,7 @@ func writeSameXLMetadata(disks []StorageAPI, bucket, prefix string, xlMeta xlMet // Start writing `xl.json` to all disks in parallel. for index, disk := range disks { if disk == nil { - mErrs[index] = errDiskNotFound + mErrs[index] = traceError(errDiskNotFound) continue } wg.Add(1) @@ -354,7 +375,7 @@ func writeSameXLMetadata(disks []StorageAPI, bucket, prefix string, xlMeta xlMet if !isDiskQuorum(mErrs, writeQuorum) { // Delete all `xl.json` successfully renamed. deleteAllXLMetadata(disks, bucket, prefix, mErrs) - return errXLWriteQuorum + return traceError(errXLWriteQuorum) } return reduceErrs(mErrs, []error{ diff --git a/cmd/xl-v1-metadata_test.go b/cmd/xl-v1-metadata_test.go index b45062320..8c9fc0f39 100644 --- a/cmd/xl-v1-metadata_test.go +++ b/cmd/xl-v1-metadata_test.go @@ -55,13 +55,14 @@ func TestAddObjectPart(t *testing.T) { xlMeta.AddObjectPart(testCase.partNum, "part."+partNumString, "etag."+partNumString, int64(testCase.partNum+MiB)) } - if index := xlMeta.ObjectPartIndex(testCase.partNum); index != testCase.expectedIndex { + if index := objectPartIndex(xlMeta.Parts, testCase.partNum); index != testCase.expectedIndex { t.Fatalf("%+v: expected = %d, got: %d", testCase, testCase.expectedIndex, index) } } } -// Test xlMetaV1.ObjectPartIndex() +// Test objectPartIndex(). +// generates a sample xlMeta data and asserts the output of objectPartIndex() with the expected value. func TestObjectPartIndex(t *testing.T) { testCases := []struct { partNum int @@ -94,7 +95,7 @@ func TestObjectPartIndex(t *testing.T) { // Test them. for _, testCase := range testCases { - if index := xlMeta.ObjectPartIndex(testCase.partNum); index != testCase.expectedIndex { + if index := objectPartIndex(xlMeta.Parts, testCase.partNum); index != testCase.expectedIndex { t.Fatalf("%+v: expected = %d, got: %d", testCase, testCase.expectedIndex, index) } } @@ -136,6 +137,7 @@ func TestObjectToPartOffset(t *testing.T) { // Test them. for _, testCase := range testCases { index, offset, err := xlMeta.ObjectToPartOffset(testCase.offset) + err = errorCause(err) if err != testCase.expectedErr { t.Fatalf("%+v: expected = %s, got: %s", testCase, testCase.expectedErr, err) } diff --git a/cmd/xl-v1-multipart-common.go b/cmd/xl-v1-multipart-common.go index 7d85c2693..b637b4a25 100644 --- a/cmd/xl-v1-multipart-common.go +++ b/cmd/xl-v1-multipart-common.go @@ -43,15 +43,15 @@ func (xl xlObjects) updateUploadsJSON(bucket, object string, uploadsJSON uploads defer wg.Done() uploadsBytes, wErr := json.Marshal(uploadsJSON) if wErr != nil { - errs[index] = wErr + errs[index] = traceError(wErr) return } if wErr = disk.AppendFile(minioMetaBucket, tmpUploadsPath, uploadsBytes); wErr != nil { - errs[index] = wErr + errs[index] = traceError(wErr) return } if wErr = disk.RenameFile(minioMetaBucket, tmpUploadsPath, minioMetaBucket, uploadsPath); wErr != nil { - errs[index] = wErr + errs[index] = traceError(wErr) return } }(index, disk) @@ -82,7 +82,7 @@ func (xl xlObjects) updateUploadsJSON(bucket, object string, uploadsJSON uploads }(index, disk) } wg.Wait() - return errXLWriteQuorum + return traceError(errXLWriteQuorum) } return nil } @@ -117,7 +117,7 @@ func (xl xlObjects) writeUploadJSON(bucket, object, uploadID string, initiated t // Reads `uploads.json` and returns error. uploadsJSON, err := xl.readUploadsJSON(bucket, object) if err != nil { - if err != errFileNotFound { + if errorCause(err) != errFileNotFound { return err } // Set uploads format to `xl` otherwise. @@ -129,7 +129,7 @@ func (xl xlObjects) writeUploadJSON(bucket, object, uploadID string, initiated t // Update `uploads.json` on all disks. for index, disk := range xl.storageDisks { if disk == nil { - errs[index] = errDiskNotFound + errs[index] = traceError(errDiskNotFound) continue } wg.Add(1) @@ -138,21 +138,21 @@ func (xl xlObjects) writeUploadJSON(bucket, object, uploadID string, initiated t defer wg.Done() uploadsJSONBytes, wErr := json.Marshal(&uploadsJSON) if wErr != nil { - errs[index] = wErr + errs[index] = traceError(wErr) return } // Write `uploads.json` to disk. if wErr = disk.AppendFile(minioMetaBucket, tmpUploadsPath, uploadsJSONBytes); wErr != nil { - errs[index] = wErr + errs[index] = traceError(wErr) return } wErr = disk.RenameFile(minioMetaBucket, tmpUploadsPath, minioMetaBucket, uploadsPath) if wErr != nil { if dErr := disk.DeleteFile(minioMetaBucket, tmpUploadsPath); dErr != nil { - errs[index] = dErr + errs[index] = traceError(dErr) return } - errs[index] = wErr + errs[index] = traceError(wErr) return } errs[index] = nil @@ -180,7 +180,7 @@ func (xl xlObjects) writeUploadJSON(bucket, object, uploadID string, initiated t }(index, disk) } wg.Wait() - return errXLWriteQuorum + return traceError(errXLWriteQuorum) } // Ignored errors list. @@ -248,7 +248,7 @@ func (xl xlObjects) statPart(bucket, object, uploadID, partName string) (fileInf if err == nil { return fileInfo, nil } - + err = traceError(err) // For any reason disk was deleted or goes offline we continue to next disk. if isErrIgnored(err, objMetadataOpIgnoredErrs) { continue @@ -271,7 +271,7 @@ func commitXLMetadata(disks []StorageAPI, srcPrefix, dstPrefix string, quorum in // Rename `xl.json` to all disks in parallel. for index, disk := range disks { if disk == nil { - mErrs[index] = errDiskNotFound + mErrs[index] = traceError(errDiskNotFound) continue } wg.Add(1) @@ -284,7 +284,7 @@ func commitXLMetadata(disks []StorageAPI, srcPrefix, dstPrefix string, quorum in // Renames `xl.json` from source prefix to destination prefix. rErr := disk.RenameFile(minioMetaBucket, srcJSONFile, minioMetaBucket, dstJSONFile) if rErr != nil { - mErrs[index] = rErr + mErrs[index] = traceError(rErr) return } mErrs[index] = nil @@ -297,7 +297,7 @@ func commitXLMetadata(disks []StorageAPI, srcPrefix, dstPrefix string, quorum in if !isDiskQuorum(mErrs, quorum) { // Delete all `xl.json` successfully renamed. deleteAllXLMetadata(disks, minioMetaBucket, dstPrefix, mErrs) - return errXLWriteQuorum + return traceError(errXLWriteQuorum) } // List of ignored errors. diff --git a/cmd/xl-v1-multipart.go b/cmd/xl-v1-multipart.go index 7b1cdd929..1f083850d 100644 --- a/cmd/xl-v1-multipart.go +++ b/cmd/xl-v1-multipart.go @@ -62,7 +62,11 @@ func (xl xlObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark // List all upload ids for the keyMarker starting from // uploadIDMarker first. if uploadIDMarker != "" { - nsMutex.RLock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, keyMarker)) + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + + nsMutex.RLock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, keyMarker), opsID) for _, disk := range xl.getLoadBalancedDisks() { if disk == nil { continue @@ -76,7 +80,7 @@ func (xl xlObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark } break } - nsMutex.RUnlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, keyMarker)) + nsMutex.RUnlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, keyMarker), opsID) if err != nil { return ListMultipartsInfo{}, err } @@ -127,8 +131,13 @@ func (xl xlObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark var newUploads []uploadMetadata var end bool uploadIDMarker = "" + + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + // For the new object entry we get all its pending uploadIDs. - nsMutex.RLock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, entry)) + nsMutex.RLock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, entry), opsID) var disk StorageAPI for _, disk = range xl.getLoadBalancedDisks() { if disk == nil { @@ -143,7 +152,7 @@ func (xl xlObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark } break } - nsMutex.RUnlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, entry)) + nsMutex.RUnlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, entry), opsID) if err != nil { if isErrIgnored(err, walkResultIgnoredErrs) { continue @@ -205,42 +214,42 @@ func (xl xlObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMark // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return ListMultipartsInfo{}, BucketNameInvalid{Bucket: bucket} + return ListMultipartsInfo{}, traceError(BucketNameInvalid{Bucket: bucket}) } if !xl.isBucketExist(bucket) { - return ListMultipartsInfo{}, BucketNotFound{Bucket: bucket} + return ListMultipartsInfo{}, traceError(BucketNotFound{Bucket: bucket}) } if !IsValidObjectPrefix(prefix) { - return ListMultipartsInfo{}, ObjectNameInvalid{Bucket: bucket, Object: prefix} + return ListMultipartsInfo{}, traceError(ObjectNameInvalid{Bucket: bucket, Object: prefix}) } // Verify if delimiter is anything other than '/', which we do not support. if delimiter != "" && delimiter != slashSeparator { - return ListMultipartsInfo{}, UnsupportedDelimiter{ + return ListMultipartsInfo{}, traceError(UnsupportedDelimiter{ Delimiter: delimiter, - } + }) } // Verify if marker has prefix. if keyMarker != "" && !strings.HasPrefix(keyMarker, prefix) { - return ListMultipartsInfo{}, InvalidMarkerPrefixCombination{ + return ListMultipartsInfo{}, traceError(InvalidMarkerPrefixCombination{ Marker: keyMarker, Prefix: prefix, - } + }) } if uploadIDMarker != "" { if strings.HasSuffix(keyMarker, slashSeparator) { - return result, InvalidUploadIDKeyCombination{ + return result, traceError(InvalidUploadIDKeyCombination{ UploadIDMarker: uploadIDMarker, KeyMarker: keyMarker, - } + }) } id, err := uuid.Parse(uploadIDMarker) if err != nil { - return result, err + return result, traceError(err) } if id.IsZero() { - return result, MalformedUploadID{ + return result, traceError(MalformedUploadID{ UploadID: uploadIDMarker, - } + }) } } return xl.listMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads) @@ -250,7 +259,7 @@ func (xl xlObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMark // request, returns back a unique upload id. // // Internally this function creates 'uploads.json' associated for the -// incoming object at '.minio/multipart/bucket/object/uploads.json' on +// incoming object at '.minio.sys/multipart/bucket/object/uploads.json' on // all the disks. `uploads.json` carries metadata regarding on going // multipart operation on the object. func (xl xlObjects) newMultipartUpload(bucket string, object string, meta map[string]string) (uploadID string, err error) { @@ -269,9 +278,13 @@ func (xl xlObjects) newMultipartUpload(bucket string, object string, meta map[st xlMeta.Stat.ModTime = time.Now().UTC() xlMeta.Meta = meta - // This lock needs to be held for any changes to the directory contents of ".minio/multipart/object/" - nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object)) - defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object)) + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + + // This lock needs to be held for any changes to the directory contents of ".minio.sys/multipart/object/" + nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object), opsID) + defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object), opsID) uploadID = getUUID() initiated := time.Now().UTC() @@ -301,15 +314,15 @@ func (xl xlObjects) newMultipartUpload(bucket string, object string, meta map[st func (xl xlObjects) NewMultipartUpload(bucket, object string, meta map[string]string) (string, error) { // Verify if bucket name is valid. if !IsValidBucketName(bucket) { - return "", BucketNameInvalid{Bucket: bucket} + return "", traceError(BucketNameInvalid{Bucket: bucket}) } // Verify whether the bucket exists. if !xl.isBucketExist(bucket) { - return "", BucketNotFound{Bucket: bucket} + return "", traceError(BucketNotFound{Bucket: bucket}) } // Verify if object name is valid. if !IsValidObjectName(object) { - return "", ObjectNameInvalid{Bucket: bucket, Object: object} + return "", traceError(ObjectNameInvalid{Bucket: bucket, Object: object}) } // No metadata is set, allocate a new one. if meta == nil { @@ -326,33 +339,38 @@ func (xl xlObjects) NewMultipartUpload(bucket, object string, meta map[string]st func (xl xlObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return "", BucketNameInvalid{Bucket: bucket} + return "", traceError(BucketNameInvalid{Bucket: bucket}) } // Verify whether the bucket exists. if !xl.isBucketExist(bucket) { - return "", BucketNotFound{Bucket: bucket} + return "", traceError(BucketNotFound{Bucket: bucket}) } if !IsValidObjectName(object) { - return "", ObjectNameInvalid{Bucket: bucket, Object: object} + return "", traceError(ObjectNameInvalid{Bucket: bucket, Object: object}) } var partsMetadata []xlMetaV1 var errs []error uploadIDPath := pathJoin(mpartMetaPrefix, bucket, object, uploadID) - nsMutex.RLock(minioMetaBucket, uploadIDPath) + + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + + nsMutex.RLock(minioMetaBucket, uploadIDPath, opsID) // Validates if upload ID exists. if !xl.isUploadIDExists(bucket, object, uploadID) { - nsMutex.RUnlock(minioMetaBucket, uploadIDPath) - return "", InvalidUploadID{UploadID: uploadID} + nsMutex.RUnlock(minioMetaBucket, uploadIDPath, opsID) + return "", traceError(InvalidUploadID{UploadID: uploadID}) } // Read metadata associated with the object from all disks. partsMetadata, errs = readAllXLMetadata(xl.storageDisks, minioMetaBucket, uploadIDPath) if !isDiskQuorum(errs, xl.writeQuorum) { - nsMutex.RUnlock(minioMetaBucket, uploadIDPath) - return "", toObjectErr(errXLWriteQuorum, bucket, object) + nsMutex.RUnlock(minioMetaBucket, uploadIDPath, opsID) + return "", toObjectErr(traceError(errXLWriteQuorum), bucket, object) } - nsMutex.RUnlock(minioMetaBucket, uploadIDPath) + nsMutex.RUnlock(minioMetaBucket, uploadIDPath, opsID) // List all online disks. onlineDisks, modTime := listOnlineDisks(xl.storageDisks, partsMetadata, errs) @@ -361,7 +379,7 @@ func (xl xlObjects) PutObjectPart(bucket, object, uploadID string, partID int, s xlMeta := pickValidXLMeta(partsMetadata, modTime) onlineDisks = getOrderedDisks(xlMeta.Erasure.Distribution, onlineDisks) - partsMetadata = getOrderedPartsMetadata(xlMeta.Erasure.Distribution, partsMetadata) + _ = getOrderedPartsMetadata(xlMeta.Erasure.Distribution, partsMetadata) // Need a unique name for the part being written in minioMetaBucket to // accommodate concurrent PutObjectPart requests @@ -391,7 +409,7 @@ func (xl xlObjects) PutObjectPart(bucket, object, uploadID string, partID int, s // Should return IncompleteBody{} error when reader has fewer bytes // than specified in request header. if sizeWritten < size { - return "", IncompleteBody{} + return "", traceError(IncompleteBody{}) } // For size == -1, perhaps client is sending in chunked encoding @@ -417,16 +435,20 @@ func (xl xlObjects) PutObjectPart(bucket, object, uploadID string, partID int, s // MD5 mismatch, delete the temporary object. xl.deleteObject(minioMetaBucket, tmpPartPath) // Returns md5 mismatch. - return "", BadDigest{md5Hex, newMD5Hex} + return "", traceError(BadDigest{md5Hex, newMD5Hex}) } } - nsMutex.Lock(minioMetaBucket, uploadIDPath) - defer nsMutex.Unlock(minioMetaBucket, uploadIDPath) + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID = getOpsID() + + nsMutex.Lock(minioMetaBucket, uploadIDPath, opsID) + defer nsMutex.Unlock(minioMetaBucket, uploadIDPath, opsID) // Validate again if upload ID still exists. if !xl.isUploadIDExists(bucket, object, uploadID) { - return "", InvalidUploadID{UploadID: uploadID} + return "", traceError(InvalidUploadID{UploadID: uploadID}) } // Rename temporary part file to its final location. @@ -439,7 +461,7 @@ func (xl xlObjects) PutObjectPart(bucket, object, uploadID string, partID int, s // Read metadata again because it might be updated with parallel upload of another part. partsMetadata, errs = readAllXLMetadata(onlineDisks, minioMetaBucket, uploadIDPath) if !isDiskQuorum(errs, xl.writeQuorum) { - return "", toObjectErr(errXLWriteQuorum, bucket, object) + return "", toObjectErr(traceError(errXLWriteQuorum), bucket, object) } // Get current highest version based on re-read partsMetadata. @@ -490,7 +512,7 @@ func (xl xlObjects) listObjectParts(bucket, object, uploadID string, partNumberM uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID) - xlMeta, err := xl.readXLMetadata(minioMetaBucket, uploadIDPath) + xlParts, err := xl.readXLMetaParts(minioMetaBucket, uploadIDPath) if err != nil { return ListPartsInfo{}, toObjectErr(err, minioMetaBucket, uploadIDPath) } @@ -502,7 +524,7 @@ func (xl xlObjects) listObjectParts(bucket, object, uploadID string, partNumberM result.MaxParts = maxParts // For empty number of parts or maxParts as zero, return right here. - if len(xlMeta.Parts) == 0 || maxParts == 0 { + if len(xlParts) == 0 || maxParts == 0 { return result, nil } @@ -512,10 +534,10 @@ func (xl xlObjects) listObjectParts(bucket, object, uploadID string, partNumberM } // Only parts with higher part numbers will be listed. - partIdx := xlMeta.ObjectPartIndex(partNumberMarker) - parts := xlMeta.Parts + partIdx := objectPartIndex(xlParts, partNumberMarker) + parts := xlParts if partIdx != -1 { - parts = xlMeta.Parts[partIdx+1:] + parts = xlParts[partIdx+1:] } count := maxParts for _, part := range parts { @@ -556,21 +578,26 @@ func (xl xlObjects) listObjectParts(bucket, object, uploadID string, partNumberM func (xl xlObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return ListPartsInfo{}, BucketNameInvalid{Bucket: bucket} + return ListPartsInfo{}, traceError(BucketNameInvalid{Bucket: bucket}) } // Verify whether the bucket exists. if !xl.isBucketExist(bucket) { - return ListPartsInfo{}, BucketNotFound{Bucket: bucket} + return ListPartsInfo{}, traceError(BucketNotFound{Bucket: bucket}) } if !IsValidObjectName(object) { - return ListPartsInfo{}, ObjectNameInvalid{Bucket: bucket, Object: object} + return ListPartsInfo{}, traceError(ObjectNameInvalid{Bucket: bucket, Object: object}) } + + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + // Hold lock so that there is no competing abort-multipart-upload or complete-multipart-upload. - nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID)) - defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID)) + nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID), opsID) + defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID), opsID) if !xl.isUploadIDExists(bucket, object, uploadID) { - return ListPartsInfo{}, InvalidUploadID{UploadID: uploadID} + return ListPartsInfo{}, traceError(InvalidUploadID{UploadID: uploadID}) } result, err := xl.listObjectParts(bucket, object, uploadID, partNumberMarker, maxParts) return result, err @@ -585,26 +612,31 @@ func (xl xlObjects) ListObjectParts(bucket, object, uploadID string, partNumberM func (xl xlObjects) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (string, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return "", BucketNameInvalid{Bucket: bucket} + return "", traceError(BucketNameInvalid{Bucket: bucket}) } // Verify whether the bucket exists. if !xl.isBucketExist(bucket) { - return "", BucketNotFound{Bucket: bucket} + return "", traceError(BucketNotFound{Bucket: bucket}) } if !IsValidObjectName(object) { - return "", ObjectNameInvalid{ + return "", traceError(ObjectNameInvalid{ Bucket: bucket, Object: object, - } + }) } + + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + // Hold lock so that // 1) no one aborts this multipart upload // 2) no one does a parallel complete-multipart-upload on this multipart upload - nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID)) - defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID)) + nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID), opsID) + defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID), opsID) if !xl.isUploadIDExists(bucket, object, uploadID) { - return "", InvalidUploadID{UploadID: uploadID} + return "", traceError(InvalidUploadID{UploadID: uploadID}) } // Calculate s3 compatible md5sum for complete multipart. s3MD5, err := completeMultipartMD5(parts...) @@ -618,7 +650,7 @@ func (xl xlObjects) CompleteMultipartUpload(bucket string, object string, upload partsMetadata, errs := readAllXLMetadata(xl.storageDisks, minioMetaBucket, uploadIDPath) // Do we have writeQuorum?. if !isDiskQuorum(errs, xl.writeQuorum) { - return "", toObjectErr(errXLWriteQuorum, bucket, object) + return "", toObjectErr(traceError(errXLWriteQuorum), bucket, object) } onlineDisks, modTime := listOnlineDisks(xl.storageDisks, partsMetadata, errs) @@ -643,24 +675,24 @@ func (xl xlObjects) CompleteMultipartUpload(bucket string, object string, upload // Validate each part and then commit to disk. for i, part := range parts { - partIdx := currentXLMeta.ObjectPartIndex(part.PartNumber) + partIdx := objectPartIndex(currentXLMeta.Parts, part.PartNumber) // All parts should have same part number. if partIdx == -1 { - return "", InvalidPart{} + return "", traceError(InvalidPart{}) } // All parts should have same ETag as previously generated. if currentXLMeta.Parts[partIdx].ETag != part.ETag { - return "", BadDigest{} + return "", traceError(BadDigest{}) } // All parts except the last part has to be atleast 5MB. if (i < len(parts)-1) && !isMinAllowedPartSize(currentXLMeta.Parts[partIdx].Size) { - return "", PartTooSmall{ + return "", traceError(PartTooSmall{ PartNumber: part.PartNumber, PartSize: currentXLMeta.Parts[partIdx].Size, PartETag: part.ETag, - } + }) } // Last part could have been uploaded as 0bytes, do not need @@ -684,7 +716,7 @@ func (xl xlObjects) CompleteMultipartUpload(bucket string, object string, upload // Check if an object is present as one of the parent dir. if xl.parentDirIsObject(bucket, path.Dir(object)) { - return "", toObjectErr(errFileAccessDenied, bucket, object) + return "", toObjectErr(traceError(errFileAccessDenied), bucket, object) } // Save the final object size and modtime. @@ -712,15 +744,20 @@ func (xl xlObjects) CompleteMultipartUpload(bucket string, object string, upload if rErr != nil { return "", toObjectErr(rErr, minioMetaBucket, uploadIDPath) } + + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID = getOpsID() + // Hold write lock on the destination before rename. - nsMutex.Lock(bucket, object) + nsMutex.Lock(bucket, object, opsID) defer func() { // A new complete multipart upload invalidates any // previously cached object in memory. xl.objCache.Delete(path.Join(bucket, object)) // This lock also protects the cache namespace. - nsMutex.Unlock(bucket, object) + nsMutex.Unlock(bucket, object, opsID) // Prefetch the object from disk by triggering a fake GetObject call // Unlike a regular single PutObject, multipart PutObject is comes in @@ -742,7 +779,7 @@ func (xl xlObjects) CompleteMultipartUpload(bucket string, object string, upload // Remove parts that weren't present in CompleteMultipartUpload request. for _, curpart := range currentXLMeta.Parts { - if xlMeta.ObjectPartIndex(curpart.Number) == -1 { + if objectPartIndex(xlMeta.Parts, curpart.Number) == -1 { // Delete the missing part files. e.g, // Request 1: NewMultipart // Request 2: PutObjectPart 1 @@ -761,10 +798,14 @@ func (xl xlObjects) CompleteMultipartUpload(bucket string, object string, upload // Delete the previously successfully renamed object. xl.deleteObject(minioMetaBucket, path.Join(tmpMetaPrefix, uniqueID)) + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID = getOpsID() + // Hold the lock so that two parallel complete-multipart-uploads do not // leave a stale uploads.json behind. - nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object)) - defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object)) + nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object), opsID) + defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object), opsID) // Validate if there are other incomplete upload-id's present for // the object, if yes do not attempt to delete 'uploads.json'. @@ -785,7 +826,7 @@ func (xl xlObjects) CompleteMultipartUpload(bucket string, object string, upload // Return success. return s3MD5, nil } // No more pending uploads for the object, proceed to delete - // object completely from '.minio/multipart'. + // object completely from '.minio.sys/multipart'. if err = xl.deleteObject(minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object)); err != nil { return "", toObjectErr(err, minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object)) } @@ -796,7 +837,7 @@ func (xl xlObjects) CompleteMultipartUpload(bucket string, object string, upload // abortMultipartUpload - wrapper for purging an ongoing multipart // transaction, deletes uploadID entry from `uploads.json` and purges -// the directory at '.minio/multipart/bucket/object/uploadID' holding +// the directory at '.minio.sys/multipart/bucket/object/uploadID' holding // all the upload parts. func (xl xlObjects) abortMultipartUpload(bucket, object, uploadID string) (err error) { // Cleanup all uploaded parts. @@ -804,8 +845,12 @@ func (xl xlObjects) abortMultipartUpload(bucket, object, uploadID string) (err e return toObjectErr(err, bucket, object) } - nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object)) - defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object)) + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + + nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object), opsID) + defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object), opsID) // Validate if there are other incomplete upload-id's present for // the object, if yes do not attempt to delete 'uploads.json'. uploadsJSON, err := xl.readUploadsJSON(bucket, object) @@ -825,7 +870,7 @@ func (xl xlObjects) abortMultipartUpload(bucket, object, uploadID string) (err e } return nil } // No more pending uploads for the object, we purge the entire - // entry at '.minio/multipart/bucket/object'. + // entry at '.minio.sys/multipart/bucket/object'. if err = xl.deleteObject(minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object)); err != nil { return toObjectErr(err, minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object)) } @@ -848,21 +893,25 @@ func (xl xlObjects) abortMultipartUpload(bucket, object, uploadID string) (err e func (xl xlObjects) AbortMultipartUpload(bucket, object, uploadID string) error { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return BucketNameInvalid{Bucket: bucket} + return traceError(BucketNameInvalid{Bucket: bucket}) } if !xl.isBucketExist(bucket) { - return BucketNotFound{Bucket: bucket} + return traceError(BucketNotFound{Bucket: bucket}) } if !IsValidObjectName(object) { - return ObjectNameInvalid{Bucket: bucket, Object: object} + return traceError(ObjectNameInvalid{Bucket: bucket, Object: object}) } + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + // Hold lock so that there is no competing complete-multipart-upload or put-object-part. - nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID)) - defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID)) + nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID), opsID) + defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID), opsID) if !xl.isUploadIDExists(bucket, object, uploadID) { - return InvalidUploadID{UploadID: uploadID} + return traceError(InvalidUploadID{UploadID: uploadID}) } err := xl.abortMultipartUpload(bucket, object, uploadID) return err diff --git a/cmd/xl-v1-object.go b/cmd/xl-v1-object.go index 560407d51..08d9edb85 100644 --- a/cmd/xl-v1-object.go +++ b/cmd/xl-v1-object.go @@ -42,29 +42,34 @@ import ( func (xl xlObjects) GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) error { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return BucketNameInvalid{Bucket: bucket} + return traceError(BucketNameInvalid{Bucket: bucket}) } // Verify if object is valid. if !IsValidObjectName(object) { - return ObjectNameInvalid{Bucket: bucket, Object: object} + return traceError(ObjectNameInvalid{Bucket: bucket, Object: object}) } // Start offset and length cannot be negative. if startOffset < 0 || length < 0 { - return toObjectErr(errUnexpected, bucket, object) + return traceError(errUnexpected) } // Writer cannot be nil. if writer == nil { - return toObjectErr(errUnexpected, bucket, object) + return traceError(errUnexpected) } + + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + // Lock the object before reading. - nsMutex.RLock(bucket, object) - defer nsMutex.RUnlock(bucket, object) + nsMutex.RLock(bucket, object, opsID) + defer nsMutex.RUnlock(bucket, object, opsID) // Read metadata associated with the object from all disks. metaArr, errs := readAllXLMetadata(xl.storageDisks, bucket, object) // Do we have read quorum? if !isDiskQuorum(errs, xl.readQuorum) { - return toObjectErr(errXLReadQuorum, bucket, object) + return traceError(InsufficientReadQuorum{}, errs...) } if reducedErr := reduceErrs(errs, []error{ @@ -89,24 +94,24 @@ func (xl xlObjects) GetObject(bucket, object string, startOffset int64, length i // Reply back invalid range if the input offset and length fall out of range. if startOffset > xlMeta.Stat.Size || length > xlMeta.Stat.Size { - return InvalidRange{startOffset, length, xlMeta.Stat.Size} + return traceError(InvalidRange{startOffset, length, xlMeta.Stat.Size}) } // Reply if we have inputs with offset and length. if startOffset+length > xlMeta.Stat.Size { - return InvalidRange{startOffset, length, xlMeta.Stat.Size} + return traceError(InvalidRange{startOffset, length, xlMeta.Stat.Size}) } // Get start part index and offset. partIndex, partOffset, err := xlMeta.ObjectToPartOffset(startOffset) if err != nil { - return toObjectErr(err, bucket, object) + return traceError(InvalidRange{startOffset, length, xlMeta.Stat.Size}) } // Get last part index to read given length. lastPartIndex, _, err := xlMeta.ObjectToPartOffset(startOffset + length - 1) if err != nil { - return toObjectErr(err, bucket, object) + return traceError(InvalidRange{startOffset, length, xlMeta.Stat.Size}) } // Save the writer. @@ -120,17 +125,17 @@ func (xl xlObjects) GetObject(bucket, object string, startOffset int64, length i if err == nil { // Cache hit. // Advance the buffer to offset as if it was read. if _, err = cachedBuffer.Seek(startOffset, 0); err != nil { // Seek to the offset. - return err + return traceError(err) } // Write the requested length. if _, err = io.CopyN(writer, cachedBuffer, length); err != nil { - return err + return traceError(err) } return nil } // Cache miss. // For unknown error, return and error out. if err != objcache.ErrKeyNotFoundInCache { - return err + return traceError(err) } // Cache has not been found, fill the cache. // Cache is only set if whole object is being read. @@ -147,7 +152,7 @@ func (xl xlObjects) GetObject(bucket, object string, startOffset int64, length i // Ignore error if cache is full, proceed to write the object. if err != nil && err != objcache.ErrCacheFull { // For any other error return here. - return toObjectErr(err, bucket, object) + return toObjectErr(traceError(err), bucket, object) } } } @@ -218,17 +223,21 @@ func (xl xlObjects) GetObject(bucket, object string, startOffset int64, length i func (xl xlObjects) HealObject(bucket, object string) error { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return BucketNameInvalid{Bucket: bucket} + return traceError(BucketNameInvalid{Bucket: bucket}) } // Verify if object is valid. if !IsValidObjectName(object) { // FIXME: return Invalid prefix. - return ObjectNameInvalid{Bucket: bucket, Object: object} + return traceError(ObjectNameInvalid{Bucket: bucket, Object: object}) } + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + // Lock the object before healing. - nsMutex.RLock(bucket, object) - defer nsMutex.RUnlock(bucket, object) + nsMutex.RLock(bucket, object, opsID) + defer nsMutex.RUnlock(bucket, object, opsID) partsMetadata, errs := readAllXLMetadata(xl.storageDisks, bucket, object) if err := reduceErrs(errs, nil); err != nil { @@ -266,13 +275,13 @@ func (xl xlObjects) HealObject(bucket, object string) error { err := disk.DeleteFile(bucket, pathJoin(object, outDatedMeta.Parts[partIndex].Name)) if err != nil { - return err + return traceError(err) } } // Delete xl.json file. err := disk.DeleteFile(bucket, pathJoin(object, xlMetaJSONFile)) if err != nil { - return err + return traceError(err) } } @@ -334,7 +343,7 @@ func (xl xlObjects) HealObject(bucket, object string) error { } err := disk.RenameFile(minioMetaBucket, retainSlash(pathJoin(tmpMetaPrefix, tmpID)), bucket, retainSlash(object)) if err != nil { - return err + return traceError(err) } } return nil @@ -350,8 +359,13 @@ func (xl xlObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) { if !IsValidObjectName(object) { return ObjectInfo{}, ObjectNameInvalid{Bucket: bucket, Object: object} } - nsMutex.RLock(bucket, object) - defer nsMutex.RUnlock(bucket, object) + + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + + nsMutex.RLock(bucket, object, opsID) + defer nsMutex.RUnlock(bucket, object, opsID) info, err := xl.getObjectInfo(bucket, object) if err != nil { return ObjectInfo{}, toObjectErr(err, bucket, object) @@ -361,22 +375,23 @@ func (xl xlObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) { // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo. func (xl xlObjects) getObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) { - var xlMeta xlMetaV1 - xlMeta, err = xl.readXLMetadata(bucket, object) + // returns xl meta map and stat info. + xlStat, xlMetaMap, err := xl.readXLMetaStat(bucket, object) if err != nil { // Return error. return ObjectInfo{}, err } + objInfo = ObjectInfo{ IsDir: false, Bucket: bucket, Name: object, - Size: xlMeta.Stat.Size, - ModTime: xlMeta.Stat.ModTime, - MD5Sum: xlMeta.Meta["md5Sum"], - ContentType: xlMeta.Meta["content-type"], - ContentEncoding: xlMeta.Meta["content-encoding"], - UserDefined: xlMeta.Meta, + Size: xlStat.Size, + ModTime: xlStat.ModTime, + MD5Sum: xlMetaMap["md5Sum"], + ContentType: xlMetaMap["content-type"], + ContentEncoding: xlMetaMap["content-encoding"], + UserDefined: xlMetaMap, } return objInfo, nil } @@ -433,7 +448,7 @@ func rename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, defer wg.Done() err := disk.RenameFile(srcBucket, srcEntry, dstBucket, dstEntry) if err != nil && err != errFileNotFound { - errs[index] = err + errs[index] = traceError(err) } }(index, disk) } @@ -446,7 +461,7 @@ func rename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, if !isDiskQuorum(errs, quorum) { // Undo all the partial rename operations. undoRename(disks, srcBucket, srcEntry, dstBucket, dstEntry, isPart, errs) - return errXLWriteQuorum + return traceError(errXLWriteQuorum) } // Return on first error, also undo any partially successful rename operations. return reduceErrs(errs, []error{ @@ -478,20 +493,20 @@ func renameObject(disks []StorageAPI, srcBucket, srcObject, dstBucket, dstObject // until EOF, erasure codes the data across all disk and additionally // writes `xl.json` which carries the necessary metadata for future // object operations. -func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (md5Sum string, err error) { +func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return "", BucketNameInvalid{Bucket: bucket} + return ObjectInfo{}, traceError(BucketNameInvalid{Bucket: bucket}) } // Verify bucket exists. if !xl.isBucketExist(bucket) { - return "", BucketNotFound{Bucket: bucket} + return ObjectInfo{}, traceError(BucketNotFound{Bucket: bucket}) } if !IsValidObjectName(object) { - return "", ObjectNameInvalid{ + return ObjectInfo{}, traceError(ObjectNameInvalid{ Bucket: bucket, Object: object, - } + }) } // No metadata is set, allocate a new one. if metadata == nil { @@ -524,7 +539,7 @@ func (xl xlObjects) PutObject(bucket string, object string, size int64, data io. // Ignore error if cache is full, proceed to write the object. if err != nil && err != objcache.ErrCacheFull { // For any other error return here. - return "", toObjectErr(err, bucket, object) + return ObjectInfo{}, toObjectErr(traceError(err), bucket, object) } } else { mw = md5Writer @@ -554,14 +569,14 @@ func (xl xlObjects) PutObject(bucket string, object string, size int64, data io. if err != nil { // Create file failed, delete temporary object. xl.deleteObject(minioMetaTmpBucket, tempObj) - return "", toObjectErr(err, minioMetaBucket, tempErasureObj) + return ObjectInfo{}, toObjectErr(err, minioMetaBucket, tempErasureObj) } // Should return IncompleteBody{} error when reader has fewer bytes // than specified in request header. if sizeWritten < size { // Short write, delete temporary object. xl.deleteObject(minioMetaTmpBucket, tempObj) - return "", IncompleteBody{} + return ObjectInfo{}, traceError(IncompleteBody{}) } // For size == -1, perhaps client is sending in chunked encoding @@ -594,7 +609,7 @@ func (xl xlObjects) PutObject(bucket string, object string, size int64, data io. // Incoming payload wrong, delete the temporary object. xl.deleteObject(minioMetaTmpBucket, tempObj) // Error return. - return "", toObjectErr(vErr, bucket, object) + return ObjectInfo{}, toObjectErr(traceError(vErr), bucket, object) } } @@ -605,20 +620,24 @@ func (xl xlObjects) PutObject(bucket string, object string, size int64, data io. // MD5 mismatch, delete the temporary object. xl.deleteObject(minioMetaTmpBucket, tempObj) // Returns md5 mismatch. - return "", BadDigest{md5Hex, newMD5Hex} + return ObjectInfo{}, traceError(BadDigest{md5Hex, newMD5Hex}) } } + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + // Lock the object. - nsMutex.Lock(bucket, object) - defer nsMutex.Unlock(bucket, object) + nsMutex.Lock(bucket, object, opsID) + defer nsMutex.Unlock(bucket, object, opsID) // Check if an object is present as one of the parent dir. // -- FIXME. (needs a new kind of lock). if xl.parentDirIsObject(bucket, path.Dir(object)) { // Parent (in the namespace) is an object, delete temporary object. xl.deleteObject(minioMetaTmpBucket, tempObj) - return "", toObjectErr(errFileAccessDenied, bucket, object) + return ObjectInfo{}, toObjectErr(traceError(errFileAccessDenied), bucket, object) } // Rename if an object already exists to temporary location. @@ -629,7 +648,7 @@ func (xl xlObjects) PutObject(bucket string, object string, size int64, data io. // regardless of `xl.json` status and rolled back in case of errors. err = renameObject(xl.storageDisks, bucket, object, minioMetaTmpBucket, newUniqueID, xl.writeQuorum) if err != nil { - return "", toObjectErr(err, bucket, object) + return ObjectInfo{}, toObjectErr(err, bucket, object) } } @@ -654,13 +673,13 @@ func (xl xlObjects) PutObject(bucket string, object string, size int64, data io. // Write unique `xl.json` for each disk. if err = writeUniqueXLMetadata(onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, xl.writeQuorum); err != nil { - return "", toObjectErr(err, bucket, object) + return ObjectInfo{}, toObjectErr(err, bucket, object) } // Rename the successfully written temporary object to final location. err = renameObject(onlineDisks, minioMetaTmpBucket, tempObj, bucket, object, xl.writeQuorum) if err != nil { - return "", toObjectErr(err, bucket, object) + return ObjectInfo{}, toObjectErr(err, bucket, object) } // Delete the temporary object. @@ -672,8 +691,18 @@ func (xl xlObjects) PutObject(bucket string, object string, size int64, data io. newBuffer.Close() } - // Return md5sum, successfully wrote object. - return newMD5Hex, nil + objInfo = ObjectInfo{ + IsDir: false, + Bucket: bucket, + Name: object, + Size: xlMeta.Stat.Size, + ModTime: xlMeta.Stat.ModTime, + MD5Sum: xlMeta.Meta["md5Sum"], + ContentType: xlMeta.Meta["content-type"], + ContentEncoding: xlMeta.Meta["content-encoding"], + UserDefined: xlMeta.Meta, + } + return objInfo, nil } // deleteObject - wrapper for delete object, deletes an object from @@ -688,14 +717,14 @@ func (xl xlObjects) deleteObject(bucket, object string) error { for index, disk := range xl.storageDisks { if disk == nil { - dErrs[index] = errDiskNotFound + dErrs[index] = traceError(errDiskNotFound) continue } wg.Add(1) go func(index int, disk StorageAPI) { defer wg.Done() err := cleanupDir(disk, bucket, object) - if err != nil && err != errFileNotFound { + if err != nil && errorCause(err) != errVolumeNotFound { dErrs[index] = err } }(index, disk) @@ -707,7 +736,7 @@ func (xl xlObjects) deleteObject(bucket, object string) error { // Do we have write quorum? if !isDiskQuorum(dErrs, xl.writeQuorum) { // Return errXLWriteQuorum if errors were more than allowed write quorum. - return errXLWriteQuorum + return traceError(errXLWriteQuorum) } return nil @@ -719,17 +748,22 @@ func (xl xlObjects) deleteObject(bucket, object string) error { func (xl xlObjects) DeleteObject(bucket, object string) (err error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return BucketNameInvalid{Bucket: bucket} + return traceError(BucketNameInvalid{Bucket: bucket}) } if !IsValidObjectName(object) { - return ObjectNameInvalid{Bucket: bucket, Object: object} + return traceError(ObjectNameInvalid{Bucket: bucket, Object: object}) } - nsMutex.Lock(bucket, object) - defer nsMutex.Unlock(bucket, object) + + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + + nsMutex.Lock(bucket, object, opsID) + defer nsMutex.Unlock(bucket, object, opsID) // Validate object exists. if !xl.isObject(bucket, object) { - return ObjectNotFound{bucket, object} + return traceError(ObjectNotFound{bucket, object}) } // else proceed to delete the object. // Delete the object on all disks. diff --git a/cmd/xl-v1-object_test.go b/cmd/xl-v1-object_test.go index 801bc6237..aa978539a 100644 --- a/cmd/xl-v1-object_test.go +++ b/cmd/xl-v1-object_test.go @@ -29,7 +29,7 @@ func TestRepeatPutObjectPart(t *testing.T) { var disks []string var err error - objLayer, disks, err = getXLObjectLayer() + objLayer, disks, err = prepareXL() if err != nil { t.Fatal(err) } @@ -77,7 +77,7 @@ func TestXLDeleteObjectBasic(t *testing.T) { } // Create an instance of xl backend - xl, fsDirs, err := getXLObjectLayer() + xl, fsDirs, err := prepareXL() if err != nil { t.Fatal(err) } @@ -95,6 +95,7 @@ func TestXLDeleteObjectBasic(t *testing.T) { } for i, test := range testCases { actualErr := xl.DeleteObject(test.bucket, test.object) + actualErr = errorCause(actualErr) if test.expectedErr != nil && actualErr != test.expectedErr { t.Errorf("Test %d: Expected to fail with %s, but failed with %s", i+1, test.expectedErr, actualErr) } @@ -108,7 +109,7 @@ func TestXLDeleteObjectBasic(t *testing.T) { func TestXLDeleteObjectDiskNotFound(t *testing.T) { // Create an instance of xl backend. - obj, fsDirs, err := getXLObjectLayer() + obj, fsDirs, err := prepareXL() if err != nil { t.Fatal(err) } @@ -125,7 +126,9 @@ func TestXLDeleteObjectDiskNotFound(t *testing.T) { object := "object" // Create object "obj" under bucket "bucket". _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil) - + if err != nil { + t.Fatal(err) + } // for a 16 disk setup, quorum is 9. To simulate disks not found yet // quorum is available, we remove disks leaving quorum disks behind. for i := range xl.storageDisks[:7] { @@ -146,6 +149,7 @@ func TestXLDeleteObjectDiskNotFound(t *testing.T) { xl.storageDisks[7] = nil xl.storageDisks[8] = nil err = obj.DeleteObject(bucket, object) + err = errorCause(err) if err != toObjectErr(errXLWriteQuorum, bucket, object) { t.Errorf("Expected deleteObject to fail with %v, but failed with %v", toObjectErr(errXLWriteQuorum, bucket, object), err) } @@ -155,7 +159,7 @@ func TestXLDeleteObjectDiskNotFound(t *testing.T) { func TestGetObjectNoQuorum(t *testing.T) { // Create an instance of xl backend. - obj, fsDirs, err := getXLObjectLayer() + obj, fsDirs, err := prepareXL() if err != nil { t.Fatal(err) } @@ -196,6 +200,7 @@ func TestGetObjectNoQuorum(t *testing.T) { } // Fetch object from store. err = xl.GetObject(bucket, object, 0, int64(len("abcd")), ioutil.Discard) + err = errorCause(err) if err != toObjectErr(errXLReadQuorum, bucket, object) { t.Errorf("Expected putObject to fail with %v, but failed with %v", toObjectErr(errXLWriteQuorum, bucket, object), err) } @@ -206,7 +211,7 @@ func TestGetObjectNoQuorum(t *testing.T) { func TestPutObjectNoQuorum(t *testing.T) { // Create an instance of xl backend. - obj, fsDirs, err := getXLObjectLayer() + obj, fsDirs, err := prepareXL() if err != nil { t.Fatal(err) } @@ -246,6 +251,7 @@ func TestPutObjectNoQuorum(t *testing.T) { } // Upload new content to same object "object" _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil) + err = errorCause(err) if err != toObjectErr(errXLWriteQuorum, bucket, object) { t.Errorf("Expected putObject to fail with %v, but failed with %v", toObjectErr(errXLWriteQuorum, bucket, object), err) } diff --git a/cmd/xl-v1-utils.go b/cmd/xl-v1-utils.go index 66c5ac4a9..4813bbec5 100644 --- a/cmd/xl-v1-utils.go +++ b/cmd/xl-v1-utils.go @@ -17,10 +17,12 @@ package cmd import ( - "encoding/json" "hash/crc32" "path" "sync" + "time" + + "github.com/tidwall/gjson" ) // Returns number of errors that occurred the most (incl. nil) and the @@ -32,6 +34,7 @@ import ( func reduceErrs(errs []error, ignoredErrs []error) error { errorCounts := make(map[error]int) + errs = errorsCause(errs) for _, err := range errs { if isErrIgnored(err, ignoredErrs) { continue @@ -46,13 +49,14 @@ func reduceErrs(errs []error, ignoredErrs []error) error { errMax = err } } - return errMax + return traceError(errMax, errs...) } // Validates if we have quorum based on the errors related to disk only. // Returns 'true' if we have quorum, 'false' if we don't. func isDiskQuorum(errs []error, minQuorumCount int) bool { var count int + errs = errorsCause(errs) for _, err := range errs { switch err { case errDiskNotFound, errFaultyDisk, errDiskAccessDenied: @@ -60,6 +64,7 @@ func isDiskQuorum(errs []error, minQuorumCount int) bool { } count++ } + return count >= minQuorumCount } @@ -96,19 +101,160 @@ func hashOrder(key string, cardinality int) []int { return nums } -// readXLMeta reads `xl.json` and returns back XL metadata structure. -func readXLMeta(disk StorageAPI, bucket string, object string) (xlMeta xlMetaV1, err error) { - // Reads entire `xl.json`. - buf, err := disk.ReadAll(bucket, path.Join(object, xlMetaJSONFile)) +func parseXLStat(xlMetaBuf []byte) (statInfo, error) { + // obtain stat info. + stat := statInfo{} + // fetching modTime. + modTime, err := time.Parse(time.RFC3339, gjson.GetBytes(xlMetaBuf, "stat.modTime").String()) + if err != nil { + return statInfo{}, err + } + stat.ModTime = modTime + // obtain Stat.Size . + stat.Size = gjson.GetBytes(xlMetaBuf, "stat.size").Int() + return stat, nil +} + +func parseXLVersion(xlMetaBuf []byte) string { + return gjson.GetBytes(xlMetaBuf, "version").String() +} + +func parseXLFormat(xlMetaBuf []byte) string { + return gjson.GetBytes(xlMetaBuf, "format").String() +} + +func parseXLRelease(xlMetaBuf []byte) string { + return gjson.GetBytes(xlMetaBuf, "minio.release").String() +} + +func parseXLErasureInfo(xlMetaBuf []byte) erasureInfo { + erasure := erasureInfo{} + erasureResult := gjson.GetBytes(xlMetaBuf, "erasure") + // parse the xlV1Meta.Erasure.Distribution. + disResult := erasureResult.Get("distribution").Array() + + distribution := make([]int, len(disResult)) + for i, dis := range disResult { + distribution[i] = int(dis.Int()) + } + erasure.Distribution = distribution + + erasure.Algorithm = erasureResult.Get("algorithm").String() + erasure.DataBlocks = int(erasureResult.Get("data").Int()) + erasure.ParityBlocks = int(erasureResult.Get("parity").Int()) + erasure.BlockSize = erasureResult.Get("blockSize").Int() + erasure.Index = int(erasureResult.Get("index").Int()) + // Pare xlMetaV1.Erasure.Checksum array. + checkSumsResult := erasureResult.Get("checksum").Array() + checkSums := make([]checkSumInfo, len(checkSumsResult)) + for i, checkSumResult := range checkSumsResult { + checkSum := checkSumInfo{} + checkSum.Name = checkSumResult.Get("name").String() + checkSum.Algorithm = checkSumResult.Get("algorithm").String() + checkSum.Hash = checkSumResult.Get("hash").String() + checkSums[i] = checkSum + } + erasure.Checksum = checkSums + + return erasure +} + +func parseXLParts(xlMetaBuf []byte) []objectPartInfo { + // Parse the XL Parts. + partsResult := gjson.GetBytes(xlMetaBuf, "parts").Array() + partInfo := make([]objectPartInfo, len(partsResult)) + for i, p := range partsResult { + info := objectPartInfo{} + info.Number = int(p.Get("number").Int()) + info.Name = p.Get("name").String() + info.ETag = p.Get("etag").String() + info.Size = p.Get("size").Int() + partInfo[i] = info + } + return partInfo +} + +func parseXLMetaMap(xlMetaBuf []byte) map[string]string { + // Get xlMetaV1.Meta map. + metaMapResult := gjson.GetBytes(xlMetaBuf, "meta").Map() + metaMap := make(map[string]string) + for key, valResult := range metaMapResult { + metaMap[key] = valResult.String() + } + return metaMap +} + +// Constructs XLMetaV1 using `gjson` lib to retrieve each field. +func xlMetaV1UnmarshalJSON(xlMetaBuf []byte) (xlMetaV1, error) { + xlMeta := xlMetaV1{} + // obtain version. + xlMeta.Version = parseXLVersion(xlMetaBuf) + // obtain format. + xlMeta.Format = parseXLFormat(xlMetaBuf) + // Parse xlMetaV1.Stat . + stat, err := parseXLStat(xlMetaBuf) if err != nil { return xlMetaV1{}, err } - // Unmarshal xl metadata. - if err = json.Unmarshal(buf, &xlMeta); err != nil { - return xlMetaV1{}, err - } + xlMeta.Stat = stat + // parse the xlV1Meta.Erasure fields. + xlMeta.Erasure = parseXLErasureInfo(xlMetaBuf) + // Parse the XL Parts. + xlMeta.Parts = parseXLParts(xlMetaBuf) + // Get the xlMetaV1.Realse field. + xlMeta.Minio.Release = parseXLRelease(xlMetaBuf) + // parse xlMetaV1. + xlMeta.Meta = parseXLMetaMap(xlMetaBuf) + + return xlMeta, nil +} + +// read xl.json from the given disk, parse and return xlV1MetaV1.Parts. +func readXLMetaParts(disk StorageAPI, bucket string, object string) ([]objectPartInfo, error) { + // Reads entire `xl.json`. + xlMetaBuf, err := disk.ReadAll(bucket, path.Join(object, xlMetaJSONFile)) + if err != nil { + return nil, traceError(err) + } + // obtain xlMetaV1{}.Partsusing `github.com/tidwall/gjson`. + xlMetaParts := parseXLParts(xlMetaBuf) + + return xlMetaParts, nil +} + +// read xl.json from the given disk and parse xlV1Meta.Stat and xlV1Meta.Meta using gjson. +func readXLMetaStat(disk StorageAPI, bucket string, object string) (statInfo, map[string]string, error) { + // Reads entire `xl.json`. + xlMetaBuf, err := disk.ReadAll(bucket, path.Join(object, xlMetaJSONFile)) + if err != nil { + return statInfo{}, nil, traceError(err) + } + // obtain xlMetaV1{}.Meta using `github.com/tidwall/gjson`. + xlMetaMap := parseXLMetaMap(xlMetaBuf) + + // obtain xlMetaV1{}.Stat using `github.com/tidwall/gjson`. + xlStat, err := parseXLStat(xlMetaBuf) + if err != nil { + return statInfo{}, nil, traceError(err) + } + // Return structured `xl.json`. + return xlStat, xlMetaMap, nil +} + +// readXLMeta reads `xl.json` and returns back XL metadata structure. +func readXLMeta(disk StorageAPI, bucket string, object string) (xlMeta xlMetaV1, err error) { + // Reads entire `xl.json`. + xlMetaBuf, err := disk.ReadAll(bucket, path.Join(object, xlMetaJSONFile)) + if err != nil { + return xlMetaV1{}, traceError(err) + } + // obtain xlMetaV1{} using `github.com/tidwall/gjson`. + xlMeta, err = xlMetaV1UnmarshalJSON(xlMetaBuf) + if err != nil { + return xlMetaV1{}, traceError(err) + } // Return structured `xl.json`. return xlMeta, nil } diff --git a/cmd/xl-v1-utils_test.go b/cmd/xl-v1-utils_test.go index 85af8d0d9..422550aaa 100644 --- a/cmd/xl-v1-utils_test.go +++ b/cmd/xl-v1-utils_test.go @@ -17,8 +17,11 @@ package cmd import ( + "encoding/json" "reflect" + "strconv" "testing" + "time" ) // Test for reduceErrs, reduceErr reduces collection @@ -55,7 +58,7 @@ func TestReduceErrs(t *testing.T) { // Validates list of all the testcases for returning valid errors. for i, testCase := range testCases { gotErr := reduceErrs(testCase.errs, testCase.ignoredErrs) - if testCase.err != gotErr { + if errorCause(gotErr) != testCase.err { t.Errorf("Test %d : expected %s, got %s", i+1, testCase.err, gotErr) } } @@ -93,3 +96,201 @@ func TestHashOrder(t *testing.T) { t.Errorf("Test: Expect \"nil\" but failed \"%#v\"", hashedOrder) } } + +// newTestXLMetaV1 - initializes new xlMetaV1, adds version, allocates a fresh erasure info and metadata. +func newTestXLMetaV1() xlMetaV1 { + xlMeta := xlMetaV1{} + xlMeta.Version = "1.0.0" + xlMeta.Format = "xl" + xlMeta.Minio.Release = "1.0.0" + xlMeta.Erasure = erasureInfo{ + Algorithm: "klauspost/reedsolomon/vandermonde", + DataBlocks: 5, + ParityBlocks: 5, + BlockSize: 10485760, + Index: 10, + Distribution: []int{9, 10, 1, 2, 3, 4, 5, 6, 7, 8}, + } + xlMeta.Stat = statInfo{ + Size: int64(20), + ModTime: time.Now().UTC(), + } + // Set meta data. + xlMeta.Meta = make(map[string]string) + xlMeta.Meta["testKey1"] = "val1" + xlMeta.Meta["testKey2"] = "val2" + return xlMeta +} + +func (m *xlMetaV1) AddTestObjectCheckSum(checkSumNum int, name string, hash string, algo string) { + checkSum := checkSumInfo{ + Name: name, + Algorithm: algo, + Hash: hash, + } + m.Erasure.Checksum[checkSumNum] = checkSum +} + +// AddTestObjectPart - add a new object part in order. +func (m *xlMetaV1) AddTestObjectPart(partNumber int, partName string, partETag string, partSize int64) { + partInfo := objectPartInfo{ + Number: partNumber, + Name: partName, + ETag: partETag, + Size: partSize, + } + + // Proceed to include new part info. + m.Parts[partNumber] = partInfo +} + +// Constructs xlMetaV1{} for given number of parts and converts it into bytes. +func getXLMetaBytes(totalParts int) []byte { + xlSampleMeta := getSampleXLMeta(totalParts) + xlMetaBytes, err := json.Marshal(xlSampleMeta) + if err != nil { + panic(err) + } + return xlMetaBytes +} + +// Returns sample xlMetaV1{} for number of parts. +func getSampleXLMeta(totalParts int) xlMetaV1 { + xlMeta := newTestXLMetaV1() + // Number of checksum info == total parts. + xlMeta.Erasure.Checksum = make([]checkSumInfo, totalParts) + // total number of parts. + xlMeta.Parts = make([]objectPartInfo, totalParts) + for i := 0; i < totalParts; i++ { + partName := "part." + strconv.Itoa(i+1) + // hard coding hash and algo value for the checksum, Since we are benchmarking the parsing of xl.json the magnitude doesn't affect the test, + // The magnitude doesn't make a difference, only the size does. + xlMeta.AddTestObjectCheckSum(i, partName, "a23f5eff248c4372badd9f3b2455a285cd4ca86c3d9a570b091d3fc5cd7ca6d9484bbea3f8c5d8d4f84daae96874419eda578fd736455334afbac2c924b3915a", "blake2b") + xlMeta.AddTestObjectPart(i, partName, "d3fdd79cc3efd5fe5c068d7be397934b", 67108864) + } + return xlMeta +} + +// Compare the unmarshaled XLMetaV1 with the one obtained from gjson parsing. +func compareXLMetaV1(t *testing.T, unMarshalXLMeta, gjsonXLMeta xlMetaV1) { + + // Start comparing the fields of xlMetaV1 obtained from gjson parsing with one parsed using json unmarshaling. + if unMarshalXLMeta.Version != gjsonXLMeta.Version { + t.Errorf("Expected the Version to be \"%s\", but got \"%s\".", unMarshalXLMeta.Version, gjsonXLMeta.Version) + } + if unMarshalXLMeta.Format != gjsonXLMeta.Format { + t.Errorf("Expected the format to be \"%s\", but got \"%s\".", unMarshalXLMeta.Format, gjsonXLMeta.Format) + } + if unMarshalXLMeta.Stat.Size != gjsonXLMeta.Stat.Size { + t.Errorf("Expected the stat size to be %v, but got %v.", unMarshalXLMeta.Stat.Size, gjsonXLMeta.Stat.Size) + } + if unMarshalXLMeta.Stat.ModTime != gjsonXLMeta.Stat.ModTime { + t.Errorf("Expected the modTime to be \"%v\", but got \"%v\".", unMarshalXLMeta.Stat.ModTime, gjsonXLMeta.Stat.ModTime) + } + if unMarshalXLMeta.Erasure.Algorithm != gjsonXLMeta.Erasure.Algorithm { + t.Errorf("Expected the erasure algorithm to be \"%v\", but got \"%v\".", unMarshalXLMeta.Erasure.Algorithm, gjsonXLMeta.Erasure.Algorithm) + } + if unMarshalXLMeta.Erasure.DataBlocks != gjsonXLMeta.Erasure.DataBlocks { + t.Errorf("Expected the erasure data blocks to be %v, but got %v.", unMarshalXLMeta.Erasure.DataBlocks, gjsonXLMeta.Erasure.DataBlocks) + } + if unMarshalXLMeta.Erasure.ParityBlocks != gjsonXLMeta.Erasure.ParityBlocks { + t.Errorf("Expected the erasure parity blocks to be %v, but got %v.", unMarshalXLMeta.Erasure.ParityBlocks, gjsonXLMeta.Erasure.ParityBlocks) + } + if unMarshalXLMeta.Erasure.BlockSize != gjsonXLMeta.Erasure.BlockSize { + t.Errorf("Expected the erasure block size to be %v, but got %v.", unMarshalXLMeta.Erasure.BlockSize, gjsonXLMeta.Erasure.BlockSize) + } + if unMarshalXLMeta.Erasure.Index != gjsonXLMeta.Erasure.Index { + t.Errorf("Expected the erasure index to be %v, but got %v.", unMarshalXLMeta.Erasure.Index, gjsonXLMeta.Erasure.Index) + } + if len(unMarshalXLMeta.Erasure.Distribution) != len(gjsonXLMeta.Erasure.Distribution) { + t.Errorf("Expected the size of Erasure Distribution to be %d, but got %d.", len(unMarshalXLMeta.Erasure.Distribution), len(gjsonXLMeta.Erasure.Distribution)) + } else { + for i := 0; i < len(unMarshalXLMeta.Erasure.Distribution); i++ { + if unMarshalXLMeta.Erasure.Distribution[i] != gjsonXLMeta.Erasure.Distribution[i] { + t.Errorf("Expected the Erasure Distribution to be %d, got %d.", unMarshalXLMeta.Erasure.Distribution[i], gjsonXLMeta.Erasure.Distribution[i]) + } + } + } + + if len(unMarshalXLMeta.Erasure.Checksum) != len(gjsonXLMeta.Erasure.Checksum) { + t.Errorf("Expected the size of Erasure Checksum to be %d, but got %d.", len(unMarshalXLMeta.Erasure.Checksum), len(gjsonXLMeta.Erasure.Checksum)) + } else { + for i := 0; i < len(unMarshalXLMeta.Erasure.Checksum); i++ { + if unMarshalXLMeta.Erasure.Checksum[i].Name != gjsonXLMeta.Erasure.Checksum[i].Name { + t.Errorf("Expected the Erasure Checksum Name to be \"%s\", got \"%s\".", unMarshalXLMeta.Erasure.Checksum[i].Name, gjsonXLMeta.Erasure.Checksum[i].Name) + } + if unMarshalXLMeta.Erasure.Checksum[i].Algorithm != gjsonXLMeta.Erasure.Checksum[i].Algorithm { + t.Errorf("Expected the Erasure Checksum Algorithm to be \"%s\", got \"%s.\"", unMarshalXLMeta.Erasure.Checksum[i].Algorithm, gjsonXLMeta.Erasure.Checksum[i].Algorithm) + } + if unMarshalXLMeta.Erasure.Checksum[i] != gjsonXLMeta.Erasure.Checksum[i] { + t.Errorf("Expected the Erasure Checksum Hash to be \"%s\", got \"%s\".", unMarshalXLMeta.Erasure.Checksum[i].Hash, gjsonXLMeta.Erasure.Checksum[i].Hash) + } + } + } + if unMarshalXLMeta.Minio.Release != gjsonXLMeta.Minio.Release { + t.Errorf("Expected the Release string to be \"%s\", but got \"%s\".", unMarshalXLMeta.Minio.Release, gjsonXLMeta.Minio.Release) + } + if len(unMarshalXLMeta.Parts) != len(gjsonXLMeta.Parts) { + t.Errorf("Expected info of %d parts to be present, but got %d instead.", len(unMarshalXLMeta.Parts), len(gjsonXLMeta.Parts)) + } else { + for i := 0; i < len(unMarshalXLMeta.Parts); i++ { + if unMarshalXLMeta.Parts[i].Name != gjsonXLMeta.Parts[i].Name { + t.Errorf("Expected the name of part %d to be \"%s\", got \"%s\".", i+1, unMarshalXLMeta.Parts[i].Name, gjsonXLMeta.Parts[i].Name) + } + if unMarshalXLMeta.Parts[i].ETag != gjsonXLMeta.Parts[i].ETag { + t.Errorf("Expected the ETag of part %d to be \"%s\", got \"%s\".", i+1, unMarshalXLMeta.Parts[i].ETag, gjsonXLMeta.Parts[i].ETag) + } + if unMarshalXLMeta.Parts[i].Number != gjsonXLMeta.Parts[i].Number { + t.Errorf("Expected the number of part %d to be \"%d\", got \"%d\".", i+1, unMarshalXLMeta.Parts[i].Number, gjsonXLMeta.Parts[i].Number) + } + if unMarshalXLMeta.Parts[i].Size != gjsonXLMeta.Parts[i].Size { + t.Errorf("Expected the size of part %d to be %v, got %v.", i+1, unMarshalXLMeta.Parts[i].Size, gjsonXLMeta.Parts[i].Size) + } + } + } + + for key, val := range unMarshalXLMeta.Meta { + gjsonVal, exists := gjsonXLMeta.Meta[key] + if !exists { + t.Errorf("No meta data entry for Key \"%s\" exists.", key) + } + if val != gjsonVal { + t.Errorf("Expected the value for Meta data key \"%s\" to be \"%s\", but got \"%s\".", key, val, gjsonVal) + } + + } +} + +// Tests the correctness of constructing XLMetaV1 using gjson lib. +// The result will be compared with the result obtained from json.unMarshal of the byte data. +func TestGetXLMetaV1GJson1(t *testing.T) { + xlMetaJSON := getXLMetaBytes(1) + + var unMarshalXLMeta xlMetaV1 + if err := json.Unmarshal(xlMetaJSON, &unMarshalXLMeta); err != nil { + t.Errorf("Unmarshalling failed") + } + + gjsonXLMeta, err := xlMetaV1UnmarshalJSON(xlMetaJSON) + if err != nil { + t.Errorf("gjson parsing of XLMeta failed") + } + compareXLMetaV1(t, unMarshalXLMeta, gjsonXLMeta) +} + +// Tests the correctness of constructing XLMetaV1 using gjson lib for XLMetaV1 of size 10 parts. +// The result will be compared with the result obtained from json.unMarshal of the byte data. +func TestGetXLMetaV1GJson10(t *testing.T) { + + xlMetaJSON := getXLMetaBytes(10) + + var unMarshalXLMeta xlMetaV1 + if err := json.Unmarshal(xlMetaJSON, &unMarshalXLMeta); err != nil { + t.Errorf("Unmarshalling failed") + } + gjsonXLMeta, err := xlMetaV1UnmarshalJSON(xlMetaJSON) + if err != nil { + t.Errorf("gjson parsing of XLMeta failed") + } + compareXLMetaV1(t, unMarshalXLMeta, gjsonXLMeta) +} diff --git a/cmd/xl-v1.go b/cmd/xl-v1.go index f80f1a49f..5a98d503f 100644 --- a/cmd/xl-v1.go +++ b/cmd/xl-v1.go @@ -20,6 +20,7 @@ import ( "fmt" "sort" + "github.com/minio/minio-go/pkg/set" "github.com/minio/minio/pkg/disk" "github.com/minio/minio/pkg/objcache" ) @@ -50,12 +51,11 @@ const ( // xlObjects - Implements XL object layer. type xlObjects struct { - physicalDisks []string // Collection of regular disks. - storageDisks []StorageAPI // Collection of initialized backend disks. - dataBlocks int // dataBlocks count caculated for erasure. - parityBlocks int // parityBlocks count calculated for erasure. - readQuorum int // readQuorum minimum required disks to read data. - writeQuorum int // writeQuorum minimum required disks to write data. + storageDisks []StorageAPI // Collection of initialized backend disks. + dataBlocks int // dataBlocks count caculated for erasure. + parityBlocks int // parityBlocks count calculated for erasure. + readQuorum int // readQuorum minimum required disks to read data. + writeQuorum int // writeQuorum minimum required disks to write data. // ListObjects pool management. listPool *treeWalkPool @@ -67,49 +67,50 @@ type xlObjects struct { objCacheEnabled bool } -// Validate if input disks are sufficient for initializing XL. -func checkSufficientDisks(disks []string) error { - // Verify total number of disks. - totalDisks := len(disks) - if totalDisks > maxErasureBlocks { - return errXLMaxDisks - } - if totalDisks < minErasureBlocks { - return errXLMinDisks +func repairDiskMetadata(storageDisks []StorageAPI) error { + // Attempt to load all `format.json`. + formatConfigs, sErrs := loadAllFormats(storageDisks) + + // Generic format check validates + // if (no quorum) return error + // if (disks not recognized) // Always error. + if err := genericFormatCheck(formatConfigs, sErrs); err != nil { + return err } - // isEven function to verify if a given number if even. - isEven := func(number int) bool { - return number%2 == 0 + // Handles different cases properly. + switch reduceFormatErrs(sErrs, len(storageDisks)) { + case errCorruptedFormat: + if err := healFormatXLCorruptedDisks(storageDisks); err != nil { + return fmt.Errorf("Unable to repair corrupted format, %s", err) + } + case errSomeDiskUnformatted: + // All drives online but some report missing format.json. + if err := healFormatXLFreshDisks(storageDisks); err != nil { + // There was an unexpected unrecoverable error during healing. + return fmt.Errorf("Unable to heal backend %s", err) + } + case errSomeDiskOffline: + // FIXME: in future. + return fmt.Errorf("Unable to initialize format %s and %s", errSomeDiskOffline, errSomeDiskUnformatted) } - - // Verify if we have even number of disks. - // only combination of 4, 6, 8, 10, 12, 14, 16 are supported. - if !isEven(totalDisks) { - return errXLNumDisks - } - - // Success. return nil } -// isDiskFound - validates if the disk is found in a list of input disks. -func isDiskFound(disk string, disks []string) bool { - return contains(disks, disk) -} - // newXLObjects - initialize new xl object layer. func newXLObjects(disks, ignoredDisks []string) (ObjectLayer, error) { - // Validate if input disks are sufficient. - if err := checkSufficientDisks(disks); err != nil { - return nil, err + if disks == nil { + return nil, errInvalidArgument + } + disksSet := set.NewStringSet() + if len(ignoredDisks) > 0 { + disksSet = set.CreateStringSet(ignoredDisks...) } - // Bootstrap disks. storageDisks := make([]StorageAPI, len(disks)) for index, disk := range disks { // Check if disk is ignored. - if isDiskFound(disk, ignoredDisks) { + if disksSet.Contains(disk) { storageDisks[index] = nil continue } @@ -118,46 +119,16 @@ func newXLObjects(disks, ignoredDisks []string) (ObjectLayer, error) { // to handle these errors internally. storageDisks[index], err = newStorageAPI(disk) if err != nil && err != errDiskNotFound { + switch diskType := storageDisks[index].(type) { + case networkStorage: + diskType.rpcClient.Close() + } return nil, err } } - // Attempt to load all `format.json`. - formatConfigs, sErrs := loadAllFormats(storageDisks) - - // Generic format check validates - // if (no quorum) return error - // if (disks not recognized) // Always error. - if err := genericFormatCheck(formatConfigs, sErrs); err != nil { - return nil, err - } - - // Initialize meta volume, if volume already exists ignores it. - if err := initMetaVolume(storageDisks); err != nil { - return nil, fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err) - } - - // Handles different cases properly. - switch reduceFormatErrs(sErrs, len(storageDisks)) { - case errCorruptedFormat: - if err := healFormatXLCorruptedDisks(storageDisks); err != nil { - return nil, fmt.Errorf("Unable to repair corrupted format, %s", err) - } - case errUnformattedDisk: - // All drives online but fresh, initialize format. - if err := initFormatXL(storageDisks); err != nil { - return nil, fmt.Errorf("Unable to initialize format, %s", err) - } - case errSomeDiskUnformatted: - // All drives online but some report missing format.json. - if err := healFormatXLFreshDisks(storageDisks); err != nil { - // There was an unexpected unrecoverable error during healing. - return nil, fmt.Errorf("Unable to heal backend %s", err) - } - case errSomeDiskOffline: - // FIXME: in future. - return nil, fmt.Errorf("Unable to initialize format %s and %s", errSomeDiskOffline, errSomeDiskUnformatted) - } + // Fix format files in case of fresh or corrupted disks + repairDiskMetadata(storageDisks) // Runs house keeping code, like t, cleaning up tmp files etc. if err := xlHouseKeeping(storageDisks); err != nil { @@ -182,7 +153,6 @@ func newXLObjects(disks, ignoredDisks []string) (ObjectLayer, error) { // Initialize xl objects. xl := xlObjects{ - physicalDisks: disks, storageDisks: newPosixDisks, dataBlocks: dataBlocks, parityBlocks: parityBlocks, @@ -206,6 +176,17 @@ func (xl xlObjects) Shutdown() error { return nil } +// HealDiskMetadata function for object storage interface. +func (xl xlObjects) HealDiskMetadata() error { + // generates random string on setting MINIO_DEBUG=lock, else returns empty string. + // used for instrumentation on locks. + opsID := getOpsID() + + nsMutex.Lock(minioMetaBucket, formatConfigFile, opsID) + defer nsMutex.Unlock(minioMetaBucket, formatConfigFile, opsID) + return repairDiskMetadata(xl.storageDisks) +} + // byDiskTotal is a collection satisfying sort.Interface. type byDiskTotal []disk.Info @@ -218,10 +199,10 @@ func (d byDiskTotal) Less(i, j int) bool { // StorageInfo - returns underlying storage statistics. func (xl xlObjects) StorageInfo() StorageInfo { var disksInfo []disk.Info - for _, diskPath := range xl.physicalDisks { - info, err := disk.GetInfo(diskPath) + for _, storageDisk := range xl.storageDisks { + info, err := storageDisk.DiskInfo() if err != nil { - errorIf(err, "Unable to fetch disk info for "+diskPath) + errorIf(err, "Unable to fetch disk info for %#v", storageDisk) continue } disksInfo = append(disksInfo, info) diff --git a/cmd/xl-v1_test.go b/cmd/xl-v1_test.go index e31aa803a..5542c109a 100644 --- a/cmd/xl-v1_test.go +++ b/cmd/xl-v1_test.go @@ -92,7 +92,7 @@ func TestCheckSufficientDisks(t *testing.T) { // TestStorageInfo - tests storage info. func TestStorageInfo(t *testing.T) { - objLayer, fsDirs, err := getXLObjectLayer() + objLayer, fsDirs, err := prepareXL() if err != nil { t.Fatalf("Unable to initialize 'XL' object layer.") } @@ -130,8 +130,25 @@ func TestNewXL(t *testing.T) { erasureDisks = append(erasureDisks, disk) defer removeAll(disk) } + + // No disks input. + _, err := newXLObjects(nil, nil) + if err != errInvalidArgument { + t.Fatalf("Unable to initialize erasure, %s", err) + } + // Initializes all erasure disks - _, err := newXLObjects(erasureDisks, nil) + err = formatDisks(erasureDisks, nil) + if err != nil { + t.Fatalf("Unable to format disks for erasure, %s", err) + } + _, err = newXLObjects(erasureDisks, nil) + if err != nil { + t.Fatalf("Unable to initialize erasure, %s", err) + } + + // Initializes all erasure disks, ignoring first two. + _, err = newXLObjects(erasureDisks, erasureDisks[:2]) if err != nil { t.Fatalf("Unable to initialize erasure, %s", err) } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..010656ee6 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,46 @@ +version: '2' + +# starts 4 docker containers running minio server instances. Each +# minio server's web interface will be accessible on the host at port +# 9001 through 9004. +services: + minio1: + image: minio/minio + ports: + - "9001:9000" + volumes: + - /mnt/export/minio1:/export + environment: + MINIO_ACCESS_KEY: abcd1 + MINIO_SECRET_KEY: abcd1234 + command: minio1:/export minio2:/export minio3:/export minio4:/export + minio2: + image: minio/minio + ports: + - "9002:9000" + volumes: + - /mnt/export/minio2:/export + environment: + MINIO_ACCESS_KEY: abcd1 + MINIO_SECRET_KEY: abcd1234 + command: minio1:/export minio2:/export minio3:/export minio4:/export + minio3: + image: minio/minio + ports: + - "9003:9000" + volumes: + - /mnt/export/minio3:/export + environment: + MINIO_ACCESS_KEY: abcd1 + MINIO_SECRET_KEY: abcd1234 + command: minio1:/export minio2:/export minio3:/export minio4:/export + minio4: + image: minio/minio + ports: + - "9004:9000" + volumes: + - /mnt/export/minio4:/export + environment: + MINIO_ACCESS_KEY: abcd1 + MINIO_SECRET_KEY: abcd1234 + command: minio1:/export minio2:/export minio3:/export minio4:/export diff --git a/docs/Docker.md b/docs/Docker.md index fd8b880a3..c03ca3c0f 100644 --- a/docs/Docker.md +++ b/docs/Docker.md @@ -10,9 +10,9 @@ docker run -p 9000:9000 minio/minio /export ``` -## 2. Run Minio Docker Container +## 2. Run One Minio Docker Container -Minio container requires a persistent volume to store configuration and application data. Following command maps local persistent directories from the host OS to virtual config `~/.minio` and export `/export` directories. +Minio container requires a persistent volume to store configuration and application data. Following command maps local persistent directories from the host OS to virtual config `~/.minio` and export `/export` directories. ```sh @@ -37,3 +37,17 @@ docker run -p 9000:9000 --name minio1 \ minio/minio /export ``` + +## 4. Run Minio in clustered mode + +Let's consider that we need to run 4 minio servers inside different docker containers. The `docker-compose.yml` file in the root of the project sets this up using the [docker-compose](https://docs.docker.com/compose/) tool. + +### Start Minio docker instances + +From the root directory of the project, run: + +``` shell +$ docker-compose up +``` + +Each instance's minio web-server is accessible on the host at ports 9001 through 9004, so you may access the first one at http://localhost:9001/ diff --git a/vendor/github.com/minio/dsync/LICENSE b/vendor/github.com/minio/dsync/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/vendor/github.com/minio/dsync/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/minio/dsync/README.md b/vendor/github.com/minio/dsync/README.md new file mode 100644 index 000000000..80326d6f5 --- /dev/null +++ b/vendor/github.com/minio/dsync/README.md @@ -0,0 +1,81 @@ +dsync +===== + +A distributed sync package. + +Introduction +------------ + +`dsync` is a package for doing distributed locks over a network of `n` nodes. It is designed with simplicity in mind and hence offers limited scalability (`n <= 16`). Each node will be connected to all other nodes and lock requests from any node will be broadcast to all connected nodes. A node will succeed in getting the lock if `n/2 + 1` nodes (including itself) respond positively. If the lock is acquired it can be held for some time and needs to be released afterwards. This will cause the release to be broadcast to all nodes after which the lock becomes available again. + +Design goals +------------ + +* Simple design: by keeping the design simple, many tricky edge cases can be avoided. +* No master node: there is no concept of a master node which, if this would be used and the master would be down, causes locking to come to a complete stop. (Unless you have a design with a slave node but this adds yet more complexity.) +* Resilient: if one or more nodes go down, the other nodes should not be affected and can continue to acquire locks (provided not more than `n/2 - 1` nodes are down). +* Automatically reconnect to (restarted) nodes. +* Compatible with `sync/mutex` API. + + +Restrictions +------------ + +* Limited scalability: up to 16 nodes. +* Fixed configuration: changes in the number and/or network names/IP addresses need a restart of all nodes in order to take effect. +* If a down node comes up, it will not in any way (re)acquire any locks that it may have held. +* Not designed for high performance applications such as key/value stores + +Performance +----------- + +* Lock requests (successful) should not take longer than 1ms (provided decent network connection of 1 Gbit or more between the nodes) +* Support up to 4000 locks per node per second. +* Scale linearly with the number of locks. For the maximum size case of 16 nodes this means a maximum of 64K locks/sec (and 2048K lock request & release messages/sec) +* Do not take more than (overall) 10% CPU usage + +Issues +------ + +* In case the node that has the lock goes down, the lock release will not be broadcast: what do we do? (periodically ping 'back' to requesting node from all nodes that have the lock?) Or detect that the network connection has gone down. +* If one of the nodes that participated in the lock goes down, this is not a problem since (when it comes back online) the node that originally acquired the lock will still have it, and a request for a new lock will fail due to only `n/2` being available. +* If two nodes go down and both participated in the lock then there is a chance that a new lock will acquire locks from `n/2 + 1` nodes and will success, so we would have two concurrent locks. One way to counter this would be to monitor the network connections from the nodes that originated the lock, and, upon losing a connection to a node that granted a lock, get a new lock from a free node. +* When two nodes want to acquire the same lock, it is possible for both to just acquire `n` locks and there is no majority winner so both would fail (and presumably fail back to their clients?). This then requires a retry in order to acquire the lock at a later time. +* What if late acquire response still comes in after lock has been obtained (quorum is in) and has already been released again. + +Comparison to other techniques +------------------------------ + +We are well aware that there are more sophisticated systems such as zookeeper, raft, etc but we found that for our limited use case this was adding too much complexity. So if `dsync` does not meet your requirements than you are probably better off using one of those systems. + +Performance +----------- + +``` +benchmark old ns/op new ns/op delta +BenchmarkMutexUncontended-8 4.22 1164018 +27583264.93% +BenchmarkMutex-8 96.5 1223266 +1267533.16% +BenchmarkMutexSlack-8 120 1192900 +993983.33% +BenchmarkMutexWork-8 108 1239893 +1147949.07% +BenchmarkMutexWorkSlack-8 142 1210129 +852103.52% +BenchmarkMutexNoSpin-8 292 319479 +109310.62% +BenchmarkMutexSpin-8 1163 1270066 +109106.02% +``` + +Usage +----- + +Explain usage +``` +``` + + +License +------- + +Released under the Apache License v2.0. You can find the complete text in the file LICENSE. + +Contributing +------------ + +Contributions are welcome, please send PRs for any enhancements. diff --git a/vendor/github.com/minio/dsync/drwmutex.go b/vendor/github.com/minio/dsync/drwmutex.go new file mode 100644 index 000000000..4fee737bc --- /dev/null +++ b/vendor/github.com/minio/dsync/drwmutex.go @@ -0,0 +1,376 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dsync + +import ( + "log" + "math" + "math/rand" + "net" + "os" + "sync" + "time" +) + +// Indicator if logging is enabled. +var dsyncLog bool + +func init() { + // Check for DSYNC_LOG env variable, if set logging will be enabled for failed RPC operations. + dsyncLog = os.Getenv("DSYNC_LOG") == "1" +} + +// DRWMutexAcquireTimeout - tolerance limit to wait for lock acquisition before. +const DRWMutexAcquireTimeout = 25 * time.Millisecond // 25ms. + +// A DRWMutex is a distributed mutual exclusion lock. +type DRWMutex struct { + Name string + locks []bool // Array of nodes that granted a lock + m sync.Mutex // Mutex to prevent multiple simultaneous locks from this node +} + +type Granted struct { + index int + locked bool +} + +type LockArgs struct { + Token string + Timestamp time.Time + Name string +} + +func (l *LockArgs) SetToken(token string) { + l.Token = token +} + +func (l *LockArgs) SetTimestamp(tstamp time.Time) { + l.Timestamp = tstamp +} + +func NewDRWMutex(name string) *DRWMutex { + return &DRWMutex{ + Name: name, + locks: make([]bool, dnodeCount), + } +} + +// RLock holds a read lock on dm. +// +// If the lock is already in use, the calling goroutine +// blocks until the mutex is available. +func (dm *DRWMutex) RLock() { + // Shield RLock() with local mutex in order to prevent more than + // one broadcast going out at the same time from this node + dm.m.Lock() + defer dm.m.Unlock() + + runs, backOff := 1, 1 + + for { + + // create temp arrays on stack + locks := make([]bool, dnodeCount) + + // try to acquire the lock + isReadLock := true + success := lock(clnts, &locks, dm.Name, isReadLock) + if success { + // if success, copy array to object + copy(dm.locks, locks[:]) + return + } + + // We timed out on the previous lock, incrementally wait for a longer back-off time, + // and try again afterwards + time.Sleep(time.Duration(backOff) * time.Millisecond) + + backOff += int(rand.Float64() * math.Pow(2, float64(runs))) + if backOff > 1024 { + backOff = backOff % 64 + + runs = 1 // reset runs + } else if runs < 10 { + runs++ + } + } +} + +// Lock locks dm. +// +// If the lock is already in use, the calling goroutine +// blocks until the mutex is available. +func (dm *DRWMutex) Lock() { + + // Shield Lock() with local mutex in order to prevent more than + // one broadcast going out at the same time from this node + dm.m.Lock() + defer dm.m.Unlock() + + runs, backOff := 1, 1 + + for { + // create temp arrays on stack + locks := make([]bool, dnodeCount) + + // try to acquire the lock + isReadLock := false + success := lock(clnts, &locks, dm.Name, isReadLock) + if success { + // if success, copy array to object + copy(dm.locks, locks[:]) + return + } + + // We timed out on the previous lock, incrementally wait for a longer back-off time, + // and try again afterwards + time.Sleep(time.Duration(backOff) * time.Millisecond) + + backOff += int(rand.Float64() * math.Pow(2, float64(runs))) + if backOff > 1024 { + backOff = backOff % 64 + + runs = 1 // reset runs + } else if runs < 10 { + runs++ + } + } +} + +// lock tries to acquire the distributed lock, returning true or false +// +func lock(clnts []RPC, locks *[]bool, lockName string, isReadLock bool) bool { + + // Create buffered channel of quorum size + ch := make(chan Granted, dnodeCount) + + for index, c := range clnts { + + // broadcast lock request to all nodes + go func(index int, isReadLock bool, c RPC) { + // All client methods issuing RPCs are thread-safe and goroutine-safe, + // i.e. it is safe to call them from multiple concurrently running go routines. + var locked bool + if isReadLock { + if err := c.Call("Dsync.RLock", &LockArgs{Name: lockName}, &locked); err != nil { + if dsyncLog { + log.Println("Unable to call Dsync.RLock", err) + } + } + } else { + if err := c.Call("Dsync.Lock", &LockArgs{Name: lockName}, &locked); err != nil { + if dsyncLog { + log.Println("Unable to call Dsync.Lock", err) + } + } + } + + ch <- Granted{index: index, locked: locked} + + }(index, isReadLock, c) + } + + quorum := false + + var wg sync.WaitGroup + wg.Add(1) + go func(isReadLock bool) { + + // Wait until we have either a) received all lock responses, b) received too many 'non-'locks for quorum to be or c) time out + i, locksFailed := 0, 0 + done := false + timeout := time.After(DRWMutexAcquireTimeout) + + for ; i < dnodeCount; i++ { // Loop until we acquired all locks + + select { + case grant := <-ch: + if grant.locked { + // Mark that this node has acquired the lock + (*locks)[grant.index] = true + } else { + locksFailed++ + if locksFailed > dnodeCount - dquorum { + // We know that we are not going to get the lock anymore, so exit out + // and release any locks that did get acquired + done = true + releaseAll(clnts, locks, lockName, isReadLock) + } + } + + case <-timeout: + done = true + // timeout happened, maybe one of the nodes is slow, count + // number of locks to check whether we have quorum or not + if !quorumMet(locks) { + releaseAll(clnts, locks, lockName, isReadLock) + } + } + + if done { + break + } + } + + // Count locks in order to determine whterh we have quorum or not + quorum = quorumMet(locks) + + // Signal that we have the quorum + wg.Done() + + // Wait for the other responses and immediately release the locks + // (do not add them to the locks array because the DRWMutex could + // already has been unlocked again by the original calling thread) + for ; i < dnodeCount; i++ { + grantToBeReleased := <-ch + if grantToBeReleased.locked { + // release lock + sendRelease(clnts[grantToBeReleased.index], lockName, isReadLock) + } + } + }(isReadLock) + + wg.Wait() + + return quorum +} + +// quorumMet determines whether we have acquired n/2+1 underlying locks or not +func quorumMet(locks *[]bool) bool { + + count := 0 + for _, locked := range *locks { + if locked { + count++ + } + } + + return count >= dquorum +} + +// releaseAll releases all locks that are marked as locked +func releaseAll(clnts []RPC, locks *[]bool, lockName string, isReadLock bool) { + for lock := 0; lock < dnodeCount; lock++ { + if (*locks)[lock] { + sendRelease(clnts[lock], lockName, isReadLock) + (*locks)[lock] = false + } + } + +} + +// RUnlock releases a read lock held on dm. +// +// It is a run-time error if dm is not locked on entry to RUnlock. +func (dm *DRWMutex) RUnlock() { + // We don't panic like sync.Mutex, when an unlock is issued on an + // un-locked lock, since the lock rpc server may have restarted and + // "forgotten" about the lock. + + // We don't need to wait until we have released all the locks (or the quorum) + // (a subsequent lock will retry automatically in case it would fail to get + // quorum) + for index, c := range clnts { + + if dm.locks[index] { + // broadcast lock release to all nodes the granted the lock + isReadLock := true + sendRelease(c, dm.Name, isReadLock) + + dm.locks[index] = false + } + } +} + +// Unlock unlocks dm. +// +// It is a run-time error if dm is not locked on entry to Unlock. +func (dm *DRWMutex) Unlock() { + + // We don't panic like sync.Mutex, when an unlock is issued on an + // un-locked lock, since the lock rpc server may have restarted and + // "forgotten" about the lock. + + // We don't need to wait until we have released all the locks (or the quorum) + // (a subsequent lock will retry automatically in case it would fail to get + // quorum) + for index, c := range clnts { + + if dm.locks[index] { + // broadcast lock release to all nodes the granted the lock + isReadLock := false + sendRelease(c, dm.Name, isReadLock) + + dm.locks[index] = false + } + } +} + +// sendRelease sends a release message to a node that previously granted a lock +func sendRelease(c RPC, name string, isReadLock bool) { + + backOffArray := []time.Duration{ + 30 * time.Second, // 30secs. + 1 * time.Minute, // 1min. + 3 * time.Minute, // 3min. + 10 * time.Minute, // 10min. + 30 * time.Minute, // 30min. + 1 * time.Hour, // 1hr. + } + + go func(c RPC, name string) { + + for _, backOff := range backOffArray { + + // All client methods issuing RPCs are thread-safe and goroutine-safe, + // i.e. it is safe to call them from multiple concurrently running goroutines. + var unlocked bool + + if isReadLock { + if err := c.Call("Dsync.RUnlock", &LockArgs{Name: name}, &unlocked); err == nil { + // RUnlock delivered, exit out + return + } else if err != nil { + if dsyncLog { + log.Println("Unable to call Dsync.RUnlock", err) + } + if nErr, ok := err.(net.Error); ok && nErr.Timeout() { + // RUnlock possibly failed with server timestamp mismatch, server may have restarted. + return + } + } + } else { + if err := c.Call("Dsync.Unlock", &LockArgs{Name: name}, &unlocked); err == nil { + // Unlock delivered, exit out + return + } else if err != nil { + if dsyncLog { + log.Println("Unable to call Dsync.Unlock", err) + } + if nErr, ok := err.(net.Error); ok && nErr.Timeout() { + // Unlock possibly failed with server timestamp mismatch, server may have restarted. + return + } + } + } + + // Wait.. + time.Sleep(backOff) + } + }(c, name) +} diff --git a/vendor/github.com/minio/dsync/dsync.go b/vendor/github.com/minio/dsync/dsync.go new file mode 100644 index 000000000..8c28abdf4 --- /dev/null +++ b/vendor/github.com/minio/dsync/dsync.go @@ -0,0 +1,55 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dsync + +import "errors" + +const RpcPath = "/dsync" +const DebugPath = "/debug" + +const DefaultPath = "/rpc/dsync" + +// Number of nodes participating in the distributed locking. +var dnodeCount int + +// List of rpc client objects, one per lock server. +var clnts []RPC + +// Simple majority based quorum, set to dNodeCount/2+1 +var dquorum int + +// SetNodesWithPath - initializes package-level global state variables such as clnts. +// N B - This function should be called only once inside any program that uses +// dsync. +func SetNodesWithClients(rpcClnts []RPC) (err error) { + + // Validate if number of nodes is within allowable range. + if dnodeCount != 0 { + return errors.New("Cannot reinitialize dsync package") + } else if len(rpcClnts) < 4 { + return errors.New("Dsync not designed for less than 4 nodes") + } else if len(rpcClnts) > 16 { + return errors.New("Dsync not designed for more than 16 nodes") + } + + dnodeCount = len(rpcClnts) + dquorum = dnodeCount/2 + 1 + // Initialize node name and rpc path for each RPCClient object. + clnts = make([]RPC, dnodeCount) + copy(clnts, rpcClnts) + return nil +} diff --git a/vendor/github.com/minio/dsync/rpc-client-interface.go b/vendor/github.com/minio/dsync/rpc-client-interface.go new file mode 100644 index 000000000..44c0fa5a8 --- /dev/null +++ b/vendor/github.com/minio/dsync/rpc-client-interface.go @@ -0,0 +1,28 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dsync + +import "time" + +// RPC - is dsync compatible client interface. +type RPC interface { + Call(serviceMethod string, args interface { + SetToken(token string) + SetTimestamp(tstamp time.Time) + }, reply interface{}) error + Close() error +} diff --git a/vendor/github.com/tidwall/gjson/LICENSE b/vendor/github.com/tidwall/gjson/LICENSE new file mode 100644 index 000000000..58f5819a4 --- /dev/null +++ b/vendor/github.com/tidwall/gjson/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2016 Josh Baker + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/tidwall/gjson/README.md b/vendor/github.com/tidwall/gjson/README.md new file mode 100644 index 000000000..1ee5ae3dd --- /dev/null +++ b/vendor/github.com/tidwall/gjson/README.md @@ -0,0 +1,278 @@ +

+GJSON +
+Build Status +GoDoc +

+ +

get a json value quickly

+ +GJSON is a Go package the provides a [very fast](#performance) and simple way to get a value from a json document. The reason for this library it to give efficient json indexing for the [BuntDB](https://github.com/tidwall/buntdb) project. + +Getting Started +=============== + +## Installing + +To start using GJSON, install Go and run `go get`: + +```sh +$ go get -u github.com/tidwall/gjson +``` + +This will retrieve the library. + +## Get a value +Get searches json for the specified path. A path is in dot syntax, such as "name.last" or "age". This function expects that the json is well-formed and validates. Invalid json will not panic, but it may return back unexpected results. When the value is found it's returned immediately. + +```go +package main + +import "github.com/tidwall/gjson" + +const json = `{"name":{"first":"Janet","last":"Prichard"},"age":47}` + +func main() { + value := gjson.Get(json, "name.last") + println(value.String()) +} +``` + +This will print: + +``` +Prichard +``` + +## Path Syntax + +A path is a series of keys separated by a dot. +A key may contain special wildcard characters '\*' and '?'. +To access an array value use the index as the key. +To get the number of elements in an array or to access a child path, use the '#' character. +The dot and wildcard characters can be escaped with '\'. + +```json +{ + "name": {"first": "Tom", "last": "Anderson"}, + "age":37, + "children": ["Sara","Alex","Jack"], + "fav.movie": "Deer Hunter", + "friends": [ + {"first": "James", "last": "Murphy"}, + {"first": "Roger", "last": "Craig"} + ] +} +``` +``` +"name.last" >> "Anderson" +"age" >> 37 +"children.#" >> 3 +"children.1" >> "Alex" +"child*.2" >> "Jack" +"c?ildren.0" >> "Sara" +"fav\.movie" >> "Deer Hunter" +"friends.#.first" >> [ "James", "Roger" ] +"friends.1.last" >> "Craig" +``` +To query an array: +``` +`friends.#[last="Murphy"].first` >> "James" +``` + +## Result Type + +GJSON supports the json types `string`, `number`, `bool`, and `null`. +Arrays and Objects are returned as their raw json types. + +The `Result` type holds one of these: + +``` +bool, for JSON booleans +float64, for JSON numbers +string, for JSON string literals +nil, for JSON null +``` + +To directly access the value: + +```go +result.Type // can be String, Number, True, False, Null, or JSON +result.Str // holds the string +result.Num // holds the float64 number +result.Raw // holds the raw json +result.Multi // holds nested array values +``` + +There are a variety of handy functions that work on a result: + +```go +result.Value() interface{} +result.Int() int64 +result.Float() float64 +result.String() string +result.Bool() bool +result.Array() []gjson.Result +result.Map() map[string]gjson.Result +result.Get(path string) Result +``` + +The `result.Value()` function returns an `interface{}` which requires type assertion and is one of the following Go types: + +```go +boolean >> bool +number >> float64 +string >> string +null >> nil +array >> []interface{} +object >> map[string]interface{} +``` + +## Get nested array values + +Suppose you want all the last names from the following json: + +```json +{ + "programmers": [ + { + "firstName": "Janet", + "lastName": "McLaughlin", + }, { + "firstName": "Elliotte", + "lastName": "Hunter", + }, { + "firstName": "Jason", + "lastName": "Harold", + } + ] +}` +``` + +You would use the path "programmers.#.lastName" like such: + +```go +result := gjson.Get(json, "programmers.#.lastName") +for _,name := range result.Array() { + println(name.String()) +} +``` + +You can also query an object inside an array: + +```go +name := gjson.Get(json, `programmers.#[lastName="Hunter"].firstName`) +println(name.String()) // prints "Elliotte" +``` + + +## Simple Parse and Get + +There's a `Parse(json)` function that will do a simple parse, and `result.Get(path)` that will search a result. + +For example, all of these will return the same result: + +```go +gjson.Parse(json).Get("name").Get("last") +gjson.Get(json, "name").Get("last") +gjson.Get(json, "name.last") +``` + +## Check for the existence of a value + +Sometimes you just want to know you if a value exists. + +```go +value := gjson.Get(json, "name.last") +if !value.Exists() { + println("no last name") +} else { + println(value.String()) +} + +// Or as one step +if gjson.Get(json, "name.last").Exists(){ + println("has a last name") +} +``` + +## Unmarshal to a map + +To unmarshal to a `map[string]interface{}`: + +```go +m, ok := gjson.Parse(json).Value().(map[string]interface{}) +if !ok{ + // not a map +} +``` + +## Performance + +Benchmarks of GJSON alongside [encoding/json](https://golang.org/pkg/encoding/json/), +[ffjson](https://github.com/pquerna/ffjson), +[EasyJSON](https://github.com/mailru/easyjson), +and [jsonparser](https://github.com/buger/jsonparser) + +``` +BenchmarkGJSONGet-8 15000000 333 ns/op 0 B/op 0 allocs/op +BenchmarkGJSONUnmarshalMap-8 900000 4188 ns/op 1920 B/op 26 allocs/op +BenchmarkJSONUnmarshalMap-8 600000 8908 ns/op 3048 B/op 69 allocs/op +BenchmarkJSONUnmarshalStruct-8 600000 9026 ns/op 1832 B/op 69 allocs/op +BenchmarkJSONDecoder-8 300000 14339 ns/op 4224 B/op 184 allocs/op +BenchmarkFFJSONLexer-8 1500000 3156 ns/op 896 B/op 8 allocs/op +BenchmarkEasyJSONLexer-8 3000000 938 ns/op 613 B/op 6 allocs/op +BenchmarkJSONParserGet-8 3000000 442 ns/op 21 B/op 0 allocs/op +``` + +JSON document used: + +```json +{ + "widget": { + "debug": "on", + "window": { + "title": "Sample Konfabulator Widget", + "name": "main_window", + "width": 500, + "height": 500 + }, + "image": { + "src": "Images/Sun.png", + "hOffset": 250, + "vOffset": 250, + "alignment": "center" + }, + "text": { + "data": "Click Here", + "size": 36, + "style": "bold", + "vOffset": 100, + "alignment": "center", + "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;" + } + } +} +``` + +Each operation was rotated though one of the following search paths: + +``` +widget.window.name +widget.image.hOffset +widget.text.onMouseUp +``` + + +*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.7.* + +## Contact +Josh Baker [@tidwall](http://twitter.com/tidwall) + +## License + +GJSON source code is available under the MIT [License](/LICENSE). diff --git a/vendor/github.com/tidwall/gjson/gjson.go b/vendor/github.com/tidwall/gjson/gjson.go new file mode 100644 index 000000000..5ad877455 --- /dev/null +++ b/vendor/github.com/tidwall/gjson/gjson.go @@ -0,0 +1,1291 @@ +// Package gjson provides searching for json strings. +package gjson + +import ( + "reflect" + "strconv" + "unsafe" + + "github.com/tidwall/match" +) + +// Type is Result type +type Type int + +const ( + // Null is a null json value + Null Type = iota + // False is a json false boolean + False + // Number is json number + Number + // String is a json string + String + // True is a json true boolean + True + // JSON is a raw block of JSON + JSON +) + +// Result represents a json value that is returned from Get(). +type Result struct { + // Type is the json type + Type Type + // Raw is the raw json + Raw string + // Str is the json string + Str string + // Num is the json number + Num float64 +} + +// String returns a string representation of the value. +func (t Result) String() string { + switch t.Type { + default: + return "null" + case False: + return "false" + case Number: + return strconv.FormatFloat(t.Num, 'f', -1, 64) + case String: + return t.Str + case JSON: + return t.Raw + case True: + return "true" + } +} + +// Bool returns an boolean representation. +func (t Result) Bool() bool { + switch t.Type { + default: + return false + case True: + return true + case String: + return t.Str != "" && t.Str != "0" + case Number: + return t.Num != 0 + } +} + +// Int returns an integer representation. +func (t Result) Int() int64 { + switch t.Type { + default: + return 0 + case True: + return 1 + case String: + n, _ := strconv.ParseInt(t.Str, 10, 64) + return n + case Number: + return int64(t.Num) + } +} + +// Float returns an float64 representation. +func (t Result) Float() float64 { + switch t.Type { + default: + return 0 + case True: + return 1 + case String: + n, _ := strconv.ParseFloat(t.Str, 64) + return n + case Number: + return t.Num + } +} + +// Array returns back an array of children. The result must be a JSON array. +func (t Result) Array() []Result { + if t.Type != JSON { + return nil + } + r := t.arrayOrMap('[', false) + return r.a +} + +// Map returns back an map of children. The result should be a JSON array. +func (t Result) Map() map[string]Result { + if t.Type != JSON { + return map[string]Result{} + } + r := t.arrayOrMap('{', false) + return r.o +} + +// Get searches result for the specified path. +// The result should be a JSON array or object. +func (t Result) Get(path string) Result { + return Get(t.Raw, path) +} + +type arrayOrMapResult struct { + a []Result + ai []interface{} + o map[string]Result + oi map[string]interface{} + vc byte +} + +func (t Result) arrayOrMap(vc byte, valueize bool) (r arrayOrMapResult) { + var json = t.Raw + var i int + var value Result + var count int + var key Result + if vc == 0 { + for ; i < len(json); i++ { + if json[i] == '{' || json[i] == '[' { + r.vc = json[i] + i++ + break + } + if json[i] > ' ' { + goto end + } + } + } else { + for ; i < len(json); i++ { + if json[i] == vc { + i++ + break + } + if json[i] > ' ' { + goto end + } + } + r.vc = vc + } + if r.vc == '{' { + if valueize { + r.oi = make(map[string]interface{}) + } else { + r.o = make(map[string]Result) + } + } else { + if valueize { + r.ai = make([]interface{}, 0) + } else { + r.a = make([]Result, 0) + } + } + for ; i < len(json); i++ { + if json[i] <= ' ' { + continue + } + // get next value + if json[i] == ']' || json[i] == '}' { + break + } + switch json[i] { + default: + if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' { + value.Type = Number + value.Raw, value.Num = tonum(json[i:]) + } else { + continue + } + case '{', '[': + value.Type = JSON + value.Raw = squash(json[i:]) + case 'n': + value.Type = Null + value.Raw = tolit(json[i:]) + case 't': + value.Type = True + value.Raw = tolit(json[i:]) + case 'f': + value.Type = False + value.Raw = tolit(json[i:]) + case '"': + value.Type = String + value.Raw, value.Str = tostr(json[i:]) + } + i += len(value.Raw) - 1 + + if r.vc == '{' { + if count%2 == 0 { + key = value + } else { + if valueize { + r.oi[key.Str] = value.Value() + } else { + r.o[key.Str] = value + } + } + count++ + } else { + if valueize { + r.ai = append(r.ai, value.Value()) + } else { + r.a = append(r.a, value) + } + } + } +end: + return +} + +// Parse parses the json and returns a result +func Parse(json string) Result { + var value Result + for i := 0; i < len(json); i++ { + if json[i] == '{' || json[i] == '[' { + value.Type = JSON + value.Raw = json[i:] // just take the entire raw + break + } + if json[i] <= ' ' { + continue + } + switch json[i] { + default: + if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' { + value.Type = Number + value.Raw, value.Num = tonum(json[i:]) + } else { + return Result{} + } + case 'n': + value.Type = Null + value.Raw = tolit(json[i:]) + case 't': + value.Type = True + value.Raw = tolit(json[i:]) + case 'f': + value.Type = False + value.Raw = tolit(json[i:]) + case '"': + value.Type = String + value.Raw, value.Str = tostr(json[i:]) + } + break + } + return value +} + +func squash(json string) string { + // expects that the lead character is a '[' or '{' + // squash the value, ignoring all nested arrays and objects. + // the first '[' or '{' has already been read + depth := 1 + for i := 1; i < len(json); i++ { + if json[i] >= '"' && json[i] <= '}' { + switch json[i] { + case '"': + i++ + s2 := i + for ; i < len(json); i++ { + if json[i] > '\\' { + continue + } + if json[i] == '"' { + // look for an escaped slash + if json[i-1] == '\\' { + n := 0 + for j := i - 2; j > s2-1; j-- { + if json[j] != '\\' { + break + } + n++ + } + if n%2 == 0 { + continue + } + } + break + } + } + case '{', '[': + depth++ + case '}', ']': + depth-- + if depth == 0 { + return json[:i+1] + } + } + } + } + return json +} + +func tonum(json string) (raw string, num float64) { + for i := 1; i < len(json); i++ { + // less than dash might have valid characters + if json[i] <= '-' { + if json[i] <= ' ' || json[i] == ',' { + // break on whitespace and comma + raw = json[:i] + num, _ = strconv.ParseFloat(raw, 64) + return + } + // could be a '+' or '-'. let's assume so. + continue + } + if json[i] < ']' { + // probably a valid number + continue + } + if json[i] == 'e' || json[i] == 'E' { + // allow for exponential numbers + continue + } + // likely a ']' or '}' + raw = json[:i] + num, _ = strconv.ParseFloat(raw, 64) + return + } + raw = json + num, _ = strconv.ParseFloat(raw, 64) + return +} + +func tolit(json string) (raw string) { + for i := 1; i < len(json); i++ { + if json[i] <= 'a' || json[i] >= 'z' { + return json[:i] + } + } + return json +} + +func tostr(json string) (raw string, str string) { + // expects that the lead character is a '"' + for i := 1; i < len(json); i++ { + if json[i] > '\\' { + continue + } + if json[i] == '"' { + return json[:i+1], json[1:i] + } + if json[i] == '\\' { + i++ + for ; i < len(json); i++ { + if json[i] > '\\' { + continue + } + if json[i] == '"' { + // look for an escaped slash + if json[i-1] == '\\' { + n := 0 + for j := i - 2; j > 0; j-- { + if json[j] != '\\' { + break + } + n++ + } + if n%2 == 0 { + continue + } + } + break + } + } + return json[:i+1], unescape(json[1:i]) + } + } + return json, json[1:] +} + +// Exists returns true if value exists. +// +// if gjson.Get(json, "name.last").Exists(){ +// println("value exists") +// } +func (t Result) Exists() bool { + return t.Type != Null || len(t.Raw) != 0 +} + +// Value returns one of these types: +// +// bool, for JSON booleans +// float64, for JSON numbers +// Number, for JSON numbers +// string, for JSON string literals +// nil, for JSON null +// +func (t Result) Value() interface{} { + if t.Type == String { + return t.Str + } + switch t.Type { + default: + return nil + case False: + return false + case Number: + return t.Num + case JSON: + r := t.arrayOrMap(0, true) + if r.vc == '{' { + return r.oi + } else if r.vc == '[' { + return r.ai + } + return nil + case True: + return true + } +} + +func parseString(json string, i int) (int, string, bool, bool) { + var s = i + for ; i < len(json); i++ { + if json[i] > '\\' { + continue + } + if json[i] == '"' { + return i + 1, json[s-1 : i+1], false, true + } + if json[i] == '\\' { + i++ + for ; i < len(json); i++ { + if json[i] > '\\' { + continue + } + if json[i] == '"' { + // look for an escaped slash + if json[i-1] == '\\' { + n := 0 + for j := i - 2; j > 0; j-- { + if json[j] != '\\' { + break + } + n++ + } + if n%2 == 0 { + continue + } + } + return i + 1, json[s-1 : i+1], true, true + } + } + break + } + } + return i, json[s-1:], false, false +} + +func parseNumber(json string, i int) (int, string) { + var s = i + i++ + for ; i < len(json); i++ { + if json[i] <= ' ' || json[i] == ',' || json[i] == ']' || json[i] == '}' { + return i, json[s:i] + } + } + return i, json[s:] +} + +func parseLiteral(json string, i int) (int, string) { + var s = i + i++ + for ; i < len(json); i++ { + if json[i] < 'a' || json[i] > 'z' { + return i, json[s:i] + } + } + return i, json[s:] +} + +type arrayPathResult struct { + part string + path string + more bool + alogok bool + arrch bool + alogkey string + query struct { + on bool + path string + op string + value string + } +} + +func parseArrayPath(path string) (r arrayPathResult) { + for i := 0; i < len(path); i++ { + if path[i] == '.' { + r.part = path[:i] + r.path = path[i+1:] + r.more = true + return + } + if path[i] == '#' { + r.arrch = true + if i == 0 && len(path) > 1 { + if path[1] == '.' { + r.alogok = true + r.alogkey = path[2:] + r.path = path[:1] + } else if path[1] == '[' { + r.query.on = true + // query + i += 2 + // whitespace + for ; i < len(path); i++ { + if path[i] > ' ' { + break + } + } + s := i + for ; i < len(path); i++ { + if path[i] <= ' ' || path[i] == '=' || + path[i] == '<' || path[i] == '>' || + path[i] == ']' { + break + } + } + r.query.path = path[s:i] + // whitespace + for ; i < len(path); i++ { + if path[i] > ' ' { + break + } + } + if i < len(path) { + s = i + if path[i] == '<' || path[i] == '>' { + if i < len(path)-1 && path[i+1] == '=' { + i++ + } + } else if path[i] == '=' { + if i < len(path)-1 && path[i+1] == '=' { + s++ + i++ + } + } + i++ + r.query.op = path[s:i] + // whitespace + for ; i < len(path); i++ { + if path[i] > ' ' { + break + } + } + s = i + for ; i < len(path); i++ { + if path[i] == '"' { + i++ + s2 := i + for ; i < len(path); i++ { + if path[i] > '\\' { + continue + } + if path[i] == '"' { + // look for an escaped slash + if path[i-1] == '\\' { + n := 0 + for j := i - 2; j > s2-1; j-- { + if path[j] != '\\' { + break + } + n++ + } + if n%2 == 0 { + continue + } + } + break + } + } + } else if path[i] == ']' { + break + } + } + if i > len(path) { + i = len(path) + } + v := path[s:i] + for len(v) > 0 && v[len(v)-1] <= ' ' { + v = v[:len(v)-1] + } + r.query.value = v + } + } + } + continue + } + } + r.part = path + r.path = "" + return +} + +type objectPathResult struct { + part string + path string + wild bool + more bool +} + +func parseObjectPath(path string) (r objectPathResult) { + for i := 0; i < len(path); i++ { + if path[i] == '.' { + r.part = path[:i] + r.path = path[i+1:] + r.more = true + return + } + if path[i] == '*' || path[i] == '?' { + r.wild = true + continue + } + if path[i] == '\\' { + // go into escape mode. this is a slower path that + // strips off the escape character from the part. + epart := []byte(path[:i]) + i++ + if i < len(path) { + epart = append(epart, path[i]) + i++ + for ; i < len(path); i++ { + if path[i] == '\\' { + i++ + if i < len(path) { + epart = append(epart, path[i]) + } + continue + } else if path[i] == '.' { + r.part = string(epart) + r.path = path[i+1:] + r.more = true + return + } else if path[i] == '*' || path[i] == '?' { + r.wild = true + } + epart = append(epart, path[i]) + } + } + // append the last part + r.part = string(epart) + return + } + } + r.part = path + return +} + +func parseSquash(json string, i int) (int, string) { + // expects that the lead character is a '[' or '{' + // squash the value, ignoring all nested arrays and objects. + // the first '[' or '{' has already been read + s := i + i++ + depth := 1 + for ; i < len(json); i++ { + if json[i] >= '"' && json[i] <= '}' { + switch json[i] { + case '"': + i++ + s2 := i + for ; i < len(json); i++ { + if json[i] > '\\' { + continue + } + if json[i] == '"' { + // look for an escaped slash + if json[i-1] == '\\' { + n := 0 + for j := i - 2; j > s2-1; j-- { + if json[j] != '\\' { + break + } + n++ + } + if n%2 == 0 { + continue + } + } + break + } + } + case '{', '[': + depth++ + case '}', ']': + depth-- + if depth == 0 { + i++ + return i, json[s:i] + } + } + } + } + return i, json[s:] +} + +func parseObject(c *parseContext, i int, path string) (int, bool) { + var pmatch, kesc, vesc, ok, hit bool + var key, val string + rp := parseObjectPath(path) + for i < len(c.json) { + for ; i < len(c.json); i++ { + if c.json[i] == '"' { + // parse_key_string + // this is slightly different from getting s string value + // because we don't need the outer quotes. + i++ + var s = i + for ; i < len(c.json); i++ { + if c.json[i] > '\\' { + continue + } + if c.json[i] == '"' { + i, key, kesc, ok = i+1, c.json[s:i], false, true + goto parse_key_string_done + } + if c.json[i] == '\\' { + i++ + for ; i < len(c.json); i++ { + if c.json[i] > '\\' { + continue + } + if c.json[i] == '"' { + // look for an escaped slash + if c.json[i-1] == '\\' { + n := 0 + for j := i - 2; j > 0; j-- { + if c.json[j] != '\\' { + break + } + n++ + } + if n%2 == 0 { + continue + } + } + i, key, kesc, ok = i+1, c.json[s:i], true, true + goto parse_key_string_done + } + } + break + } + } + i, key, kesc, ok = i, c.json[s:], false, false + parse_key_string_done: + break + } + if c.json[i] == '}' { + return i + 1, false + } + } + if !ok { + return i, false + } + if rp.wild { + if kesc { + pmatch = match.Match(unescape(key), rp.part) + } else { + pmatch = match.Match(key, rp.part) + } + } else { + if kesc { + pmatch = rp.part == unescape(key) + } else { + pmatch = rp.part == key + } + } + hit = pmatch && !rp.more + for ; i < len(c.json); i++ { + switch c.json[i] { + default: + continue + case '"': + i++ + i, val, vesc, ok = parseString(c.json, i) + if !ok { + return i, false + } + if hit { + if vesc { + c.value.Str = unescape(val[1 : len(val)-1]) + } else { + c.value.Str = val[1 : len(val)-1] + } + c.value.Raw = val + c.value.Type = String + return i, true + } + case '{': + if pmatch && !hit { + i, hit = parseObject(c, i+1, rp.path) + if hit { + return i, true + } + } else { + i, val = parseSquash(c.json, i) + if hit { + c.value.Raw = val + c.value.Type = JSON + return i, true + } + } + case '[': + if pmatch && !hit { + i, hit = parseArray(c, i+1, rp.path) + if hit { + return i, true + } + } else { + i, val = parseSquash(c.json, i) + if hit { + c.value.Raw = val + c.value.Type = JSON + return i, true + } + } + case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + i, val = parseNumber(c.json, i) + if hit { + c.value.Raw = val + c.value.Type = Number + c.value.Num, _ = strconv.ParseFloat(val, 64) + return i, true + } + case 't', 'f', 'n': + vc := c.json[i] + i, val = parseLiteral(c.json, i) + if hit { + c.value.Raw = val + switch vc { + case 't': + c.value.Type = True + case 'f': + c.value.Type = False + } + return i, true + } + } + break + } + } + return i, false +} +func queryMatches(rp *arrayPathResult, value Result) bool { + rpv := rp.query.value + if len(rpv) > 2 && rpv[0] == '"' && rpv[len(rpv)-1] == '"' { + rpv = rpv[1 : len(rpv)-1] + } + switch value.Type { + case String: + switch rp.query.op { + case "=": + return value.Str == rpv + case "<": + return value.Str < rpv + case "<=": + return value.Str <= rpv + case ">": + return value.Str > rpv + case ">=": + return value.Str >= rpv + } + case Number: + rpvn, _ := strconv.ParseFloat(rpv, 64) + switch rp.query.op { + case "=": + return value.Num == rpvn + case "<": + return value.Num < rpvn + case "<=": + return value.Num <= rpvn + case ">": + return value.Num > rpvn + case ">=": + return value.Num >= rpvn + } + case True: + switch rp.query.op { + case "=": + return rpv == "true" + case ">": + return rpv == "false" + case ">=": + return true + } + case False: + switch rp.query.op { + case "=": + return rpv == "false" + case "<": + return rpv == "true" + case "<=": + return true + } + } + return false +} +func parseArray(c *parseContext, i int, path string) (int, bool) { + var pmatch, vesc, ok, hit bool + var val string + var h int + var alog []int + var partidx int + rp := parseArrayPath(path) + if !rp.arrch { + n, err := strconv.ParseUint(rp.part, 10, 64) + if err != nil { + partidx = -1 + } else { + partidx = int(n) + } + } + for i < len(c.json) { + if !rp.arrch { + pmatch = partidx == h + hit = pmatch && !rp.more + } + h++ + if rp.alogok { + alog = append(alog, i) + } + for ; i < len(c.json); i++ { + switch c.json[i] { + default: + continue + case '"': + i++ + i, val, vesc, ok = parseString(c.json, i) + if !ok { + return i, false + } + if hit { + if rp.alogok { + break + } + if vesc { + c.value.Str = unescape(val[1 : len(val)-1]) + } else { + c.value.Str = val[1 : len(val)-1] + } + c.value.Raw = val + c.value.Type = String + return i, true + } + case '{': + if pmatch && !hit { + i, hit = parseObject(c, i+1, rp.path) + if hit { + if rp.alogok { + break + } + return i, true + } + } else { + i, val = parseSquash(c.json, i) + if rp.query.on { + res := Get(val, rp.query.path) + if queryMatches(&rp, res) { + if rp.more { + c.value = Get(val, rp.path) + } else { + c.value.Raw = val + c.value.Type = JSON + } + return i, true + } + } else if hit { + if rp.alogok { + break + } + c.value.Raw = val + c.value.Type = JSON + return i, true + } + } + case '[': + if pmatch && !hit { + i, hit = parseArray(c, i+1, rp.path) + if hit { + if rp.alogok { + break + } + return i, true + } + } else { + i, val = parseSquash(c.json, i) + if hit { + if rp.alogok { + break + } + c.value.Raw = val + c.value.Type = JSON + return i, true + } + } + case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + i, val = parseNumber(c.json, i) + if hit { + if rp.alogok { + break + } + c.value.Raw = val + c.value.Type = Number + c.value.Num, _ = strconv.ParseFloat(val, 64) + return i, true + } + case 't', 'f', 'n': + vc := c.json[i] + i, val = parseLiteral(c.json, i) + if hit { + if rp.alogok { + break + } + c.value.Raw = val + switch vc { + case 't': + c.value.Type = True + case 'f': + c.value.Type = False + } + return i, true + } + case ']': + if rp.arrch && rp.part == "#" { + if rp.alogok { + var jsons = make([]byte, 0, 64) + jsons = append(jsons, '[') + for j := 0; j < len(alog); j++ { + res := Get(c.json[alog[j]:], rp.alogkey) + if res.Exists() { + if j > 0 { + jsons = append(jsons, ',') + } + jsons = append(jsons, []byte(res.Raw)...) + } + } + jsons = append(jsons, ']') + c.value.Type = JSON + c.value.Raw = string(jsons) + return i + 1, true + } else { + if rp.alogok { + break + } + c.value.Raw = val + c.value.Type = Number + c.value.Num = float64(h - 1) + return i + 1, true + } + } + return i + 1, false + } + break + } + } + return i, false +} + +type parseContext struct { + json string + value Result +} + +// Get searches json for the specified path. +// A path is in dot syntax, such as "name.last" or "age". +// This function expects that the json is well-formed, and does not validate. +// Invalid json will not panic, but it may return back unexpected results. +// When the value is found it's returned immediately. +// +// A path is a series of keys seperated by a dot. +// A key may contain special wildcard characters '*' and '?'. +// To access an array value use the index as the key. +// To get the number of elements in an array or to access a child path, use the '#' character. +// The dot and wildcard character can be escaped with '\'. +// +// { +// "name": {"first": "Tom", "last": "Anderson"}, +// "age":37, +// "children": ["Sara","Alex","Jack"], +// "friends": [ +// {"first": "James", "last": "Murphy"}, +// {"first": "Roger", "last": "Craig"} +// ] +// } +// "name.last" >> "Anderson" +// "age" >> 37 +// "children.#" >> 3 +// "children.1" >> "Alex" +// "child*.2" >> "Jack" +// "c?ildren.0" >> "Sara" +// "friends.#.first" >> [ "James", "Roger" ] +// +func Get(json, path string) Result { + var i int + var c = &parseContext{json: json} + for ; i < len(c.json); i++ { + if c.json[i] == '{' { + i++ + parseObject(c, i, path) + break + } + if c.json[i] == '[' { + i++ + parseArray(c, i, path) + break + } + } + return c.value +} + +// GetBytes searches json for the specified path. +// If working with bytes, this method preferred over Get(string(data), path) +func GetBytes(json []byte, path string) Result { + var result Result + if json != nil { + // unsafe cast to string + result = Get(*(*string)(unsafe.Pointer(&json)), path) + // copy of string data for safety. + rawh := *(*reflect.SliceHeader)(unsafe.Pointer(&result.Raw)) + strh := *(*reflect.SliceHeader)(unsafe.Pointer(&result.Str)) + if strh.Data == 0 { + if rawh.Data == 0 { + result.Raw = "" + } else { + result.Raw = string(*(*[]byte)(unsafe.Pointer(&result.Raw))) + } + result.Str = "" + } else if rawh.Data == 0 { + result.Raw = "" + result.Str = string(*(*[]byte)(unsafe.Pointer(&result.Str))) + } else if strh.Data >= rawh.Data && + int(strh.Data)+strh.Len <= int(rawh.Data)+rawh.Len { + // Str is a substring of Raw. + start := int(strh.Data - rawh.Data) + result.Raw = string(*(*[]byte)(unsafe.Pointer(&result.Raw))) + result.Str = result.Raw[start : start+strh.Len] + } else { + result.Raw = string(*(*[]byte)(unsafe.Pointer(&result.Raw))) + result.Str = string(*(*[]byte)(unsafe.Pointer(&result.Str))) + } + } + return result +} + +// unescape unescapes a string +func unescape(json string) string { //, error) { + var str = make([]byte, 0, len(json)) + for i := 0; i < len(json); i++ { + switch { + default: + str = append(str, json[i]) + case json[i] < ' ': + return "" //, errors.New("invalid character in string") + case json[i] == '\\': + i++ + if i >= len(json) { + return "" //, errors.New("invalid escape sequence") + } + switch json[i] { + default: + return "" //, errors.New("invalid escape sequence") + case '\\': + str = append(str, '\\') + case '/': + str = append(str, '/') + case 'b': + str = append(str, '\b') + case 'f': + str = append(str, '\f') + case 'n': + str = append(str, '\n') + case 'r': + str = append(str, '\r') + case 't': + str = append(str, '\t') + case '"': + str = append(str, '"') + case 'u': + if i+5 > len(json) { + return "" //, errors.New("invalid escape sequence") + } + i++ + // extract the codepoint + var code int + for j := i; j < i+4; j++ { + switch { + default: + return "" //, errors.New("invalid escape sequence") + case json[j] >= '0' && json[j] <= '9': + code += (int(json[j]) - '0') << uint(12-(j-i)*4) + case json[j] >= 'a' && json[j] <= 'f': + code += (int(json[j]) - 'a' + 10) << uint(12-(j-i)*4) + case json[j] >= 'a' && json[j] <= 'f': + code += (int(json[j]) - 'a' + 10) << uint(12-(j-i)*4) + } + } + str = append(str, []byte(string(code))...) + i += 3 // only 3 because we will increment on the for-loop + } + } + } + return string(str) //, nil +} + +// Less return true if a token is less than another token. +// The caseSensitive paramater is used when the tokens are Strings. +// The order when comparing two different type is: +// +// Null < False < Number < String < True < JSON +// +func (t Result) Less(token Result, caseSensitive bool) bool { + if t.Type < token.Type { + return true + } + if t.Type > token.Type { + return false + } + if t.Type == String { + if caseSensitive { + return t.Str < token.Str + } + return stringLessInsensitive(t.Str, token.Str) + } + if t.Type == Number { + return t.Num < token.Num + } + return t.Raw < token.Raw +} + +func stringLessInsensitive(a, b string) bool { + for i := 0; i < len(a) && i < len(b); i++ { + if a[i] >= 'A' && a[i] <= 'Z' { + if b[i] >= 'A' && b[i] <= 'Z' { + // both are uppercase, do nothing + if a[i] < b[i] { + return true + } else if a[i] > b[i] { + return false + } + } else { + // a is uppercase, convert a to lowercase + if a[i]+32 < b[i] { + return true + } else if a[i]+32 > b[i] { + return false + } + } + } else if b[i] >= 'A' && b[i] <= 'Z' { + // b is uppercase, convert b to lowercase + if a[i] < b[i]+32 { + return true + } else if a[i] > b[i]+32 { + return false + } + } else { + // neither are uppercase + if a[i] < b[i] { + return true + } else if a[i] > b[i] { + return false + } + } + } + return len(a) < len(b) +} diff --git a/vendor/github.com/tidwall/gjson/logo.png b/vendor/github.com/tidwall/gjson/logo.png new file mode 100644 index 000000000..17a8bbe9d Binary files /dev/null and b/vendor/github.com/tidwall/gjson/logo.png differ diff --git a/vendor/github.com/tidwall/match/LICENSE b/vendor/github.com/tidwall/match/LICENSE new file mode 100644 index 000000000..58f5819a4 --- /dev/null +++ b/vendor/github.com/tidwall/match/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2016 Josh Baker + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/tidwall/match/README.md b/vendor/github.com/tidwall/match/README.md new file mode 100644 index 000000000..04b0aaa9d --- /dev/null +++ b/vendor/github.com/tidwall/match/README.md @@ -0,0 +1,31 @@ +Match +===== +Build Status +GoDoc + +Match is a very simple pattern matcher where '*' matches on any +number characters and '?' matches on any one character. +Installing +---------- + +``` +go get -u github.com/tidwall/match +``` + +Example +------- + +```go +match.Match("hello", "*llo") +match.Match("jello", "?ello") +match.Match("hello", "h*o") +``` + + +Contact +------- +Josh Baker [@tidwall](http://twitter.com/tidwall) + +License +------- +Redcon source code is available under the MIT [License](/LICENSE). diff --git a/vendor/github.com/tidwall/match/match.go b/vendor/github.com/tidwall/match/match.go new file mode 100644 index 000000000..8885add63 --- /dev/null +++ b/vendor/github.com/tidwall/match/match.go @@ -0,0 +1,192 @@ +// Match provides a simple pattern matcher with unicode support. +package match + +import "unicode/utf8" + +// Match returns true if str matches pattern. This is a very +// simple wildcard match where '*' matches on any number characters +// and '?' matches on any one character. + +// pattern: +// { term } +// term: +// '*' matches any sequence of non-Separator characters +// '?' matches any single non-Separator character +// c matches character c (c != '*', '?', '\\') +// '\\' c matches character c +// +func Match(str, pattern string) bool { + if pattern == "*" { + return true + } + return deepMatch(str, pattern) +} +func deepMatch(str, pattern string) bool { + for len(pattern) > 0 { + if pattern[0] > 0x7f { + return deepMatchRune(str, pattern) + } + switch pattern[0] { + default: + if len(str) == 0 { + return false + } + if str[0] > 0x7f { + return deepMatchRune(str, pattern) + } + if str[0] != pattern[0] { + return false + } + case '?': + if len(str) == 0 { + return false + } + case '*': + return deepMatch(str, pattern[1:]) || + (len(str) > 0 && deepMatch(str[1:], pattern)) + } + str = str[1:] + pattern = pattern[1:] + } + return len(str) == 0 && len(pattern) == 0 +} + +func deepMatchRune(str, pattern string) bool { + var sr, pr rune + var srsz, prsz int + + // read the first rune ahead of time + if len(str) > 0 { + if str[0] > 0x7f { + sr, srsz = utf8.DecodeRuneInString(str) + } else { + sr, srsz = rune(str[0]), 1 + } + } else { + sr, srsz = utf8.RuneError, 0 + } + if len(pattern) > 0 { + if pattern[0] > 0x7f { + pr, prsz = utf8.DecodeRuneInString(pattern) + } else { + pr, prsz = rune(pattern[0]), 1 + } + } else { + pr, prsz = utf8.RuneError, 0 + } + // done reading + for pr != utf8.RuneError { + switch pr { + default: + if srsz == utf8.RuneError { + return false + } + if sr != pr { + return false + } + case '?': + if srsz == utf8.RuneError { + return false + } + case '*': + return deepMatchRune(str, pattern[prsz:]) || + (srsz > 0 && deepMatchRune(str[srsz:], pattern)) + } + str = str[srsz:] + pattern = pattern[prsz:] + // read the next runes + if len(str) > 0 { + if str[0] > 0x7f { + sr, srsz = utf8.DecodeRuneInString(str) + } else { + sr, srsz = rune(str[0]), 1 + } + } else { + sr, srsz = utf8.RuneError, 0 + } + if len(pattern) > 0 { + if pattern[0] > 0x7f { + pr, prsz = utf8.DecodeRuneInString(pattern) + } else { + pr, prsz = rune(pattern[0]), 1 + } + } else { + pr, prsz = utf8.RuneError, 0 + } + // done reading + } + + return srsz == 0 && prsz == 0 +} + +var maxRuneBytes = func() []byte { + b := make([]byte, 4) + if utf8.EncodeRune(b, '\U0010FFFF') != 4 { + panic("invalid rune encoding") + } + return b +}() + +// Allowable parses the pattern and determines the minimum and maximum allowable +// values that the pattern can represent. +// When the max cannot be determined, 'true' will be returned +// for infinite. +func Allowable(pattern string) (min, max string) { + if pattern == "" || pattern[0] == '*' { + return "", "" + } + + minb := make([]byte, 0, len(pattern)) + maxb := make([]byte, 0, len(pattern)) + var wild bool + for i := 0; i < len(pattern); i++ { + if pattern[i] == '*' { + wild = true + break + } + if pattern[i] == '?' { + minb = append(minb, 0) + maxb = append(maxb, maxRuneBytes...) + } else { + minb = append(minb, pattern[i]) + maxb = append(maxb, pattern[i]) + } + } + if wild { + r, n := utf8.DecodeLastRune(maxb) + if r != utf8.RuneError { + if r < utf8.MaxRune { + r++ + if r > 0x7f { + b := make([]byte, 4) + nn := utf8.EncodeRune(b, r) + maxb = append(maxb[:len(maxb)-n], b[:nn]...) + } else { + maxb = append(maxb[:len(maxb)-n], byte(r)) + } + } + } + } + return string(minb), string(maxb) + /* + return + if wild { + r, n := utf8.DecodeLastRune(maxb) + if r != utf8.RuneError { + if r < utf8.MaxRune { + infinite = true + } else { + r++ + if r > 0x7f { + b := make([]byte, 4) + nn := utf8.EncodeRune(b, r) + maxb = append(maxb[:len(maxb)-n], b[:nn]...) + } else { + maxb = append(maxb[:len(maxb)-n], byte(r)) + } + } + } + } + return string(minb), string(maxb), infinite + */ +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 803181ae5..0d9b10232 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -97,6 +97,12 @@ "revision": "c4a07c7b68db77ccd119183fb1d01dd5972434ab", "revisionTime": "2015-11-18T20:00:48-08:00" }, + { + "checksumSHA1": "0raNaLP/AhxXhEeB5CdSnbED3O4=", + "path": "github.com/minio/dsync", + "revision": "1f615ccd013d35489becfe710e0ba7dce98b59e5", + "revisionTime": "2016-08-29T17:06:27Z" + }, { "path": "github.com/minio/go-homedir", "revision": "0b1069c753c94b3633cc06a1995252dbcc27c7a6", @@ -163,6 +169,18 @@ "revision": "2e25825abdbd7752ff08b270d313b93519a0a232", "revisionTime": "2016-03-11T21:55:03Z" }, + { + "checksumSHA1": "+Pcohsuq0Mi/y8bgaDFjb/CGzkk=", + "path": "github.com/tidwall/gjson", + "revision": "7c631e98686a791e5fc60ff099512968122afb52", + "revisionTime": "2016-09-08T16:02:40Z" + }, + { + "checksumSHA1": "qmePMXEDYGwkAfT9QvtMC58JN/E=", + "path": "github.com/tidwall/match", + "revision": "173748da739a410c5b0b813b956f89ff94730b4c", + "revisionTime": "2016-08-30T17:39:30Z" + }, { "path": "golang.org/x/crypto/bcrypt", "revision": "7b85b097bf7527677d54d3220065e966a0e3b613",