From f8a3fd0c2a9818400004181f2c52997eb6ace79c Mon Sep 17 00:00:00 2001 From: kannappanr <30541348+kannappanr@users.noreply.github.com> Date: Thu, 5 Apr 2018 15:04:40 -0700 Subject: [PATCH] Create logger package and rename errorIf to LogIf (#5678) Removing message from error logging Replace errors.Trace with LogIf --- cmd/admin-handlers.go | 50 ++--- cmd/admin-handlers_test.go | 7 +- cmd/admin-heal-ops.go | 8 +- cmd/admin-rpc-client.go | 21 ++- cmd/admin-rpc-server.go | 14 +- cmd/api-errors.go | 2 +- cmd/api-router.go | 3 +- cmd/auth-handler.go | 12 +- cmd/auth-rpc-client.go | 7 +- cmd/browser-peer-rpc.go | 4 +- cmd/browser-rpc-router.go | 8 +- cmd/bucket-handlers-listobjects.go | 4 +- cmd/bucket-handlers.go | 54 +++--- cmd/bucket-notification-handlers.go | 29 +-- cmd/bucket-policy-handlers.go | 17 +- cmd/bucket-policy.go | 25 +-- cmd/common-main.go | 39 ++-- cmd/config-migrate.go | 45 ++--- cmd/disk-cache-fs.go | 91 +++++---- cmd/disk-cache.go | 13 +- cmd/endpoint.go | 3 +- cmd/erasure-createfile.go | 22 ++- cmd/erasure-createfile_test.go | 11 +- cmd/erasure-healfile.go | 16 +- cmd/erasure-healfile_test.go | 7 +- cmd/erasure-readfile.go | 29 +-- cmd/erasure-readfile_test.go | 21 ++- cmd/erasure-utils.go | 20 +- cmd/erasure.go | 23 ++- cmd/erasure_test.go | 9 +- cmd/format-disk-cache.go | 36 ++-- cmd/format-disk-cache_test.go | 10 +- cmd/format-fs.go | 38 ++-- cmd/format-fs_test.go | 7 +- cmd/format-xl.go | 9 +- cmd/fs-v1-helpers.go | 193 ++++++++++++-------- cmd/fs-v1-helpers_test.go | 79 ++++---- cmd/fs-v1-metadata.go | 18 +- cmd/fs-v1-metadata_test.go | 4 +- cmd/fs-v1-multipart.go | 197 +++++++++++--------- cmd/fs-v1-multipart_test.go | 2 +- cmd/fs-v1-rwpool.go | 6 +- cmd/fs-v1.go | 231 +++++++++++++---------- cmd/fs-v1_test.go | 2 +- cmd/gateway-common.go | 25 +-- cmd/gateway-main.go | 35 ++-- cmd/gateway-startup-msg.go | 12 +- cmd/gateway-unsupported.go | 62 ++++--- cmd/gateway/azure/gateway-azure.go | 184 ++++++++++++------- cmd/gateway/azure/gateway-azure_test.go | 46 ++--- cmd/gateway/b2/gateway-b2.go | 104 ++++++----- cmd/gateway/b2/gateway-b2_test.go | 35 ++-- cmd/gateway/gcs/gateway-gcs.go | 177 ++++++++++-------- cmd/gateway/gcs/gateway-gcs_test.go | 53 +++--- cmd/gateway/manta/gateway-manta.go | 38 ++-- cmd/gateway/oss/gateway-oss.go | 192 +++++++++++-------- cmd/gateway/oss/gateway-oss_test.go | 9 +- cmd/gateway/s3/gateway-s3.go | 76 +++++--- cmd/gateway/s3/gateway-s3_test.go | 4 +- cmd/gateway/sia/gateway-sia.go | 91 +++++---- cmd/handler-utils.go | 39 ++-- cmd/handler-utils_test.go | 5 +- cmd/http/listener.go | 66 ++++--- cmd/http/listener_test.go | 27 +-- cmd/http/server.go | 21 +-- cmd/jwt.go | 6 +- cmd/lock-instrument.go | 39 ++-- cmd/lock-rpc-server-common.go | 8 +- cmd/lock-rpc-server.go | 6 +- cmd/logger/log.go | 64 +++++-- cmd/{ => logger}/logger.go | 233 ++++++++++++++++++------ cmd/namespace-lock.go | 38 ++-- cmd/net.go | 12 +- cmd/notification.go | 72 +++++--- cmd/object-api-common.go | 33 ++-- cmd/object-api-input-checks.go | 113 ++++++++---- cmd/object-api-multipart_test.go | 2 +- cmd/object-api-utils.go | 8 +- cmd/object-api-utils_test.go | 3 +- cmd/object-handlers.go | 57 +++--- cmd/object-handlers_test.go | 2 +- cmd/peer-rpc.go | 21 ++- cmd/posix-list-dir-others.go | 7 +- cmd/posix.go | 6 +- cmd/prepare-storage.go | 12 +- cmd/rpc-server.go | 6 +- cmd/server-main.go | 43 ++--- cmd/server-startup-msg.go | 41 +++-- cmd/server_test.go | 3 +- cmd/signals.go | 19 +- cmd/storage-class_test.go | 14 +- cmd/storage-rpc-server.go | 9 +- cmd/test-utils_test.go | 12 +- cmd/tree-walk.go | 17 +- cmd/tree-walk_test.go | 25 +-- cmd/update-main.go | 22 ++- cmd/utils.go | 8 +- cmd/web-handlers.go | 11 +- cmd/web-handlers_test.go | 4 +- cmd/xl-sets.go | 50 ++--- cmd/xl-sets_test.go | 8 +- cmd/xl-v1-bucket.go | 43 +++-- cmd/xl-v1-common.go | 10 +- cmd/xl-v1-common_test.go | 2 +- cmd/xl-v1-healing-common.go | 8 +- cmd/xl-v1-healing-common_test.go | 19 +- cmd/xl-v1-healing.go | 46 +++-- cmd/xl-v1-list-objects.go | 16 +- cmd/xl-v1-metadata.go | 70 ++++--- cmd/xl-v1-metadata_test.go | 16 +- cmd/xl-v1-multipart.go | 172 +++++++++-------- cmd/xl-v1-multipart_test.go | 2 +- cmd/xl-v1-object.go | 167 +++++++++-------- cmd/xl-v1-object_test.go | 8 +- cmd/xl-v1-utils.go | 88 ++++----- cmd/xl-v1-utils_test.go | 13 +- cmd/xl-v1.go | 3 +- docs/shared-backend/DESIGN.md | 2 +- docs/zh_CN/shared-backend/DESIGN.md | 2 +- 119 files changed, 2608 insertions(+), 1860 deletions(-) rename cmd/{ => logger}/logger.go (50%) diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index 914280ebf..fb0641077 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -18,7 +18,9 @@ package cmd import ( "bytes" + "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -27,6 +29,7 @@ import ( "time" "github.com/gorilla/mux" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" "github.com/minio/minio/pkg/handlers" "github.com/minio/minio/pkg/madmin" @@ -71,7 +74,7 @@ func (a adminAPIHandlers) VersionHandler(w http.ResponseWriter, r *http.Request) jsonBytes, err := json.Marshal(adminAPIVersionInfo) if err != nil { writeErrorResponseJSON(w, ErrInternalError, r.URL) - errorIf(err, "Failed to marshal Admin API Version to JSON.") + logger.LogIf(context.Background(), err) return } @@ -99,7 +102,7 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re uptime, err := getPeerUptimes(globalAdminPeers) if err != nil { writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL) - errorIf(err, "Possibly failed to get uptime from majority of servers.") + logger.LogIf(context.Background(), err) return } @@ -113,7 +116,7 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re jsonBytes, err := json.Marshal(serverStatus) if err != nil { writeErrorResponseJSON(w, ErrInternalError, r.URL) - errorIf(err, "Failed to marshal storage info into json.") + logger.LogIf(context.Background(), err) return } // Reply with storage information (across nodes in a @@ -136,7 +139,7 @@ func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *h var sa madmin.ServiceAction err := json.NewDecoder(r.Body).Decode(&sa) if err != nil { - errorIf(err, "Error parsing body JSON") + logger.LogIf(context.Background(), err) writeErrorResponseJSON(w, ErrRequestBodyParse, r.URL) return } @@ -149,7 +152,7 @@ func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *h serviceSig = serviceStop default: writeErrorResponseJSON(w, ErrMalformedPOSTRequest, r.URL) - errorIf(err, "Invalid service action received") + logger.LogIf(context.Background(), errors.New("Invalid service action received")) return } @@ -243,7 +246,9 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque serverInfoData, err := peer.cmdRunner.ServerInfoData() if err != nil { - errorIf(err, "Unable to get server info from %s.", peer.addr) + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", peer.addr) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) reply[idx].Error = err.Error() return } @@ -258,7 +263,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque jsonBytes, err := json.Marshal(reply) if err != nil { writeErrorResponseJSON(w, ErrInternalError, r.URL) - errorIf(err, "Failed to marshal storage info into json.") + logger.LogIf(context.Background(), err) return } @@ -292,7 +297,7 @@ func validateLockQueryParams(vars url.Values) (string, string, time.Duration, } duration, err := time.ParseDuration(olderThanStr) if err != nil { - errorIf(err, "Failed to parse duration passed as query value.") + logger.LogIf(context.Background(), err) return "", "", time.Duration(0), ErrInvalidDuration } @@ -325,7 +330,7 @@ func (a adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http.Reques duration) if err != nil { writeErrorResponseJSON(w, ErrInternalError, r.URL) - errorIf(err, "Failed to fetch lock information from remote nodes.") + logger.LogIf(context.Background(), err) return } @@ -333,7 +338,7 @@ func (a adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http.Reques jsonBytes, err := json.Marshal(volLocks) if err != nil { writeErrorResponseJSON(w, ErrInternalError, r.URL) - errorIf(err, "Failed to marshal lock information into json.") + logger.LogIf(context.Background(), err) return } @@ -369,7 +374,7 @@ func (a adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *http.Reque duration) if err != nil { writeErrorResponseJSON(w, ErrInternalError, r.URL) - errorIf(err, "Failed to fetch lock information from remote nodes.") + logger.LogIf(ctx, err) return } @@ -377,7 +382,7 @@ func (a adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *http.Reque jsonBytes, err := json.Marshal(volLocks) if err != nil { writeErrorResponseJSON(w, ErrInternalError, r.URL) - errorIf(err, "Failed to marshal lock information into json.") + logger.LogIf(ctx, err) return } @@ -425,7 +430,7 @@ func extractHealInitParams(r *http.Request) (bucket, objPrefix string, if clientToken == "" { jerr := json.NewDecoder(r.Body).Decode(&hs) if jerr != nil { - errorIf(jerr, "Error parsing body JSON") + logger.LogIf(context.Background(), jerr) err = ErrRequestBodyParse return } @@ -583,7 +588,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques // occurring on a quorum of the servers is returned. configBytes, err := getPeerConfig(globalAdminPeers) if err != nil { - errorIf(err, "Failed to get config from peers") + logger.LogIf(context.Background(), err) writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL) return } @@ -655,6 +660,7 @@ func writeSetConfigResponse(w http.ResponseWriter, peers adminPeers, // SetConfigHandler - PUT /minio/admin/v1/config func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) { + ctx := context.Background() // Get current object layer instance. objectAPI := newObjectLayerFn() if objectAPI == nil { @@ -678,7 +684,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques return } if err != io.ErrUnexpectedEOF { - errorIf(err, "Failed to read config from request body.") + logger.LogIf(ctx, err) writeErrorResponse(w, toAPIErrorCode(err), r.URL) return } @@ -688,7 +694,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques // Validate JSON provided in the request body: check the // client has not sent JSON objects with duplicate keys. if err = checkDupJSONKeys(string(configBytes)); err != nil { - errorIf(err, "config contains duplicate JSON entries.") + logger.LogIf(ctx, err) writeErrorResponse(w, ErrAdminConfigBadJSON, r.URL) return } @@ -696,7 +702,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques var config serverConfig err = json.Unmarshal(configBytes, &config) if err != nil { - errorIf(err, "Failed to unmarshal JSON configuration", err) + logger.LogIf(ctx, err) writeErrorResponse(w, toAPIErrorCode(err), r.URL) return } @@ -718,7 +724,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques errs := writeTmpConfigPeers(globalAdminPeers, tmpFileName, configBytes) // Check if the operation succeeded in quorum or more nodes. - rErr := reduceWriteQuorumErrs(errs, nil, len(globalAdminPeers)/2+1) + rErr := reduceWriteQuorumErrs(ctx, errs, nil, len(globalAdminPeers)/2+1) if rErr != nil { writeSetConfigResponse(w, globalAdminPeers, errs, false, r.URL) return @@ -736,7 +742,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques // Rename the temporary config file to config.json errs = commitConfigPeers(globalAdminPeers, tmpFileName) - rErr = reduceWriteQuorumErrs(errs, nil, len(globalAdminPeers)/2+1) + rErr = reduceWriteQuorumErrs(ctx, errs, nil, len(globalAdminPeers)/2+1) if rErr != nil { writeSetConfigResponse(w, globalAdminPeers, errs, false, r.URL) return @@ -777,7 +783,7 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter, var req madmin.SetCredsReq err := json.NewDecoder(r.Body).Decode(&req) if err != nil { - errorIf(err, "Error parsing body JSON") + logger.LogIf(context.Background(), err) writeErrorResponseJSON(w, ErrRequestBodyParse, r.URL) return } @@ -804,7 +810,9 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter, // Notify all other Minio peers to update credentials updateErrs := updateCredsOnPeers(creds) for peer, err := range updateErrs { - errorIf(err, "Unable to update credentials on peer %s.", peer) + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", peer) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) } // Update local credentials in memory. diff --git a/cmd/admin-handlers_test.go b/cmd/admin-handlers_test.go index 4d60390d9..c8e9f02a3 100644 --- a/cmd/admin-handlers_test.go +++ b/cmd/admin-handlers_test.go @@ -33,7 +33,6 @@ import ( router "github.com/gorilla/mux" "github.com/minio/minio/pkg/auth" - "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/madmin" ) @@ -264,7 +263,7 @@ func initTestXLObjLayer() (ObjectLayer, []string, error) { return nil, nil, err } endpoints := mustGetNewEndpointList(xlDirs...) - format, err := waitForFormatXL(true, endpoints, 1, 16) + format, err := waitForFormatXL(context.Background(), true, endpoints, 1, 16) if err != nil { removeRoots(xlDirs) return nil, nil, err @@ -762,13 +761,13 @@ func buildAdminRequest(queryVal url.Values, method, path string, "/minio/admin/v1"+path+"?"+queryVal.Encode(), contentLength, bodySeeker) if err != nil { - return nil, errors.Trace(err) + return nil, err } cred := globalServerConfig.GetCredential() err = signRequestV4(req, cred.AccessKey, cred.SecretKey) if err != nil { - return nil, errors.Trace(err) + return nil, err } return req, nil diff --git a/cmd/admin-heal-ops.go b/cmd/admin-heal-ops.go index b67a199ac..cca1728aa 100644 --- a/cmd/admin-heal-ops.go +++ b/cmd/admin-heal-ops.go @@ -222,7 +222,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) ( StartTime: h.startTime, }) if err != nil { - errorIf(err, "Failed to marshal heal result into json.") + logger.LogIf(context.Background(), err) return nil, ErrInternalError, "" } return b, ErrNone, "" @@ -270,7 +270,7 @@ func (ahs *allHealState) PopHealStatusJSON(path string, jbytes, err := json.Marshal(h.currentStatus) if err != nil { - errorIf(err, "Failed to marshal heal result into json.") + logger.LogIf(context.Background(), err) return nil, ErrInternalError } @@ -321,7 +321,9 @@ type healSequence struct { func newHealSequence(bucket, objPrefix, clientAddr string, numDisks int, hs madmin.HealOpts, forceStart bool) *healSequence { - ctx := logger.SetContext(context.Background(), &logger.ReqInfo{clientAddr, "", "", "Heal", bucket, objPrefix, nil}) + reqInfo := &logger.ReqInfo{RemoteHost: clientAddr, API: "Heal", BucketName: bucket} + reqInfo.AppendTags("prefix", objPrefix) + ctx := logger.SetReqInfo(context.Background(), reqInfo) return &healSequence{ bucket: bucket, diff --git a/cmd/admin-rpc-client.go b/cmd/admin-rpc-client.go index d9c2bc134..ba53cfc29 100644 --- a/cmd/admin-rpc-client.go +++ b/cmd/admin-rpc-client.go @@ -29,7 +29,7 @@ import ( "time" "github.com/minio/minio-go/pkg/set" - "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" ) const ( @@ -201,7 +201,7 @@ func (rc remoteAdminClient) WriteTmpConfig(tmpFileName string, configBytes []byt err := rc.Call(writeTmpConfigRPC, &wArgs, &WriteConfigReply{}) if err != nil { - errorIf(err, "Failed to write temporary config file.") + logger.LogIf(context.Background(), err) return err } @@ -215,7 +215,10 @@ func (lc localAdminClient) CommitConfig(tmpFileName string) error { tmpConfigFile := filepath.Join(getConfigDir(), tmpFileName) err := os.Rename(tmpConfigFile, configFile) - errorIf(err, fmt.Sprintf("Failed to rename %s to %s", tmpConfigFile, configFile)) + reqInfo := (&logger.ReqInfo{}).AppendTags("tmpConfigFile", tmpConfigFile) + reqInfo.AppendTags("configFile", configFile) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) return err } @@ -228,7 +231,7 @@ func (rc remoteAdminClient) CommitConfig(tmpFileName string) error { cReply := CommitConfigReply{} err := rc.Call(commitConfigRPC, &cArgs, &cReply) if err != nil { - errorIf(err, "Failed to rename config file.") + logger.LogIf(context.Background(), err) return err } @@ -436,7 +439,7 @@ func getPeerUptimes(peers adminPeers) (time.Duration, error) { latestUptime := time.Duration(0) for _, uptime := range uptimes { if uptime.err != nil { - errorIf(uptime.err, "Unable to fetch uptime") + logger.LogIf(context.Background(), uptime.err) continue } @@ -489,15 +492,17 @@ func getPeerConfig(peers adminPeers) ([]byte, error) { // Unmarshal the received config files. err := json.Unmarshal(configBytes, &serverConfigs[i]) if err != nil { - errorIf(err, "Failed to unmarshal serverConfig from ", peers[i].addr) + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", peers[i].addr) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) return nil, err } } configJSON, err := getValidServerConfig(serverConfigs, errs) if err != nil { - errorIf(err, "Unable to find a valid server config") - return nil, errors.Trace(err) + logger.LogIf(context.Background(), err) + return nil, err } // Return the config.json that was present quorum or more diff --git a/cmd/admin-rpc-server.go b/cmd/admin-rpc-server.go index 22e9e2fde..1344bf0ee 100644 --- a/cmd/admin-rpc-server.go +++ b/cmd/admin-rpc-server.go @@ -26,7 +26,7 @@ import ( "time" router "github.com/gorilla/mux" - "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" ) const adminPath = "/admin" @@ -176,7 +176,9 @@ type WriteConfigReply struct { func writeTmpConfigCommon(tmpFileName string, configBytes []byte) error { tmpConfigFile := filepath.Join(getConfigDir(), tmpFileName) err := ioutil.WriteFile(tmpConfigFile, configBytes, 0666) - errorIf(err, fmt.Sprintf("Failed to write to temporary config file %s", tmpConfigFile)) + reqInfo := (&logger.ReqInfo{}).AppendTags("tmpConfigFile", tmpConfigFile) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) return err } @@ -209,7 +211,10 @@ func (s *adminCmd) CommitConfig(cArgs *CommitConfigArgs, cReply *CommitConfigRep tmpConfigFile := filepath.Join(getConfigDir(), cArgs.FileName) err := os.Rename(tmpConfigFile, configFile) - errorIf(err, fmt.Sprintf("Failed to rename %s to %s", tmpConfigFile, configFile)) + reqInfo := (&logger.ReqInfo{}).AppendTags("tmpConfigFile", tmpConfigFile) + reqInfo.AppendTags("configFile", configFile) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) return err } @@ -220,7 +225,8 @@ func registerAdminRPCRouter(mux *router.Router) error { adminRPCServer := newRPCServer() err := adminRPCServer.RegisterName("Admin", adminRPCHandler) if err != nil { - return errors.Trace(err) + logger.LogIf(context.Background(), err) + return err } adminRouter := mux.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter() adminRouter.Path(adminPath).Handler(adminRPCServer) diff --git a/cmd/api-errors.go b/cmd/api-errors.go index b971d3d80..10331f4af 100644 --- a/cmd/api-errors.go +++ b/cmd/api-errors.go @@ -793,7 +793,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{ HTTPStatusCode: http.StatusBadRequest, }, // Generic Invalid-Request error. Should be used for response errors only for unlikely - // corner case errors for which introducing new APIErrorCode is not worth it. errorIf() + // corner case errors for which introducing new APIErrorCode is not worth it. LogIf() // should be used to log the error at the source of the error for debugging purposes. ErrInvalidRequest: { Code: "InvalidRequest", diff --git a/cmd/api-router.go b/cmd/api-router.go index 9d8c21529..808fc7211 100644 --- a/cmd/api-router.go +++ b/cmd/api-router.go @@ -20,6 +20,7 @@ import ( "net/http" router "github.com/gorilla/mux" + "github.com/minio/minio/cmd/logger" ) // objectAPIHandler implements and provides http handlers for S3 API. @@ -35,7 +36,7 @@ func registerAPIRouter(mux *router.Router) { if len(cacheConfig.Drives) > 0 { // initialize the new disk cache objects. globalCacheObjectAPI, err = newServerCacheObjects(cacheConfig) - fatalIf(err, "Unable to initialize disk caching") + logger.FatalIf(err, "Unable to initialize disk caching") } // Initialize API. diff --git a/cmd/auth-handler.go b/cmd/auth-handler.go index 9e54f9b6f..324f04252 100644 --- a/cmd/auth-handler.go +++ b/cmd/auth-handler.go @@ -18,6 +18,7 @@ package cmd import ( "bytes" + "context" "encoding/base64" "encoding/hex" "errors" @@ -25,6 +26,7 @@ import ( "net/http" "strings" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/handlers" ) @@ -114,12 +116,14 @@ func checkAdminRequestAuthType(r *http.Request, region string) APIErrorCode { s3Err = isReqAuthenticated(r, region) } if s3Err != ErrNone { - errorIf(errors.New(getAPIError(s3Err).Description), "%s", dumpRequest(r)) + reqInfo := (&logger.ReqInfo{}).AppendTags("requestHeaders", dumpRequest(r)) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, errors.New(getAPIError(s3Err).Description)) } return s3Err } -func checkRequestAuthType(r *http.Request, bucket, policyAction, region string) APIErrorCode { +func checkRequestAuthType(ctx context.Context, r *http.Request, bucket, policyAction, region string) APIErrorCode { reqAuthType := getRequestAuthType(r) switch reqAuthType { @@ -136,7 +140,7 @@ func checkRequestAuthType(r *http.Request, bucket, policyAction, region string) if err != nil { return ErrInternalError } - return enforceBucketPolicy(bucket, policyAction, resource, + return enforceBucketPolicy(ctx, bucket, policyAction, resource, r.Referer(), handlers.GetSourceIP(r), r.URL.Query()) } @@ -176,7 +180,7 @@ func isReqAuthenticated(r *http.Request, region string) (s3Error APIErrorCode) { payload, err := ioutil.ReadAll(r.Body) if err != nil { - errorIf(err, "Unable to read request body for signature verification") + logger.LogIf(context.Background(), err) return ErrInternalError } diff --git a/cmd/auth-rpc-client.go b/cmd/auth-rpc-client.go index e0e0b4c77..7ba853807 100644 --- a/cmd/auth-rpc-client.go +++ b/cmd/auth-rpc-client.go @@ -18,6 +18,7 @@ package cmd import ( "bufio" + "context" "crypto/tls" "crypto/x509" "errors" @@ -29,6 +30,8 @@ import ( "strings" "sync" "time" + + "github.com/minio/minio/cmd/logger" ) // Attempt to retry only this many number of times before @@ -264,7 +267,9 @@ func rpcDial(serverAddr, serviceEndpoint string, secureConn bool) (netRPCClient // Print RPC connection errors that are worthy to display in log. switch err.(type) { case x509.HostnameError: - errorIf(err, "Unable to establish secure connection to %s", serverAddr) + reqInfo := (&logger.ReqInfo{}).AppendTags("serverAddr", serverAddr) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) } return nil, &net.OpError{ diff --git a/cmd/browser-peer-rpc.go b/cmd/browser-peer-rpc.go index febf59028..88db66b22 100644 --- a/cmd/browser-peer-rpc.go +++ b/cmd/browser-peer-rpc.go @@ -17,11 +17,13 @@ package cmd import ( + "context" "fmt" "path" "sync" "time" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" ) @@ -62,7 +64,7 @@ func (br *browserPeerAPIHandlers) SetAuthPeer(args SetAuthPeerArgs, reply *AuthR // Save the current creds when failed to update. globalServerConfig.SetCredential(prevCred) - errorIf(err, "Unable to update the config with new credentials sent from browser RPC.") + logger.LogIf(context.Background(), err) return err } diff --git a/cmd/browser-rpc-router.go b/cmd/browser-rpc-router.go index 9c6e70480..1a48be653 100644 --- a/cmd/browser-rpc-router.go +++ b/cmd/browser-rpc-router.go @@ -17,9 +17,10 @@ package cmd import ( - router "github.com/gorilla/mux" + "context" - "github.com/minio/minio/pkg/errors" + router "github.com/gorilla/mux" + "github.com/minio/minio/cmd/logger" ) // Set up an RPC endpoint that receives browser related calls. The @@ -42,7 +43,8 @@ func registerBrowserPeerRPCRouter(mux *router.Router) error { bpRPCServer := newRPCServer() err := bpRPCServer.RegisterName("BrowserPeer", bpHandlers) if err != nil { - return errors.Trace(err) + logger.LogIf(context.Background(), err) + return err } bpRouter := mux.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter() diff --git a/cmd/bucket-handlers-listobjects.go b/cmd/bucket-handlers-listobjects.go index 39a4fb078..3f4637b65 100644 --- a/cmd/bucket-handlers-listobjects.go +++ b/cmd/bucket-handlers-listobjects.go @@ -64,7 +64,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http return } - if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return } @@ -134,7 +134,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http return } - if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return } diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index 7ff6c2c14..f0fac21a1 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -33,6 +33,7 @@ import ( mux "github.com/gorilla/mux" "github.com/minio/minio-go/pkg/policy" "github.com/minio/minio-go/pkg/set" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/event" "github.com/minio/minio/pkg/hash" @@ -40,10 +41,10 @@ import ( // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html // Enforces bucket policies for a bucket for a given tatusaction. -func enforceBucketPolicy(bucket, action, resource, referer, sourceIP string, queryParams url.Values) (s3Error APIErrorCode) { +func enforceBucketPolicy(ctx context.Context, bucket, action, resource, referer, sourceIP string, queryParams url.Values) (s3Error APIErrorCode) { // Verify if bucket actually exists objAPI := newObjectLayerFn() - if err := checkBucketExist(bucket, objAPI); err != nil { + if err := checkBucketExist(ctx, bucket, objAPI); err != nil { err = errors.Cause(err) switch err.(type) { case BucketNameInvalid: @@ -53,13 +54,12 @@ func enforceBucketPolicy(bucket, action, resource, referer, sourceIP string, que // For no bucket found we return NoSuchBucket instead. return ErrNoSuchBucket } - errorIf(err, "Unable to read bucket policy.") // Return internal error for any other errors so that we can investigate. return ErrInternalError } // Fetch bucket policy, if policy is not set return access denied. - p, err := objAPI.GetBucketPolicy(context.Background(), bucket) + p, err := objAPI.GetBucketPolicy(ctx, bucket) if err != nil { return ErrAccessDenied } @@ -92,7 +92,10 @@ func enforceBucketPolicy(bucket, action, resource, referer, sourceIP string, que // Check if the action is allowed on the bucket/prefix. func isBucketActionAllowed(action, bucket, prefix string, objectAPI ObjectLayer) bool { - bp, err := objectAPI.GetBucketPolicy(context.Background(), bucket) + reqInfo := &logger.ReqInfo{BucketName: bucket} + reqInfo.AppendTags("prefix", prefix) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + bp, err := objectAPI.GetBucketPolicy(ctx, bucket) if err != nil { return false } @@ -120,10 +123,10 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r * return } - s3Error := checkRequestAuthType(r, bucket, "s3:GetBucketLocation", globalMinioDefaultRegion) + s3Error := checkRequestAuthType(ctx, r, bucket, "s3:GetBucketLocation", globalMinioDefaultRegion) if s3Error == ErrInvalidRegion { // Clients like boto3 send getBucketLocation() call signed with region that is configured. - s3Error = checkRequestAuthType(r, "", "s3:GetBucketLocation", globalServerConfig.GetRegion()) + s3Error = checkRequestAuthType(ctx, r, "", "s3:GetBucketLocation", globalServerConfig.GetRegion()) } if s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) @@ -179,7 +182,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, return } - if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucketMultipartUploads", globalServerConfig.GetRegion()); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:ListBucketMultipartUploads", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return } @@ -228,10 +231,10 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R listBuckets = api.CacheAPI().ListBuckets } // ListBuckets does not have any bucket action. - s3Error := checkRequestAuthType(r, "", "", globalMinioDefaultRegion) + s3Error := checkRequestAuthType(ctx, r, "", "", globalMinioDefaultRegion) if s3Error == ErrInvalidRegion { // Clients like boto3 send listBuckets() call signed with region that is configured. - s3Error = checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()) + s3Error = checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion()) } if s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) @@ -266,7 +269,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, } var authError APIErrorCode - if authError = checkRequestAuthType(r, bucket, "s3:DeleteObject", globalServerConfig.GetRegion()); authError != ErrNone { + if authError = checkRequestAuthType(ctx, r, bucket, "s3:DeleteObject", globalServerConfig.GetRegion()); authError != ErrNone { // In the event access is denied, a 200 response should still be returned // http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html if authError != ErrAccessDenied { @@ -294,7 +297,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, // Read incoming body XML bytes. if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil { - errorIf(err, "Unable to read HTTP body.") + logger.LogIf(ctx, err) writeErrorResponse(w, ErrInternalError, r.URL) return } @@ -302,7 +305,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, // Unmarshal list of keys to be deleted. deleteObjects := &DeleteObjectsRequest{} if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil { - errorIf(err, "Unable to unmarshal delete objects request XML.") + logger.LogIf(ctx, err) writeErrorResponse(w, ErrMalformedXML, r.URL) return } @@ -411,7 +414,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req } // PutBucket does not have any bucket action. - s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()) + s3Error := checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion()) if s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return @@ -490,7 +493,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h // be loaded in memory, the remaining being put in temporary files. reader, err := r.MultipartReader() if err != nil { - errorIf(err, "Unable to initialize multipart reader.") + logger.LogIf(ctx, err) writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL) return } @@ -498,7 +501,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h // Read multipart data and save in memory and in the disk if needed form, err := reader.ReadForm(maxFormMemory) if err != nil { - errorIf(err, "Unable to initialize multipart reader.") + logger.LogIf(ctx, err) writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL) return } @@ -507,9 +510,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h defer form.RemoveAll() // Extract all form fields - fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(form) + fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(ctx, form) if err != nil { - errorIf(err, "Unable to parse form values.") + logger.LogIf(ctx, err) writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL) return } @@ -584,16 +587,15 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h } // Extract metadata to be saved from received Form. - metadata, err := extractMetadataFromHeader(formValues) + metadata, err := extractMetadataFromHeader(ctx, formValues) if err != nil { - errorIf(err, "found invalid http request header") writeErrorResponse(w, ErrInternalError, r.URL) return } hashReader, err := hash.NewReader(fileBody, fileSize, "", "") if err != nil { - errorIf(err, "Unable to initialize hashReader.") + logger.LogIf(ctx, err) writeErrorResponse(w, toAPIErrorCode(err), r.URL) return } @@ -690,7 +692,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re return } - if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponseHeadersOnly(w, s3Error) return } @@ -717,7 +719,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http. } // DeleteBucket does not have any bucket action. - if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return } @@ -736,12 +738,14 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http. // Notify all peers (including self) to update in-memory state for addr, err := range globalNotificationSys.UpdateBucketPolicy(bucket) { - errorIf(err, "unable to update policy change in remote peer %v", addr) + logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name) + logger.LogIf(ctx, err) } globalNotificationSys.RemoveNotification(bucket) for addr, err := range globalNotificationSys.DeleteBucket(bucket) { - errorIf(err, "unable to delete bucket in remote peer %v", addr) + logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name) + logger.LogIf(ctx, err) } // Write success response. diff --git a/cmd/bucket-notification-handlers.go b/cmd/bucket-notification-handlers.go index a45f52934..4996acfd6 100644 --- a/cmd/bucket-notification-handlers.go +++ b/cmd/bucket-notification-handlers.go @@ -23,6 +23,7 @@ import ( "net/http" "github.com/gorilla/mux" + "github.com/minio/minio/cmd/logger" xerrors "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/event" "github.com/minio/minio/pkg/event/target" @@ -53,7 +54,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, writeErrorResponse(w, ErrNotImplemented, r.URL) return } - if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return } @@ -63,17 +64,15 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, _, err := objAPI.GetBucketInfo(ctx, bucketName) if err != nil { - errorIf(err, "Unable to find bucket info.") writeErrorResponse(w, toAPIErrorCode(err), r.URL) return } // Attempt to successfully load notification config. - nConfig, err := readNotificationConfig(objAPI, bucketName) + nConfig, err := readNotificationConfig(ctx, objAPI, bucketName) if err != nil { // Ignore errNoSuchNotifications to comply with AWS S3. if xerrors.Cause(err) != errNoSuchNotifications { - errorIf(err, "Unable to read notification configuration.") writeErrorResponse(w, toAPIErrorCode(err), r.URL) return } @@ -83,7 +82,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, notificationBytes, err := xml.Marshal(nConfig) if err != nil { - errorIf(err, "Unable to marshal notification configuration into XML.", err) + logger.LogIf(ctx, err) writeErrorResponse(w, toAPIErrorCode(err), r.URL) return } @@ -106,7 +105,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, writeErrorResponse(w, ErrNotImplemented, r.URL) return } - if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return } @@ -146,7 +145,8 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, rulesMap := config.ToRulesMap() globalNotificationSys.AddRulesMap(bucketName, rulesMap) for addr, err := range globalNotificationSys.PutBucketNotification(bucketName, rulesMap) { - errorIf(err, "unable to put bucket notification to remote peer %v", addr) + logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name) + logger.LogIf(ctx, err) } writeSuccessResponseHeadersOnly(w) @@ -167,7 +167,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit writeErrorResponse(w, ErrNotImplemented, r.URL) return } - if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return } @@ -217,7 +217,6 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit } if _, err := objAPI.GetBucketInfo(ctx, bucketName); err != nil { - errorIf(err, "Unable to get bucket info.") writeErrorResponse(w, toAPIErrorCode(err), r.URL) return } @@ -227,7 +226,8 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit rulesMap := event.NewRulesMap(eventNames, pattern, target.ID()) if err := globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil { - errorIf(err, "Unable to add httpclient target %v to globalNotificationSys.targetList.", target) + logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name) + logger.LogIf(ctx, err) writeErrorResponse(w, toAPIErrorCode(err), r.URL) return } @@ -236,20 +236,23 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit thisAddr := xnet.MustParseHost(GetLocalPeer(globalEndpoints)) if err := SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil { - errorIf(err, "Unable to save HTTP listener %v", target) + logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name) + logger.LogIf(ctx, err) writeErrorResponse(w, toAPIErrorCode(err), r.URL) return } errors := globalNotificationSys.ListenBucketNotification(bucketName, eventNames, pattern, target.ID(), *thisAddr) for addr, err := range errors { - errorIf(err, "unable to call listen bucket notification to remote peer %v", addr) + logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name) + logger.LogIf(ctx, err) } <-target.DoneCh if err := RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil { - errorIf(err, "Unable to save HTTP listener %v", target) + logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name) + logger.LogIf(ctx, err) writeErrorResponse(w, toAPIErrorCode(err), r.URL) return } diff --git a/cmd/bucket-policy-handlers.go b/cmd/bucket-policy-handlers.go index 9c7f81827..389c5e05f 100644 --- a/cmd/bucket-policy-handlers.go +++ b/cmd/bucket-policy-handlers.go @@ -28,6 +28,7 @@ import ( humanize "github.com/dustin/go-humanize" mux "github.com/gorilla/mux" "github.com/minio/minio-go/pkg/policy" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/wildcard" ) @@ -228,7 +229,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht return } - if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return } @@ -260,7 +261,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht // bucket policies are limited to 20KB in size, using a limit reader. policyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize)) if err != nil { - errorIf(err, "Unable to read from client.") + logger.LogIf(ctx, err) writeErrorResponse(w, toAPIErrorCode(err), r.URL) return } @@ -288,7 +289,8 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht } for addr, err := range globalNotificationSys.UpdateBucketPolicy(bucket) { - errorIf(err, "unable to update policy change in remote peer %v", addr) + logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name) + logger.LogIf(ctx, err) } // Success. @@ -308,7 +310,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r return } - if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return } @@ -331,7 +333,8 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r } for addr, err := range globalNotificationSys.UpdateBucketPolicy(bucket) { - errorIf(err, "unable to update policy change in remote peer %v", addr) + logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name) + logger.LogIf(ctx, err) } // Success. @@ -351,7 +354,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht return } - if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return } @@ -375,7 +378,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht policyBytes, err := json.Marshal(&policy) if err != nil { - errorIf(err, "Unable to marshal bucket policy.") + logger.LogIf(ctx, err) writeErrorResponse(w, toAPIErrorCode(err), r.URL) return } diff --git a/cmd/bucket-policy.go b/cmd/bucket-policy.go index 284211e1d..c098844e6 100644 --- a/cmd/bucket-policy.go +++ b/cmd/bucket-policy.go @@ -25,6 +25,7 @@ import ( "sync" "github.com/minio/minio-go/pkg/policy" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/hash" ) @@ -119,12 +120,12 @@ func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig) var buffer bytes.Buffer - err = objAPI.GetObject(context.Background(), minioMetaBucket, policyPath, 0, -1, &buffer, "") + ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{BucketName: bucket}) + err = objAPI.GetObject(ctx, minioMetaBucket, policyPath, 0, -1, &buffer, "") if err != nil { if isErrObjectNotFound(err) || isErrIncompleteBody(err) { return nil, PolicyNotFound{Bucket: bucket} } - errorIf(err, "Unable to load policy for the bucket %s.", bucket) return nil, errors.Cause(err) } @@ -151,9 +152,9 @@ func ReadBucketPolicy(bucket string, objAPI ObjectLayer) (policy.BucketAccessPol // removeBucketPolicy - removes any previously written bucket policy. Returns BucketPolicyNotFound // if no policies are found. -func removeBucketPolicy(bucket string, objAPI ObjectLayer) error { +func removeBucketPolicy(ctx context.Context, bucket string, objAPI ObjectLayer) error { policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig) - err := objAPI.DeleteObject(context.Background(), minioMetaBucket, policyPath) + err := objAPI.DeleteObject(ctx, minioMetaBucket, policyPath) if err != nil { err = errors.Cause(err) if _, ok := err.(ObjectNotFound); ok { @@ -165,21 +166,21 @@ func removeBucketPolicy(bucket string, objAPI ObjectLayer) error { } // writeBucketPolicy - save a bucket policy that is assumed to be validated. -func writeBucketPolicy(bucket string, objAPI ObjectLayer, bpy policy.BucketAccessPolicy) error { +func writeBucketPolicy(ctx context.Context, bucket string, objAPI ObjectLayer, bpy policy.BucketAccessPolicy) error { buf, err := json.Marshal(bpy) + if err != nil { - errorIf(err, "Unable to marshal bucket policy '%#v' to JSON", bpy) + logger.LogIf(ctx, err) return err } policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig) hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf)) if err != nil { - errorIf(err, "Unable to set policy for the bucket %s", bucket) + logger.LogIf(ctx, err) return errors.Cause(err) } - if _, err = objAPI.PutObject(context.Background(), minioMetaBucket, policyPath, hashReader, nil); err != nil { - errorIf(err, "Unable to set policy for the bucket %s", bucket) + if _, err = objAPI.PutObject(ctx, minioMetaBucket, policyPath, hashReader, nil); err != nil { return errors.Cause(err) } return nil @@ -188,9 +189,9 @@ func writeBucketPolicy(bucket string, objAPI ObjectLayer, bpy policy.BucketAcces // persistAndNotifyBucketPolicyChange - takes a policyChange argument, // persists it to storage, and notify nodes in the cluster about the // change. In-memory state is updated in response to the notification. -func persistAndNotifyBucketPolicyChange(bucket string, isRemove bool, bktPolicy policy.BucketAccessPolicy, objAPI ObjectLayer) error { +func persistAndNotifyBucketPolicyChange(ctx context.Context, bucket string, isRemove bool, bktPolicy policy.BucketAccessPolicy, objAPI ObjectLayer) error { if isRemove { - err := removeBucketPolicy(bucket, objAPI) + err := removeBucketPolicy(ctx, bucket, objAPI) if err != nil { return err } @@ -198,7 +199,7 @@ func persistAndNotifyBucketPolicyChange(bucket string, isRemove bool, bktPolicy if reflect.DeepEqual(bktPolicy, emptyBucketPolicy) { return errInvalidArgument } - if err := writeBucketPolicy(bucket, objAPI, bktPolicy); err != nil { + if err := writeBucketPolicy(ctx, bucket, objAPI, bktPolicy); err != nil { return err } } diff --git a/cmd/common-main.go b/cmd/common-main.go index 9420411b6..c24b1bf23 100644 --- a/cmd/common-main.go +++ b/cmd/common-main.go @@ -25,6 +25,7 @@ import ( "time" "github.com/minio/cli" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" ) @@ -33,9 +34,9 @@ func checkUpdate(mode string) { // Its OK to ignore any errors during doUpdate() here. if updateMsg, _, currentReleaseTime, latestReleaseTime, err := getUpdateInfo(2*time.Second, mode); err == nil { if globalInplaceUpdateDisabled { - log.Println(updateMsg) + logger.Println(updateMsg) } else { - log.Println(prepareUpdateMessage("Run `minio update`", latestReleaseTime.Sub(currentReleaseTime))) + logger.Println(prepareUpdateMessage("Run `minio update`", latestReleaseTime.Sub(currentReleaseTime))) } } } @@ -43,11 +44,11 @@ func checkUpdate(mode string) { func initConfig() { // Config file does not exist, we create it fresh and return upon success. if isFile(getConfigFile()) { - fatalIf(migrateConfig(), "Config migration failed.") - fatalIf(loadConfig(), "Unable to load config version: '%s'.", serverConfigVersion) + logger.FatalIf(migrateConfig(), "Config migration failed.") + logger.FatalIf(loadConfig(), "Unable to load config version: '%s'.", serverConfigVersion) } else { - fatalIf(newConfig(), "Unable to initialize minio config for the first time.") - log.Println("Created minio configuration file successfully at " + getConfigDir()) + logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time.") + logger.Println("Created minio configuration file successfully at " + getConfigDir()) } } @@ -70,17 +71,17 @@ func handleCommonCmdArgs(ctx *cli.Context) { // default config directory. configDir = getConfigDir() if configDir == "" { - fatalIf(errors.New("missing option"), "config-dir option must be provided.") + logger.FatalIf(errors.New("missing option"), "config-dir option must be provided.") } } if configDir == "" { - fatalIf(errors.New("empty directory"), "Configuration directory cannot be empty.") + logger.FatalIf(errors.New("empty directory"), "Configuration directory cannot be empty.") } // Disallow relative paths, figure out absolute paths. configDirAbs, err := filepath.Abs(configDir) - fatalIf(err, "Unable to fetch absolute path for config directory %s", configDir) + logger.FatalIf(err, "Unable to fetch absolute path for config directory %s", configDir) setConfigDir(configDirAbs) } @@ -94,7 +95,7 @@ func handleCommonEnvVars() { secretKey := os.Getenv("MINIO_SECRET_KEY") if accessKey != "" && secretKey != "" { cred, err := auth.CreateCredentials(accessKey, secretKey) - fatalIf(err, "Invalid access/secret Key set in environment.") + logger.FatalIf(err, "Invalid access/secret Key set in environment.") // credential Envs are set globally. globalIsEnvCreds = true @@ -104,7 +105,7 @@ func handleCommonEnvVars() { if browser := os.Getenv("MINIO_BROWSER"); browser != "" { browserFlag, err := ParseBrowserFlag(browser) if err != nil { - fatalIf(errors.New("invalid value"), "Unknown value ‘%s’ in MINIO_BROWSER environment variable.", browser) + logger.FatalIf(errors.New("invalid value"), "Unknown value ‘%s’ in MINIO_BROWSER environment variable.", browser) } // browser Envs are set globally, this does not represent @@ -117,7 +118,7 @@ func handleCommonEnvVars() { if traceFile != "" { var err error globalHTTPTraceFile, err = os.OpenFile(traceFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0660) - fatalIf(err, "error opening file %s", traceFile) + logger.FatalIf(err, "error opening file %s", traceFile) } globalDomainName = os.Getenv("MINIO_DOMAIN") @@ -127,18 +128,18 @@ func handleCommonEnvVars() { if drives := os.Getenv("MINIO_CACHE_DRIVES"); drives != "" { driveList, err := parseCacheDrives(strings.Split(drives, cacheEnvDelimiter)) - fatalIf(err, "Invalid value set in environment variable MINIO_CACHE_DRIVES %s.", drives) + logger.FatalIf(err, "Invalid value set in environment variable MINIO_CACHE_DRIVES %s.", drives) globalCacheDrives = driveList globalIsDiskCacheEnabled = true } if excludes := os.Getenv("MINIO_CACHE_EXCLUDE"); excludes != "" { excludeList, err := parseCacheExcludes(strings.Split(excludes, cacheEnvDelimiter)) - fatalIf(err, "Invalid value set in environment variable MINIO_CACHE_EXCLUDE %s.", excludes) + logger.FatalIf(err, "Invalid value set in environment variable MINIO_CACHE_EXCLUDE %s.", excludes) globalCacheExcludes = excludeList } if expiryStr := os.Getenv("MINIO_CACHE_EXPIRY"); expiryStr != "" { expiry, err := strconv.Atoi(expiryStr) - fatalIf(err, "Invalid value set in environment variable MINIO_CACHE_EXPIRY %s.", expiryStr) + logger.FatalIf(err, "Invalid value set in environment variable MINIO_CACHE_EXPIRY %s.", expiryStr) globalCacheExpiry = expiry } @@ -154,25 +155,25 @@ func handleCommonEnvVars() { // Check for environment variables and parse into storageClass struct if ssc := os.Getenv(standardStorageClassEnv); ssc != "" { globalStandardStorageClass, err = parseStorageClass(ssc) - fatalIf(err, "Invalid value set in environment variable %s.", standardStorageClassEnv) + logger.FatalIf(err, "Invalid value set in environment variable %s.", standardStorageClassEnv) } if rrsc := os.Getenv(reducedRedundancyStorageClassEnv); rrsc != "" { globalRRStorageClass, err = parseStorageClass(rrsc) - fatalIf(err, "Invalid value set in environment variable %s.", reducedRedundancyStorageClassEnv) + logger.FatalIf(err, "Invalid value set in environment variable %s.", reducedRedundancyStorageClassEnv) } // Validation is done after parsing both the storage classes. This is needed because we need one // storage class value to deduce the correct value of the other storage class. if globalRRStorageClass.Scheme != "" { err = validateParity(globalStandardStorageClass.Parity, globalRRStorageClass.Parity) - fatalIf(err, "Invalid value set in environment variable %s.", reducedRedundancyStorageClassEnv) + logger.FatalIf(err, "Invalid value set in environment variable %s.", reducedRedundancyStorageClassEnv) globalIsStorageClass = true } if globalStandardStorageClass.Scheme != "" { err = validateParity(globalStandardStorageClass.Parity, globalRRStorageClass.Parity) - fatalIf(err, "Invalid value set in environment variable %s.", standardStorageClassEnv) + logger.FatalIf(err, "Invalid value set in environment variable %s.", standardStorageClassEnv) globalIsStorageClass = true } } diff --git a/cmd/config-migrate.go b/cmd/config-migrate.go index b40fcdb84..bd45ed7aa 100644 --- a/cmd/config-migrate.go +++ b/cmd/config-migrate.go @@ -21,6 +21,7 @@ import ( "os" "path/filepath" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" "github.com/minio/minio/pkg/event" "github.com/minio/minio/pkg/event/target" @@ -194,7 +195,7 @@ func purgeV1() error { } os.RemoveAll(configFile) - log.Println("Removed unsupported config version ‘1’.") + logger.Println("Removed unsupported config version ‘1’.") return nil } @@ -252,7 +253,7 @@ func migrateV2ToV3() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv2.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv2.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv2.Version, srvConfig.Version) return nil } @@ -290,7 +291,7 @@ func migrateV3ToV4() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv3.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv3.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv3.Version, srvConfig.Version) return nil } @@ -331,7 +332,7 @@ func migrateV4ToV5() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv4.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv4.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv4.Version, srvConfig.Version) return nil } @@ -420,7 +421,7 @@ func migrateV5ToV6() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv5.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv5.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv5.Version, srvConfig.Version) return nil } @@ -476,7 +477,7 @@ func migrateV6ToV7() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv6.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv6.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv6.Version, srvConfig.Version) return nil } @@ -539,7 +540,7 @@ func migrateV7ToV8() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv7.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv7.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv7.Version, srvConfig.Version) return nil } @@ -609,7 +610,7 @@ func migrateV8ToV9() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv8.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv8.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv8.Version, srvConfig.Version) return nil } @@ -677,7 +678,7 @@ func migrateV9ToV10() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv9.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv9.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv9.Version, srvConfig.Version) return nil } @@ -748,7 +749,7 @@ func migrateV10ToV11() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv10.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv10.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv10.Version, srvConfig.Version) return nil } @@ -846,7 +847,7 @@ func migrateV11ToV12() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv11.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv11.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv11.Version, srvConfig.Version) return nil } @@ -926,7 +927,7 @@ func migrateV12ToV13() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv12.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv12.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv12.Version, srvConfig.Version) return nil } @@ -1011,7 +1012,7 @@ func migrateV13ToV14() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv13.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv13.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv13.Version, srvConfig.Version) return nil } @@ -1100,7 +1101,7 @@ func migrateV14ToV15() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv14.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv14.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv14.Version, srvConfig.Version) return nil } @@ -1190,7 +1191,7 @@ func migrateV15ToV16() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv15.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv15.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv15.Version, srvConfig.Version) return nil } @@ -1311,7 +1312,7 @@ func migrateV16ToV17() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv16.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv16.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv16.Version, srvConfig.Version) return nil } @@ -1415,7 +1416,7 @@ func migrateV17ToV18() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv17.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv17.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv17.Version, srvConfig.Version) return nil } @@ -1521,7 +1522,7 @@ func migrateV18ToV19() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv18.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv18.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv18.Version, srvConfig.Version) return nil } @@ -1626,7 +1627,7 @@ func migrateV19ToV20() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv19.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv19.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv19.Version, srvConfig.Version) return nil } @@ -1730,7 +1731,7 @@ func migrateV20ToV21() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv20.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv20.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv20.Version, srvConfig.Version) return nil } @@ -1834,7 +1835,7 @@ func migrateV21ToV22() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv21.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv21.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv21.Version, srvConfig.Version) return nil } @@ -1947,6 +1948,6 @@ func migrateV22ToV23() error { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv22.Version, srvConfig.Version, err) } - log.Printf(configMigrateMSGTemplate, configFile, cv22.Version, srvConfig.Version) + logger.Printf(configMigrateMSGTemplate, configFile, cv22.Version, srvConfig.Version) return nil } diff --git a/cmd/disk-cache-fs.go b/cmd/disk-cache-fs.go index b1dcd32a8..abf66c8f3 100644 --- a/cmd/disk-cache-fs.go +++ b/cmd/disk-cache-fs.go @@ -28,6 +28,7 @@ import ( "sync" "time" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/disk" errors2 "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/hash" @@ -93,7 +94,7 @@ func newCacheFSObjects(dir string, expiry int, maxDiskUsagePct int) (*cacheFSObj appendFileMap: make(map[string]*fsAppendFile), } - go fsObjects.cleanupStaleMultipartUploads(globalMultipartCleanupInterval, globalMultipartExpiry, globalServiceDoneCh) + go fsObjects.cleanupStaleMultipartUploads(context.Background(), globalMultipartCleanupInterval, globalMultipartExpiry, globalServiceDoneCh) cacheFS := cacheFSObjects{ FSObjects: fsObjects, @@ -116,7 +117,9 @@ func (cfs *cacheFSObjects) diskUsageLow() bool { minUsage := cfs.maxDiskUsagePct * 80 / 100 di, err := disk.GetInfo(cfs.dir) if err != nil { - errorIf(err, "Error getting disk information on %s", cfs.dir) + reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", cfs.dir) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) return false } usedPercent := (di.Total - di.Free) * 100 / di.Total @@ -128,7 +131,9 @@ func (cfs *cacheFSObjects) diskUsageLow() bool { func (cfs *cacheFSObjects) diskUsageHigh() bool { di, err := disk.GetInfo(cfs.dir) if err != nil { - errorIf(err, "Error getting disk information on %s", cfs.dir) + reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", cfs.dir) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) return true } usedPercent := (di.Total - di.Free) * 100 / di.Total @@ -140,7 +145,9 @@ func (cfs *cacheFSObjects) diskUsageHigh() bool { func (cfs *cacheFSObjects) diskAvailable(size int64) bool { di, err := disk.GetInfo(cfs.dir) if err != nil { - errorIf(err, "Error getting disk information on %s", cfs.dir) + reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", cfs.dir) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) return false } usedPercent := (di.Total - (di.Free - uint64(size))) * 100 / di.Total @@ -163,14 +170,15 @@ func (cfs *cacheFSObjects) purgeTrash() { return } for _, entry := range entries { - fi, err := fsStatVolume(pathJoin(trashPath, entry)) + ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{}) + fi, err := fsStatVolume(ctx, pathJoin(trashPath, entry)) if err != nil { continue } dir := path.Join(trashPath, fi.Name()) // Delete all expired cache content. - fsRemoveAll(dir) + fsRemoveAll(ctx, dir) } } } @@ -193,7 +201,7 @@ func (cfs *cacheFSObjects) purge() { deletedCount := 0 buckets, err := cfs.ListBuckets(ctx) if err != nil { - errorIf(err, "Unable to list buckets.") + logger.LogIf(ctx, err) } // Reset cache online status if drive was offline earlier. if !cfs.IsOnline() { @@ -221,7 +229,7 @@ func (cfs *cacheFSObjects) purge() { continue } if err = cfs.DeleteObject(ctx, bucket.Name, object.Name); err != nil { - errorIf(err, "Unable to remove cache entry in dir %s/%s", bucket.Name, object.Name) + logger.LogIf(ctx, err) continue } deletedCount++ @@ -313,7 +321,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object var err error // Validate if bucket name is valid and exists. - if _, err = fs.statBucketDir(bucket); err != nil { + if _, err = fs.statBucketDir(ctx, bucket); err != nil { return ObjectInfo{}, toObjectErr(err, bucket) } @@ -325,31 +333,32 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object // and return success. if isObjectDir(object, data.Size()) { // Check if an object is present as one of the parent dir. - if fs.parentDirIsObject(bucket, path.Dir(object)) { - return ObjectInfo{}, toObjectErr(errors2.Trace(errFileAccessDenied), bucket, object) + if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) { + return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object) } if err = mkdirAll(pathJoin(fs.fsPath, bucket, object), 0777); err != nil { return ObjectInfo{}, toObjectErr(err, bucket, object) } var fi os.FileInfo - if fi, err = fsStatDir(pathJoin(fs.fsPath, bucket, object)); err != nil { + if fi, err = fsStatDir(ctx, pathJoin(fs.fsPath, bucket, object)); err != nil { return ObjectInfo{}, toObjectErr(err, bucket, object) } return fsMeta.ToObjectInfo(bucket, object, fi), nil } - if err = checkPutObjectArgs(bucket, object, fs, data.Size()); err != nil { + if err = checkPutObjectArgs(ctx, bucket, object, fs, data.Size()); err != nil { return ObjectInfo{}, err } // Check if an object is present as one of the parent dir. - if fs.parentDirIsObject(bucket, path.Dir(object)) { - return ObjectInfo{}, toObjectErr(errors2.Trace(errFileAccessDenied), bucket, object) + if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) { + return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object) } // Validate input data size and it can never be less than zero. if data.Size() < 0 { - return ObjectInfo{}, errors2.Trace(errInvalidArgument) + logger.LogIf(ctx, errInvalidArgument) + return ObjectInfo{}, errInvalidArgument } var wlk *lock.LockedFile @@ -359,7 +368,8 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object wlk, err = fs.rwPool.Create(fsMetaPath) if err != nil { - return ObjectInfo{}, toObjectErr(errors2.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return ObjectInfo{}, toObjectErr(err, bucket, object) } // This close will allow for locks to be synchronized on `fs.json`. defer wlk.Close() @@ -367,7 +377,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object // Remove meta file when PutObject encounters any error if retErr != nil { tmpDir := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID) - fsRemoveMeta(bucketMetaDir, fsMetaPath, tmpDir) + fsRemoveMeta(ctx, bucketMetaDir, fsMetaPath, tmpDir) } }() } @@ -385,10 +395,9 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object buf := make([]byte, int(bufSize)) fsTmpObjPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, tempObj) - bytesWritten, err := fsCreateFile(fsTmpObjPath, data, buf, data.Size()) + bytesWritten, err := fsCreateFile(ctx, fsTmpObjPath, data, buf, data.Size()) if err != nil { - fsRemoveFile(fsTmpObjPath) - errorIf(err, "Failed to create object %s/%s", bucket, object) + fsRemoveFile(ctx, fsTmpObjPath) return ObjectInfo{}, toObjectErr(err, bucket, object) } if fsMeta.Meta["etag"] == "" { @@ -397,18 +406,18 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object // Should return IncompleteBody{} error when reader has fewer // bytes than specified in request header. if bytesWritten < data.Size() { - fsRemoveFile(fsTmpObjPath) - return ObjectInfo{}, errors2.Trace(IncompleteBody{}) + fsRemoveFile(ctx, fsTmpObjPath) + return ObjectInfo{}, IncompleteBody{} } // Delete the temporary object in the case of a // failure. If PutObject succeeds, then there would be // nothing to delete. - defer fsRemoveFile(fsTmpObjPath) + defer fsRemoveFile(ctx, fsTmpObjPath) // Entire object was written to the temp location, now it's safe to rename it to the actual location. fsNSObjPath := pathJoin(fs.fsPath, bucket, object) - if err = fsRenameFile(fsTmpObjPath, fsNSObjPath); err != nil { + if err = fsRenameFile(ctx, fsTmpObjPath, fsNSObjPath); err != nil { return ObjectInfo{}, toObjectErr(err, bucket, object) } @@ -420,7 +429,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object } // Stat the file to fetch timestamp, size. - fi, err := fsStatFile(pathJoin(fs.fsPath, bucket, object)) + fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object)) if err != nil { return ObjectInfo{}, toObjectErr(err, bucket, object) } @@ -447,11 +456,11 @@ func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, objec } } fs := cfs.FSObjects - if err := checkNewMultipartArgs(bucket, object, fs); err != nil { + if err := checkNewMultipartArgs(ctx, bucket, object, fs); err != nil { return "", toObjectErr(err, bucket) } - if _, err := fs.statBucketDir(bucket); err != nil { + if _, err := fs.statBucketDir(ctx, bucket); err != nil { return "", toObjectErr(err, bucket) } @@ -459,7 +468,8 @@ func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, objec err := mkdirAll(uploadIDDir, 0755) if err != nil { - return "", errors2.Trace(err) + logger.LogIf(ctx, err) + return "", err } // Initialize fs.json values. @@ -468,11 +478,13 @@ func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, objec fsMetaBytes, err := json.Marshal(fsMeta) if err != nil { - return "", errors2.Trace(err) + logger.LogIf(ctx, err) + return "", err } if err = ioutil.WriteFile(pathJoin(uploadIDDir, fs.metaJSONFile), fsMetaBytes, 0644); err != nil { - return "", errors2.Trace(err) + logger.LogIf(ctx, err) + return "", err } return uploadID, nil } @@ -485,7 +497,7 @@ func (cfs *cacheFSObjects) moveBucketToTrash(ctx context.Context, bucket string) return err } defer bucketLock.Unlock() - bucketDir, err := fs.getBucketDir(bucket) + bucketDir, err := fs.getBucketDir(ctx, bucket) if err != nil { return toObjectErr(err, bucket) } @@ -493,12 +505,13 @@ func (cfs *cacheFSObjects) moveBucketToTrash(ctx context.Context, bucket string) expiredDir := path.Join(trashPath, bucket) // Attempt to move regular bucket to expired directory. if err = fsRenameDir(bucketDir, expiredDir); err != nil { + logger.LogIf(ctx, err) return toObjectErr(err, bucket) } // Cleanup all the bucket metadata. ominioMetadataBucketDir := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket) nminioMetadataBucketDir := pathJoin(trashPath, MustGetUUID()) - _ = fsRenameDir(ominioMetadataBucketDir, nminioMetadataBucketDir) + logger.LogIf(ctx, fsRenameDir(ominioMetadataBucketDir, nminioMetadataBucketDir)) return nil } @@ -506,22 +519,22 @@ func (cfs *cacheFSObjects) moveBucketToTrash(ctx context.Context, bucket string) // paths for windows automatically. func fsRenameDir(dirPath, newPath string) (err error) { if dirPath == "" || newPath == "" { - return errors2.Trace(errInvalidArgument) + return errInvalidArgument } if err = checkPathLength(dirPath); err != nil { - return errors2.Trace(err) + return err } if err = checkPathLength(newPath); err != nil { - return errors2.Trace(err) + return err } if err = os.Rename(dirPath, newPath); err != nil { if os.IsNotExist(err) { - return errors2.Trace(errVolumeNotFound) + return errVolumeNotFound } else if isSysErrNotEmpty(err) { - return errors2.Trace(errVolumeNotEmpty) + return errVolumeNotEmpty } - return errors2.Trace(err) + return err } return nil } diff --git a/cmd/disk-cache.go b/cmd/disk-cache.go index 4606010f0..5495df569 100644 --- a/cmd/disk-cache.go +++ b/cmd/disk-cache.go @@ -31,6 +31,7 @@ import ( "github.com/djherbis/atime" + "github.com/minio/minio/cmd/logger" errors2 "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/wildcard" @@ -345,12 +346,12 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark if err != nil { return false } - _, err = fs.getObjectInfo(bucket, object) + _, err = fs.getObjectInfo(ctx, bucket, object) return err == nil } listDir := listDirCacheFactory(isLeaf, cacheTreeWalkIgnoredErrs, c.cache.cfs) - walkResultCh = startTreeWalk(bucket, prefix, marker, recursive, listDir, isLeaf, endWalkCh) + walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, endWalkCh) } for i := 0; i < maxKeys; { @@ -383,7 +384,7 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark } return result, toObjectErr(err, bucket, prefix) } - objInfo, err = fs.getObjectInfo(bucket, entry) + objInfo, err = fs.getObjectInfo(ctx, bucket, entry) if err != nil { // Ignore errFileNotFound if errors2.Cause(err) == errFileNotFound { @@ -754,7 +755,8 @@ func (c cacheObjects) StorageInfo(ctx context.Context) (storageInfo StorageInfo) continue } info, err := getDiskInfo((cfs.fsPath)) - errorIf(err, "Unable to get disk info %#v", cfs.fsPath) + logger.GetReqInfo(ctx).AppendTags("cachePath", cfs.fsPath) + logger.LogIf(ctx, err) total += info.Total free += info.Free } @@ -791,7 +793,8 @@ func (c cacheObjects) DeleteBucket(ctx context.Context, bucket string) (err erro // or the global env overrides. func newCache(config CacheConfig) (*diskCache, error) { var cfsObjects []*cacheFSObjects - formats, err := loadAndValidateCacheFormat(config.Drives) + ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{}) + formats, err := loadAndValidateCacheFormat(ctx, config.Drives) if err != nil { return nil, err } diff --git a/cmd/endpoint.go b/cmd/endpoint.go index ce3e8374e..2f060253c 100644 --- a/cmd/endpoint.go +++ b/cmd/endpoint.go @@ -27,6 +27,7 @@ import ( "strings" "github.com/minio/minio-go/pkg/set" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/mountinfo" ) @@ -399,7 +400,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList, } ipList, err := getHostIP4(host) - fatalIf(err, "unexpected error when resolving host '%s'", host) + logger.FatalIf(err, "unexpected error when resolving host '%s'", host) // Filter ipList by IPs those start with '127.'. loopBackIPs := ipList.FuncMatch(func(ip string, matchString string) bool { diff --git a/cmd/erasure-createfile.go b/cmd/erasure-createfile.go index 16675f2cc..5cb2c3546 100644 --- a/cmd/erasure-createfile.go +++ b/cmd/erasure-createfile.go @@ -17,18 +17,20 @@ package cmd import ( + "context" "hash" "io" - "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" ) // CreateFile creates a new bitrot encoded file spread over all available disks. CreateFile will create // the file at the given volume and path. It will read from src until an io.EOF occurs. The given algorithm will // be used to protect the erasure encoded file. -func (s *ErasureStorage) CreateFile(src io.Reader, volume, path string, buffer []byte, algorithm BitrotAlgorithm, writeQuorum int) (f ErasureFileInfo, err error) { +func (s *ErasureStorage) CreateFile(ctx context.Context, src io.Reader, volume, path string, buffer []byte, algorithm BitrotAlgorithm, writeQuorum int) (f ErasureFileInfo, err error) { if !algorithm.Available() { - return f, errors.Trace(errBitrotHashAlgoInvalid) + logger.LogIf(ctx, errBitrotHashAlgoInvalid) + return f, errBitrotHashAlgoInvalid } f.Checksums = make([][]byte, len(s.disks)) hashers := make([]hash.Hash, len(s.disks)) @@ -50,21 +52,22 @@ func (s *ErasureStorage) CreateFile(src io.Reader, volume, path string, buffer [ } blocks = make([][]byte, len(s.disks)) // write empty block } else if err == nil || (n > 0 && err == io.ErrUnexpectedEOF) { - blocks, err = s.ErasureEncode(buffer[:n]) + blocks, err = s.ErasureEncode(ctx, buffer[:n]) if err != nil { return f, err } } else { - return f, errors.Trace(err) + logger.LogIf(ctx, err) + return f, err } for i := range errChans { // span workers - go erasureAppendFile(s.disks[i], volume, path, hashers[i], blocks[i], errChans[i]) + go erasureAppendFile(ctx, s.disks[i], volume, path, hashers[i], blocks[i], errChans[i]) } for i := range errChans { // wait until all workers are finished errs[i] = <-errChans[i] } - if err = reduceWriteQuorumErrs(errs, objectOpIgnoredErrs, writeQuorum); err != nil { + if err = reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum); err != nil { return f, err } s.disks = evalDisks(s.disks, errs) @@ -83,9 +86,10 @@ func (s *ErasureStorage) CreateFile(src io.Reader, volume, path string, buffer [ // erasureAppendFile appends the content of buf to the file on the given disk and updates computes // the hash of the written data. It sends the write error (or nil) over the error channel. -func erasureAppendFile(disk StorageAPI, volume, path string, hash hash.Hash, buf []byte, errChan chan<- error) { +func erasureAppendFile(ctx context.Context, disk StorageAPI, volume, path string, hash hash.Hash, buf []byte, errChan chan<- error) { if disk == OfflineDisk { - errChan <- errors.Trace(errDiskNotFound) + logger.LogIf(ctx, errDiskNotFound) + errChan <- errDiskNotFound return } err := disk.AppendFile(volume, path, buf) diff --git a/cmd/erasure-createfile_test.go b/cmd/erasure-createfile_test.go index 430ed6ca3..4cd6dd902 100644 --- a/cmd/erasure-createfile_test.go +++ b/cmd/erasure-createfile_test.go @@ -18,6 +18,7 @@ package cmd import ( "bytes" + "context" "crypto/rand" "io" "testing" @@ -70,7 +71,7 @@ func TestErasureCreateFile(t *testing.T) { if err != nil { t.Fatalf("Test %d: failed to create test setup: %v", i, err) } - storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize) + storage, err := NewErasureStorage(context.Background(), setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize) if err != nil { setup.Remove() t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err) @@ -82,7 +83,7 @@ func TestErasureCreateFile(t *testing.T) { setup.Remove() t.Fatalf("Test %d: failed to generate random test data: %v", i, err) } - file, err := storage.CreateFile(bytes.NewReader(data[test.offset:]), "testbucket", "object", buffer, test.algorithm, test.dataBlocks+1) + file, err := storage.CreateFile(context.Background(), bytes.NewReader(data[test.offset:]), "testbucket", "object", buffer, test.algorithm, test.dataBlocks+1) if err != nil && !test.shouldFail { t.Errorf("Test %d: should pass but failed with: %v", i, err) } @@ -100,7 +101,7 @@ func TestErasureCreateFile(t *testing.T) { if test.offDisks > 0 { storage.disks[0] = OfflineDisk } - file, err = storage.CreateFile(bytes.NewReader(data[test.offset:]), "testbucket", "object2", buffer, test.algorithm, test.dataBlocks+1) + file, err = storage.CreateFile(context.Background(), bytes.NewReader(data[test.offset:]), "testbucket", "object2", buffer, test.algorithm, test.dataBlocks+1) if err != nil && !test.shouldFailQuorum { t.Errorf("Test %d: should pass but failed with: %v", i, err) } @@ -125,7 +126,7 @@ func benchmarkErasureWrite(data, parity, dataDown, parityDown int, size int64, b b.Fatalf("failed to create test setup: %v", err) } defer setup.Remove() - storage, err := NewErasureStorage(setup.disks, data, parity, blockSizeV1) + storage, err := NewErasureStorage(context.Background(), setup.disks, data, parity, blockSizeV1) if err != nil { b.Fatalf("failed to create ErasureStorage: %v", err) } @@ -143,7 +144,7 @@ func benchmarkErasureWrite(data, parity, dataDown, parityDown int, size int64, b b.SetBytes(size) b.ReportAllocs() for i := 0; i < b.N; i++ { - _, err := storage.CreateFile(bytes.NewReader(content), "testbucket", "object", buffer, DefaultBitrotAlgorithm, data+1) + _, err := storage.CreateFile(context.Background(), bytes.NewReader(content), "testbucket", "object", buffer, DefaultBitrotAlgorithm, data+1) if err != nil { panic(err) } diff --git a/cmd/erasure-healfile.go b/cmd/erasure-healfile.go index b12f5655c..bb3980bce 100644 --- a/cmd/erasure-healfile.go +++ b/cmd/erasure-healfile.go @@ -17,11 +17,12 @@ package cmd import ( + "context" "fmt" "hash" "strings" - "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" ) // HealFile tries to reconstruct an erasure-coded file spread over all @@ -45,12 +46,13 @@ import ( // // It returns bitrot checksums for the non-nil staleDisks on which // healing succeeded. -func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, blocksize int64, +func (s ErasureStorage) HealFile(ctx context.Context, staleDisks []StorageAPI, volume, path string, blocksize int64, dstVol, dstPath string, size int64, alg BitrotAlgorithm, checksums [][]byte) ( f ErasureFileInfo, err error) { if !alg.Available() { - return f, errors.Trace(errBitrotHashAlgoInvalid) + logger.LogIf(ctx, errBitrotHashAlgoInvalid) + return f, errBitrotHashAlgoInvalid } // Initialization @@ -84,7 +86,7 @@ func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, b } readLen += lastChunkSize var buffers [][]byte - buffers, _, err = s.readConcurrent(volume, path, 0, readLen, verifiers) + buffers, _, err = s.readConcurrent(ctx, volume, path, 0, readLen, verifiers) if err != nil { return f, err } @@ -131,7 +133,7 @@ func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, b } buffOffset += csize - if err = s.ErasureDecodeDataAndParityBlocks(blocks); err != nil { + if err = s.ErasureDecodeDataAndParityBlocks(ctx, blocks); err != nil { return f, err } @@ -155,7 +157,9 @@ func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, b // If all disks had write errors we quit. if !writeSucceeded { // build error from all write errors - return f, errors.Trace(joinWriteErrors(writeErrors)) + err := joinWriteErrors(writeErrors) + logger.LogIf(ctx, err) + return f, err } } diff --git a/cmd/erasure-healfile_test.go b/cmd/erasure-healfile_test.go index 5fc1721e8..5206aa867 100644 --- a/cmd/erasure-healfile_test.go +++ b/cmd/erasure-healfile_test.go @@ -18,6 +18,7 @@ package cmd import ( "bytes" + "context" "crypto/rand" "io" "reflect" @@ -74,7 +75,7 @@ func TestErasureHealFile(t *testing.T) { if err != nil { t.Fatalf("Test %d: failed to setup XL environment: %v", i, err) } - storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.disks-test.dataBlocks, test.blocksize) + storage, err := NewErasureStorage(context.Background(), setup.disks, test.dataBlocks, test.disks-test.dataBlocks, test.blocksize) if err != nil { setup.Remove() t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err) @@ -89,7 +90,7 @@ func TestErasureHealFile(t *testing.T) { algorithm = DefaultBitrotAlgorithm } buffer := make([]byte, test.blocksize, 2*test.blocksize) - file, err := storage.CreateFile(bytes.NewReader(data), "testbucket", "testobject", buffer, algorithm, test.dataBlocks+1) + file, err := storage.CreateFile(context.Background(), bytes.NewReader(data), "testbucket", "testobject", buffer, algorithm, test.dataBlocks+1) if err != nil { setup.Remove() t.Fatalf("Test %d: failed to create random test data: %v", i, err) @@ -113,7 +114,7 @@ func TestErasureHealFile(t *testing.T) { } // test case setup is complete - now call Healfile() - info, err := storage.HealFile(staleDisks, "testbucket", "testobject", test.blocksize, "testbucket", "healedobject", test.size, test.algorithm, file.Checksums) + info, err := storage.HealFile(context.Background(), staleDisks, "testbucket", "testobject", test.blocksize, "testbucket", "healedobject", test.size, test.algorithm, file.Checksums) if err != nil && !test.shouldFail { t.Errorf("Test %d: should pass but it failed with: %v", i, err) } diff --git a/cmd/erasure-readfile.go b/cmd/erasure-readfile.go index 326ea148c..16e5943a4 100644 --- a/cmd/erasure-readfile.go +++ b/cmd/erasure-readfile.go @@ -17,9 +17,10 @@ package cmd import ( + "context" "io" - "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" ) type errIdx struct { @@ -27,7 +28,7 @@ type errIdx struct { err error } -func (s ErasureStorage) readConcurrent(volume, path string, offset, length int64, +func (s ErasureStorage) readConcurrent(ctx context.Context, volume, path string, offset, length int64, verifiers []*BitrotVerifier) (buffers [][]byte, needsReconstruction bool, err error) { @@ -39,7 +40,8 @@ func (s ErasureStorage) readConcurrent(volume, path string, offset, length int64 stageBuffers[i] = make([]byte, length) disk := s.disks[i] if disk == OfflineDisk { - errChan <- errIdx{i, errors.Trace(errDiskNotFound)} + logger.LogIf(ctx, errDiskNotFound) + errChan <- errIdx{i, errDiskNotFound} return } _, rerr := disk.ReadFile(volume, path, offset, stageBuffers[i], verifiers[i]) @@ -75,7 +77,8 @@ func (s ErasureStorage) readConcurrent(volume, path string, offset, length int64 } if successCount != s.dataBlocks { // Not enough disks returns data. - err = errors.Trace(errXLReadQuorum) + err = errXLReadQuorum + logger.LogIf(ctx, err) } return } @@ -86,18 +89,21 @@ func (s ErasureStorage) readConcurrent(volume, path string, offset, length int64 // integrity of the given file. ReadFile will read data from the given // offset up to the given length. If parts of the file are corrupted // ReadFile tries to reconstruct the data. -func (s ErasureStorage) ReadFile(writer io.Writer, volume, path string, offset, +func (s ErasureStorage) ReadFile(ctx context.Context, writer io.Writer, volume, path string, offset, length, totalLength int64, checksums [][]byte, algorithm BitrotAlgorithm, blocksize int64) (f ErasureFileInfo, err error) { if offset < 0 || length < 0 { - return f, errors.Trace(errUnexpected) + logger.LogIf(ctx, errUnexpected) + return f, errUnexpected } if offset+length > totalLength { - return f, errors.Trace(errUnexpected) + logger.LogIf(ctx, errUnexpected) + return f, errUnexpected } if !algorithm.Available() { - return f, errors.Trace(errBitrotHashAlgoInvalid) + logger.LogIf(ctx, errBitrotHashAlgoInvalid) + return f, errBitrotHashAlgoInvalid } f.Checksums = make([][]byte, len(s.disks)) @@ -145,7 +151,7 @@ func (s ErasureStorage) ReadFile(writer io.Writer, volume, path string, offset, var buffers [][]byte var needsReconstruction bool - buffers, needsReconstruction, err = s.readConcurrent(volume, path, + buffers, needsReconstruction, err = s.readConcurrent(ctx, volume, path, partDataStartIndex, partDataLength, verifiers) if err != nil { // Could not read enough disks. @@ -194,7 +200,8 @@ func (s ErasureStorage) ReadFile(writer io.Writer, volume, path string, offset, if needsReconstruction { if err = s.ErasureDecodeDataBlocks(blocks); err != nil { - return f, errors.Trace(err) + logger.LogIf(ctx, err) + return f, err } } @@ -210,7 +217,7 @@ func (s ErasureStorage) ReadFile(writer io.Writer, volume, path string, offset, writeLength = lastBlockLength - writeStart } } - n, err := writeDataBlocks(writer, blocks, s.dataBlocks, writeStart, writeLength) + n, err := writeDataBlocks(ctx, writer, blocks, s.dataBlocks, writeStart, writeLength) if err != nil { return f, err } diff --git a/cmd/erasure-readfile_test.go b/cmd/erasure-readfile_test.go index 7204a948c..db164a93a 100644 --- a/cmd/erasure-readfile_test.go +++ b/cmd/erasure-readfile_test.go @@ -18,6 +18,7 @@ package cmd import ( "bytes" + "context" crand "crypto/rand" "io" "math/rand" @@ -86,7 +87,7 @@ func TestErasureReadFile(t *testing.T) { if err != nil { t.Fatalf("Test %d: failed to create test setup: %v", i, err) } - storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize) + storage, err := NewErasureStorage(context.Background(), setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize) if err != nil { setup.Remove() t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err) @@ -102,13 +103,13 @@ func TestErasureReadFile(t *testing.T) { writeAlgorithm = DefaultBitrotAlgorithm } buffer := make([]byte, test.blocksize, 2*test.blocksize) - file, err := storage.CreateFile(bytes.NewReader(data[:]), "testbucket", "object", buffer, writeAlgorithm, test.dataBlocks+1) + file, err := storage.CreateFile(context.Background(), bytes.NewReader(data[:]), "testbucket", "object", buffer, writeAlgorithm, test.dataBlocks+1) if err != nil { setup.Remove() t.Fatalf("Test %d: failed to create erasure test file: %v", i, err) } writer := bytes.NewBuffer(nil) - readInfo, err := storage.ReadFile(writer, "testbucket", "object", test.offset, test.length, test.data, file.Checksums, test.algorithm, test.blocksize) + readInfo, err := storage.ReadFile(context.Background(), writer, "testbucket", "object", test.offset, test.length, test.data, file.Checksums, test.algorithm, test.blocksize) if err != nil && !test.shouldFail { t.Errorf("Test %d: should pass but failed with: %v", i, err) } @@ -134,7 +135,7 @@ func TestErasureReadFile(t *testing.T) { if test.offDisks > 0 { storage.disks[0] = OfflineDisk } - readInfo, err = storage.ReadFile(writer, "testbucket", "object", test.offset, test.length, test.data, file.Checksums, test.algorithm, test.blocksize) + readInfo, err = storage.ReadFile(context.Background(), writer, "testbucket", "object", test.offset, test.length, test.data, file.Checksums, test.algorithm, test.blocksize) if err != nil && !test.shouldFailQuorum { t.Errorf("Test %d: should pass but failed with: %v", i, err) } @@ -174,7 +175,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) { } defer setup.Remove() - storage, err := NewErasureStorage(setup.disks, dataBlocks, parityBlocks, blockSize) + storage, err := NewErasureStorage(context.Background(), setup.disks, dataBlocks, parityBlocks, blockSize) if err != nil { t.Fatalf("failed to create ErasureStorage: %v", err) } @@ -191,7 +192,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) { // Create a test file to read from. buffer := make([]byte, blockSize, 2*blockSize) - file, err := storage.CreateFile(bytes.NewReader(data), "testbucket", "testobject", buffer, DefaultBitrotAlgorithm, dataBlocks+1) + file, err := storage.CreateFile(context.Background(), bytes.NewReader(data), "testbucket", "testobject", buffer, DefaultBitrotAlgorithm, dataBlocks+1) if err != nil { t.Fatal(err) } @@ -211,7 +212,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) { expected := data[offset : offset+readLen] - _, err = storage.ReadFile(buf, "testbucket", "testobject", offset, readLen, length, file.Checksums, DefaultBitrotAlgorithm, blockSize) + _, err = storage.ReadFile(context.Background(), buf, "testbucket", "testobject", offset, readLen, length, file.Checksums, DefaultBitrotAlgorithm, blockSize) if err != nil { t.Fatal(err, offset, readLen) } @@ -231,14 +232,14 @@ func benchmarkErasureRead(data, parity, dataDown, parityDown int, size int64, b b.Fatalf("failed to create test setup: %v", err) } defer setup.Remove() - storage, err := NewErasureStorage(setup.disks, data, parity, blockSizeV1) + storage, err := NewErasureStorage(context.Background(), setup.disks, data, parity, blockSizeV1) if err != nil { b.Fatalf("failed to create ErasureStorage: %v", err) } content := make([]byte, size) buffer := make([]byte, blockSizeV1, 2*blockSizeV1) - file, err := storage.CreateFile(bytes.NewReader(content), "testbucket", "object", buffer, DefaultBitrotAlgorithm, data+1) + file, err := storage.CreateFile(context.Background(), bytes.NewReader(content), "testbucket", "object", buffer, DefaultBitrotAlgorithm, data+1) if err != nil { b.Fatalf("failed to create erasure test file: %v", err) } @@ -255,7 +256,7 @@ func benchmarkErasureRead(data, parity, dataDown, parityDown int, size int64, b b.SetBytes(size) b.ReportAllocs() for i := 0; i < b.N; i++ { - if file, err = storage.ReadFile(bytes.NewBuffer(content[:0]), "testbucket", "object", 0, size, size, checksums, DefaultBitrotAlgorithm, blockSizeV1); err != nil { + if file, err = storage.ReadFile(context.Background(), bytes.NewBuffer(content[:0]), "testbucket", "object", 0, size, size, checksums, DefaultBitrotAlgorithm, blockSizeV1); err != nil { panic(err) } } diff --git a/cmd/erasure-utils.go b/cmd/erasure-utils.go index a6346e9d5..31544debd 100644 --- a/cmd/erasure-utils.go +++ b/cmd/erasure-utils.go @@ -18,10 +18,11 @@ package cmd import ( "bytes" + "context" "io" "github.com/klauspost/reedsolomon" - "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" ) // getDataBlockLen - get length of data blocks from encoded blocks. @@ -36,20 +37,23 @@ func getDataBlockLen(enBlocks [][]byte, dataBlocks int) int { // Writes all the data blocks from encoded blocks until requested // outSize length. Provides a way to skip bytes until the offset. -func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset int64, length int64) (int64, error) { +func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, dataBlocks int, offset int64, length int64) (int64, error) { // Offset and out size cannot be negative. if offset < 0 || length < 0 { - return 0, errors.Trace(errUnexpected) + logger.LogIf(ctx, errUnexpected) + return 0, errUnexpected } // Do we have enough blocks? if len(enBlocks) < dataBlocks { - return 0, errors.Trace(reedsolomon.ErrTooFewShards) + logger.LogIf(ctx, reedsolomon.ErrTooFewShards) + return 0, reedsolomon.ErrTooFewShards } // Do we have enough data? if int64(getDataBlockLen(enBlocks, dataBlocks)) < length { - return 0, errors.Trace(reedsolomon.ErrShortData) + logger.LogIf(ctx, reedsolomon.ErrShortData) + return 0, reedsolomon.ErrShortData } // Counter to decrement total left to write. @@ -77,7 +81,8 @@ func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset in if write < int64(len(block)) { n, err := io.Copy(dst, bytes.NewReader(block[:write])) if err != nil { - return 0, errors.Trace(err) + logger.LogIf(ctx, err) + return 0, err } totalWritten += n break @@ -85,7 +90,8 @@ func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset in // Copy the block. n, err := io.Copy(dst, bytes.NewReader(block)) if err != nil { - return 0, errors.Trace(err) + logger.LogIf(ctx, err) + return 0, err } // Decrement output size. diff --git a/cmd/erasure.go b/cmd/erasure.go index 3e0b95f65..cc74e50ba 100644 --- a/cmd/erasure.go +++ b/cmd/erasure.go @@ -17,11 +17,12 @@ package cmd import ( + "context" "crypto/subtle" "hash" "github.com/klauspost/reedsolomon" - "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" ) // OfflineDisk represents an unavailable disk. @@ -44,11 +45,12 @@ type ErasureStorage struct { // NewErasureStorage creates a new ErasureStorage. The storage erasure codes and protects all data written to // the disks. -func NewErasureStorage(disks []StorageAPI, dataBlocks, parityBlocks int, blockSize int64) (s ErasureStorage, err error) { +func NewErasureStorage(ctx context.Context, disks []StorageAPI, dataBlocks, parityBlocks int, blockSize int64) (s ErasureStorage, err error) { shardsize := (int(blockSize) + dataBlocks - 1) / dataBlocks erasure, err := reedsolomon.New(dataBlocks, parityBlocks, reedsolomon.WithAutoGoroutines(shardsize)) if err != nil { - return s, errors.Tracef("failed to create erasure coding: %v", err) + logger.LogIf(ctx, err) + return s, err } s = ErasureStorage{ disks: make([]StorageAPI, len(disks)), @@ -62,13 +64,15 @@ func NewErasureStorage(disks []StorageAPI, dataBlocks, parityBlocks int, blockSi // ErasureEncode encodes the given data and returns the erasure-coded data. // It returns an error if the erasure coding failed. -func (s *ErasureStorage) ErasureEncode(data []byte) ([][]byte, error) { +func (s *ErasureStorage) ErasureEncode(ctx context.Context, data []byte) ([][]byte, error) { encoded, err := s.erasure.Split(data) if err != nil { - return nil, errors.Tracef("failed to split data: %v", err) + logger.LogIf(ctx, err) + return nil, err } if err = s.erasure.Encode(encoded); err != nil { - return nil, errors.Tracef("failed to encode data: %v", err) + logger.LogIf(ctx, err) + return nil, err } return encoded, nil } @@ -78,16 +82,17 @@ func (s *ErasureStorage) ErasureEncode(data []byte) ([][]byte, error) { // It returns an error if the decoding failed. func (s *ErasureStorage) ErasureDecodeDataBlocks(data [][]byte) error { if err := s.erasure.ReconstructData(data); err != nil { - return errors.Tracef("failed to reconstruct data: %v", err) + return err } return nil } // ErasureDecodeDataAndParityBlocks decodes the given erasure-coded data and verifies it. // It returns an error if the decoding failed. -func (s *ErasureStorage) ErasureDecodeDataAndParityBlocks(data [][]byte) error { +func (s *ErasureStorage) ErasureDecodeDataAndParityBlocks(ctx context.Context, data [][]byte) error { if err := s.erasure.Reconstruct(data); err != nil { - return errors.Tracef("failed to reconstruct data: %v", err) + logger.LogIf(ctx, err) + return err } return nil } diff --git a/cmd/erasure_test.go b/cmd/erasure_test.go index 44cc4fb74..cef78f5d6 100644 --- a/cmd/erasure_test.go +++ b/cmd/erasure_test.go @@ -18,6 +18,7 @@ package cmd import ( "bytes" + "context" "crypto/rand" "io" "os" @@ -52,11 +53,11 @@ func TestErasureDecode(t *testing.T) { copy(buffer, data) disks := make([]StorageAPI, test.dataBlocks+test.parityBlocks) - storage, err := NewErasureStorage(disks, test.dataBlocks, test.parityBlocks, blockSizeV1) + storage, err := NewErasureStorage(context.Background(), disks, test.dataBlocks, test.parityBlocks, blockSizeV1) if err != nil { t.Fatalf("Test %d: failed to create erasure storage: %v", i, err) } - encoded, err := storage.ErasureEncode(buffer) + encoded, err := storage.ErasureEncode(context.Background(), buffer) if err != nil { t.Fatalf("Test %d: failed to encode data: %v", i, err) } @@ -69,7 +70,7 @@ func TestErasureDecode(t *testing.T) { } if test.reconstructParity { - err = storage.ErasureDecodeDataAndParityBlocks(encoded) + err = storage.ErasureDecodeDataAndParityBlocks(context.Background(), encoded) } else { err = storage.ErasureDecodeDataBlocks(encoded) } @@ -98,7 +99,7 @@ func TestErasureDecode(t *testing.T) { } decodedData := new(bytes.Buffer) - if _, err = writeDataBlocks(decodedData, decoded, test.dataBlocks, 0, int64(len(data))); err != nil { + if _, err = writeDataBlocks(context.Background(), decodedData, decoded, test.dataBlocks, 0, int64(len(data))); err != nil { t.Errorf("Test %d: failed to write data blocks: %v", i, err) } if !bytes.Equal(decodedData.Bytes(), data) { diff --git a/cmd/format-disk-cache.go b/cmd/format-disk-cache.go index eac20b701..488073d77 100644 --- a/cmd/format-disk-cache.go +++ b/cmd/format-disk-cache.go @@ -17,13 +17,14 @@ package cmd import ( + "context" "errors" "fmt" "io" "os" "reflect" - errors2 "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" ) const ( @@ -100,14 +101,14 @@ func createFormatCache(fsFormatPath string, format *formatCacheV1) error { // open file using READ & WRITE permission var file, err = os.OpenFile(fsFormatPath, os.O_RDWR|os.O_CREATE, 0600) if err != nil { - return errors2.Trace(err) + return err } // Close the locked file upon return. defer file.Close() fi, err := file.Stat() if err != nil { - return errors2.Trace(err) + return err } if fi.Size() != 0 { // format.json already got created because of another minio process's createFormatCache() @@ -118,7 +119,7 @@ func createFormatCache(fsFormatPath string, format *formatCacheV1) error { // This function creates a cache format file on disk and returns a slice // of format cache config -func initFormatCache(drives []string) (formats []*formatCacheV1, err error) { +func initFormatCache(ctx context.Context, drives []string) (formats []*formatCacheV1, err error) { nformats := newFormatCacheV1(drives) for _, drive := range drives { _, err = os.Stat(drive) @@ -126,28 +127,36 @@ func initFormatCache(drives []string) (formats []*formatCacheV1, err error) { continue } if !os.IsNotExist(err) { + logger.GetReqInfo(ctx).AppendTags("drive", drive) + logger.LogIf(ctx, err) return nil, err } if err = os.Mkdir(drive, 0777); err != nil { + logger.GetReqInfo(ctx).AppendTags("drive", drive) + logger.LogIf(ctx, err) return nil, err } } for i, drive := range drives { if err = os.Mkdir(pathJoin(drive, minioMetaBucket), 0777); err != nil { if !os.IsExist(err) { + logger.GetReqInfo(ctx).AppendTags("drive", drive) + logger.LogIf(ctx, err) return nil, err } } cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile) // Fresh disk - create format.json for this cfs if err = createFormatCache(cacheFormatPath, nformats[i]); err != nil { + logger.GetReqInfo(ctx).AppendTags("drive", drive) + logger.LogIf(ctx, err) return nil, err } } return nformats, nil } -func loadFormatCache(drives []string) ([]*formatCacheV1, error) { +func loadFormatCache(ctx context.Context, drives []string) ([]*formatCacheV1, error) { formats := make([]*formatCacheV1, len(drives)) for i, drive := range drives { cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile) @@ -156,6 +165,7 @@ func loadFormatCache(drives []string) ([]*formatCacheV1, error) { if os.IsNotExist(err) { continue } + logger.LogIf(ctx, err) return nil, err } defer f.Close() @@ -268,7 +278,7 @@ func findCacheDiskIndex(disk string, disks []string) int { } // validate whether cache drives order has changed -func validateCacheFormats(formats []*formatCacheV1) error { +func validateCacheFormats(ctx context.Context, formats []*formatCacheV1) error { count := 0 for _, format := range formats { if format == nil { @@ -279,12 +289,16 @@ func validateCacheFormats(formats []*formatCacheV1) error { return errors.New("Cache format files missing on all drives") } if _, err := checkFormatCacheValues(formats); err != nil { + logger.LogIf(ctx, err) return err } if err := checkCacheDisksSliceConsistency(formats); err != nil { + logger.LogIf(ctx, err) return err } - return checkCacheDiskConsistency(formats) + err := checkCacheDiskConsistency(formats) + logger.LogIf(ctx, err) + return err } // return true if all of the list of cache drives are @@ -303,16 +317,16 @@ func cacheDrivesUnformatted(drives []string) bool { // create format.json for each cache drive if fresh disk or load format from disk // Then validate the format for all drives in the cache to ensure order // of cache drives has not changed. -func loadAndValidateCacheFormat(drives []string) (formats []*formatCacheV1, err error) { +func loadAndValidateCacheFormat(ctx context.Context, drives []string) (formats []*formatCacheV1, err error) { if cacheDrivesUnformatted(drives) { - formats, err = initFormatCache(drives) + formats, err = initFormatCache(ctx, drives) } else { - formats, err = loadFormatCache(drives) + formats, err = loadFormatCache(ctx, drives) } if err != nil { return nil, err } - if err = validateCacheFormats(formats); err != nil { + if err = validateCacheFormats(ctx, formats); err != nil { return nil, err } return formats, nil diff --git a/cmd/format-disk-cache_test.go b/cmd/format-disk-cache_test.go index b108d0c7d..eebadee47 100644 --- a/cmd/format-disk-cache_test.go +++ b/cmd/format-disk-cache_test.go @@ -17,18 +17,20 @@ package cmd import ( + "context" "os" "testing" ) // TestDiskCacheFormat - tests initFormatCache, formatMetaGetFormatBackendCache, formatCacheGetVersion. func TestDiskCacheFormat(t *testing.T) { + ctx := context.Background() fsDirs, err := getRandomDisks(1) if err != nil { t.Fatal(err) } - _, err = initFormatCache(fsDirs) + _, err = initFormatCache(ctx, fsDirs) if err != nil { t.Fatal(err) } @@ -56,7 +58,7 @@ func TestDiskCacheFormat(t *testing.T) { t.Fatal(err) } - if _, err = loadAndValidateCacheFormat(fsDirs); err == nil { + if _, err = loadAndValidateCacheFormat(context.Background(), fsDirs); err == nil { t.Fatal("expected to fail") } @@ -69,7 +71,7 @@ func TestDiskCacheFormat(t *testing.T) { t.Fatal(err) } - if _, err = loadAndValidateCacheFormat(fsDirs); err == nil { + if _, err = loadAndValidateCacheFormat(context.Background(), fsDirs); err == nil { t.Fatal("expected to fail") } } @@ -307,7 +309,7 @@ func TestFormatCache(t *testing.T) { } for i, testCase := range testCases { - err := validateCacheFormats(testCase.formatConfigs) + err := validateCacheFormats(context.Background(), testCase.formatConfigs) if err != nil && testCase.shouldPass { t.Errorf("Test %d: Expected to pass but failed with %s", i+1, err) } diff --git a/cmd/format-fs.go b/cmd/format-fs.go index c029460c5..68f769b5a 100644 --- a/cmd/format-fs.go +++ b/cmd/format-fs.go @@ -17,13 +17,14 @@ package cmd import ( + "context" "fmt" "io" "os" "path" "time" - errors2 "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/lock" ) @@ -97,7 +98,7 @@ func formatFSGetVersion(r io.ReadSeeker) (string, error) { // Migrate from V1 to V2. V2 implements new backend format for multipart // uploads. Delete the previous multipart directory. -func formatFSMigrateV1ToV2(wlk *lock.LockedFile, fsPath string) error { +func formatFSMigrateV1ToV2(ctx context.Context, wlk *lock.LockedFile, fsPath string) error { version, err := formatFSGetVersion(wlk) if err != nil { return err @@ -107,11 +108,12 @@ func formatFSMigrateV1ToV2(wlk *lock.LockedFile, fsPath string) error { return fmt.Errorf(`format.json version expected %s, found %s`, formatFSVersionV1, version) } - if err = fsRemoveAll(path.Join(fsPath, minioMetaMultipartBucket)); err != nil { + if err = fsRemoveAll(ctx, path.Join(fsPath, minioMetaMultipartBucket)); err != nil { return err } if err = os.MkdirAll(path.Join(fsPath, minioMetaMultipartBucket), 0755); err != nil { + logger.LogIf(ctx, err) return err } @@ -122,7 +124,7 @@ func formatFSMigrateV1ToV2(wlk *lock.LockedFile, fsPath string) error { // Migration should happen when formatFSV1.FS.Version changes. This version // can change when there is a change to the struct formatFSV1.FS or if there // is any change in the backend file system tree structure. -func formatFSMigrate(wlk *lock.LockedFile, fsPath string) error { +func formatFSMigrate(ctx context.Context, wlk *lock.LockedFile, fsPath string) error { // Add any migration code here in case we bump format.FS.Version version, err := formatFSGetVersion(wlk) if err != nil { @@ -131,7 +133,7 @@ func formatFSMigrate(wlk *lock.LockedFile, fsPath string) error { switch version { case formatFSVersionV1: - if err = formatFSMigrateV1ToV2(wlk, fsPath); err != nil { + if err = formatFSMigrateV1ToV2(ctx, wlk, fsPath); err != nil { return err } fallthrough @@ -151,19 +153,21 @@ func formatFSMigrate(wlk *lock.LockedFile, fsPath string) error { } // Creates a new format.json if unformatted. -func createFormatFS(fsFormatPath string) error { +func createFormatFS(ctx context.Context, fsFormatPath string) error { // Attempt a write lock on formatConfigFile `format.json` // file stored in minioMetaBucket(.minio.sys) directory. lk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR|os.O_CREATE, 0600) if err != nil { - return errors2.Trace(err) + logger.LogIf(ctx, err) + return err } // Close the locked file upon return. defer lk.Close() fi, err := lk.Stat() if err != nil { - return errors2.Trace(err) + logger.LogIf(ctx, err) + return err } if fi.Size() != 0 { // format.json already got created because of another minio process's createFormatFS() @@ -177,7 +181,7 @@ func createFormatFS(fsFormatPath string) error { // The file descriptor should be kept open throughout the life // of the process so that another minio process does not try to // migrate the backend when we are actively working on the backend. -func initFormatFS(fsPath string) (rlk *lock.RLockedFile, err error) { +func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, err error) { fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile) // Any read on format.json should be done with read-lock. // Any write on format.json should be done with write-lock. @@ -191,7 +195,8 @@ func initFormatFS(fsPath string) (rlk *lock.RLockedFile, err error) { var fi os.FileInfo fi, err = rlk.Stat() if err != nil { - return nil, errors2.Trace(err) + logger.LogIf(ctx, err) + return nil, err } isEmpty = fi.Size() == 0 } @@ -200,7 +205,7 @@ func initFormatFS(fsPath string) (rlk *lock.RLockedFile, err error) { rlk.Close() } // Fresh disk - create format.json - err = createFormatFS(fsFormatPath) + err = createFormatFS(ctx, fsFormatPath) if err == lock.ErrAlreadyLocked { // Lock already present, sleep and attempt again. // Can happen in a rare situation when a parallel minio process @@ -209,19 +214,22 @@ func initFormatFS(fsPath string) (rlk *lock.RLockedFile, err error) { continue } if err != nil { - return nil, errors2.Trace(err) + logger.LogIf(ctx, err) + return nil, err } // After successfully creating format.json try to hold a read-lock on // the file. continue } if err != nil { - return nil, errors2.Trace(err) + logger.LogIf(ctx, err) + return nil, err } formatBackend, err := formatMetaGetFormatBackendFS(rlk) if err != nil { - return nil, errors2.Trace(err) + logger.LogIf(ctx, err) + return nil, err } if formatBackend != formatBackendFS { return nil, fmt.Errorf(`%s file: expected format-type: %s, found: %s`, formatConfigFile, formatBackendFS, formatBackend) @@ -244,7 +252,7 @@ func initFormatFS(fsPath string) (rlk *lock.RLockedFile, err error) { if err != nil { return nil, err } - err = formatFSMigrate(wlk, fsPath) + err = formatFSMigrate(ctx, wlk, fsPath) wlk.Close() if err != nil { // Migration failed, bail out so that the user can observe what happened. diff --git a/cmd/format-fs_test.go b/cmd/format-fs_test.go index e0d28c382..66d6bcaa1 100644 --- a/cmd/format-fs_test.go +++ b/cmd/format-fs_test.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "os" "path/filepath" "testing" @@ -38,7 +39,7 @@ func TestFSFormatFS(t *testing.T) { t.Fatal(err) } - rlk, err := initFormatFS(disk) + rlk, err := initFormatFS(context.Background(), disk) if err != nil { t.Fatal(err) } @@ -81,7 +82,7 @@ func TestFSFormatFS(t *testing.T) { if _, err = formatFSGetVersion(rlk); err == nil { t.Fatal("expected to fail") } - if _, err = initFormatFS(disk); err == nil { + if _, err = initFormatFS(context.Background(), disk); err == nil { t.Fatal("expected to fail") } @@ -96,7 +97,7 @@ func TestFSFormatFS(t *testing.T) { if _, err = formatMetaGetFormatBackendFS(f); err == nil { t.Fatal("expected to fail") } - if _, err = initFormatFS(disk); err == nil { + if _, err = initFormatFS(context.Background(), disk); err == nil { t.Fatal("expected to fail") } } diff --git a/cmd/format-xl.go b/cmd/format-xl.go index 301a36245..f645c7952 100644 --- a/cmd/format-xl.go +++ b/cmd/format-xl.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "encoding/json" "fmt" "io/ioutil" @@ -520,7 +521,7 @@ func formatXLV3Check(reference *formatXLV3, format *formatXLV3) error { } // saveFormatXLAll - populates `format.json` on disks in its order. -func saveFormatXLAll(storageDisks []StorageAPI, formats []*formatXLV3) error { +func saveFormatXLAll(ctx context.Context, storageDisks []StorageAPI, formats []*formatXLV3) error { var errs = make([]error, len(storageDisks)) var wg = &sync.WaitGroup{} @@ -542,7 +543,7 @@ func saveFormatXLAll(storageDisks []StorageAPI, formats []*formatXLV3) error { wg.Wait() writeQuorum := len(storageDisks)/2 + 1 - return reduceWriteQuorumErrs(errs, nil, writeQuorum) + return reduceWriteQuorumErrs(ctx, errs, nil, writeQuorum) } // relinquishes the underlying connection for all storage disks. @@ -614,7 +615,7 @@ func fixFormatXLV3(storageDisks []StorageAPI, endpoints EndpointList, formats [] } // initFormatXL - save XL format configuration on all disks. -func initFormatXL(storageDisks []StorageAPI, setCount, disksPerSet int) (format *formatXLV3, err error) { +func initFormatXL(ctx context.Context, storageDisks []StorageAPI, setCount, disksPerSet int) (format *formatXLV3, err error) { format = newFormatXLV3(setCount, disksPerSet) formats := make([]*formatXLV3, len(storageDisks)) @@ -632,7 +633,7 @@ func initFormatXL(storageDisks []StorageAPI, setCount, disksPerSet int) (format } // Save formats `format.json` across all disks. - if err = saveFormatXLAll(storageDisks, formats); err != nil { + if err = saveFormatXLAll(ctx, storageDisks, formats); err != nil { return nil, err } diff --git a/cmd/fs-v1-helpers.go b/cmd/fs-v1-helpers.go index 1791a0155..7ebeb0d0c 100644 --- a/cmd/fs-v1-helpers.go +++ b/cmd/fs-v1-helpers.go @@ -17,11 +17,13 @@ package cmd import ( + "context" "io" "os" pathutil "path" "runtime" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/lock" ) @@ -29,17 +31,23 @@ import ( // Removes only the file at given path does not remove // any parent directories, handles long paths for // windows automatically. -func fsRemoveFile(filePath string) (err error) { +func fsRemoveFile(ctx context.Context, filePath string) (err error) { if filePath == "" { - return errors.Trace(errInvalidArgument) + logger.LogIf(ctx, errInvalidArgument) + return errInvalidArgument } if err = checkPathLength(filePath); err != nil { - return errors.Trace(err) + logger.LogIf(ctx, err) + return err } if err = os.Remove((filePath)); err != nil { - return osErrToFSFileErr(err) + fsErr := osErrToFSFileErr(err) + if fsErr != errFileNotFound { + logger.LogIf(ctx, err) + } + return fsErr } return nil @@ -47,22 +55,27 @@ func fsRemoveFile(filePath string) (err error) { // Removes all files and folders at a given path, handles // long paths for windows automatically. -func fsRemoveAll(dirPath string) (err error) { +func fsRemoveAll(ctx context.Context, dirPath string) (err error) { if dirPath == "" { - return errors.Trace(errInvalidArgument) + logger.LogIf(ctx, errInvalidArgument) + return errInvalidArgument } if err = checkPathLength(dirPath); err != nil { - return errors.Trace(err) + logger.LogIf(ctx, err) + return err } if err = os.RemoveAll(dirPath); err != nil { if os.IsPermission(err) { - return errors.Trace(errVolumeAccessDenied) + logger.LogIf(ctx, errVolumeAccessDenied) + return errVolumeAccessDenied } else if isSysErrNotEmpty(err) { - return errors.Trace(errVolumeNotEmpty) + logger.LogIf(ctx, errVolumeNotEmpty) + return errVolumeNotEmpty } - return errors.Trace(err) + logger.LogIf(ctx, err) + return err } return nil @@ -70,22 +83,27 @@ func fsRemoveAll(dirPath string) (err error) { // Removes a directory only if its empty, handles long // paths for windows automatically. -func fsRemoveDir(dirPath string) (err error) { +func fsRemoveDir(ctx context.Context, dirPath string) (err error) { if dirPath == "" { - return errors.Trace(errInvalidArgument) + logger.LogIf(ctx, errInvalidArgument) + return errInvalidArgument } if err = checkPathLength(dirPath); err != nil { - return errors.Trace(err) + logger.LogIf(ctx, err) + return err } if err = os.Remove((dirPath)); err != nil { if os.IsNotExist(err) { - return errors.Trace(errVolumeNotFound) + logger.LogIf(ctx, errVolumeNotFound) + return errVolumeNotFound } else if isSysErrNotEmpty(err) { - return errors.Trace(errVolumeNotEmpty) + logger.LogIf(ctx, errVolumeNotEmpty) + return errVolumeNotEmpty } - return errors.Trace(err) + logger.LogIf(ctx, err) + return err } return nil @@ -95,29 +113,36 @@ func fsRemoveDir(dirPath string) (err error) { // otherwise returns an error. If directory already // exists returns an error. Windows long paths // are handled automatically. -func fsMkdir(dirPath string) (err error) { +func fsMkdir(ctx context.Context, dirPath string) (err error) { if dirPath == "" { - return errors.Trace(errInvalidArgument) + logger.LogIf(ctx, errInvalidArgument) + return errInvalidArgument } if err = checkPathLength(dirPath); err != nil { - return errors.Trace(err) + logger.LogIf(ctx, err) + return err } if err = os.Mkdir((dirPath), 0777); err != nil { if os.IsExist(err) { - return errors.Trace(errVolumeExists) + logger.LogIf(ctx, errVolumeExists) + return errVolumeExists } else if os.IsPermission(err) { - return errors.Trace(errDiskAccessDenied) + logger.LogIf(ctx, errDiskAccessDenied) + return errDiskAccessDenied } else if isSysErrNotDir(err) { // File path cannot be verified since // one of the parents is a file. - return errors.Trace(errDiskAccessDenied) + logger.LogIf(ctx, errDiskAccessDenied) + return errDiskAccessDenied } else if isSysErrPathNotFound(err) { // Add specific case for windows. - return errors.Trace(errDiskAccessDenied) + logger.LogIf(ctx, errDiskAccessDenied) + return errDiskAccessDenied } - return errors.Trace(err) + logger.LogIf(ctx, err) + return err } return nil @@ -128,36 +153,40 @@ func fsMkdir(dirPath string) (err error) { // not perform any higher layer interpretation of files v/s // directories. For higher level interpretation look at // fsStatFileDir, fsStatFile, fsStatDir. -func fsStat(statLoc string) (os.FileInfo, error) { +func fsStat(ctx context.Context, statLoc string) (os.FileInfo, error) { if statLoc == "" { - return nil, errors.Trace(errInvalidArgument) + logger.LogIf(ctx, errInvalidArgument) + return nil, errInvalidArgument } if err := checkPathLength(statLoc); err != nil { - return nil, errors.Trace(err) + logger.LogIf(ctx, err) + return nil, err } fi, err := os.Stat((statLoc)) if err != nil { - return nil, errors.Trace(err) + logger.LogIf(ctx, err) + return nil, err } return fi, nil } // Lookup if volume exists, returns volume attributes upon success. -func fsStatVolume(volume string) (os.FileInfo, error) { - fi, err := fsStat(volume) +func fsStatVolume(ctx context.Context, volume string) (os.FileInfo, error) { + fi, err := fsStat(ctx, volume) if err != nil { err = errors.Cause(err) if os.IsNotExist(err) { - return nil, errors.Trace(errVolumeNotFound) + return nil, errVolumeNotFound } else if os.IsPermission(err) { - return nil, errors.Trace(errVolumeAccessDenied) + return nil, errVolumeAccessDenied } - return nil, errors.Trace(err) + return nil, err } if !fi.IsDir() { - return nil, errors.Trace(errVolumeAccessDenied) + logger.LogIf(ctx, errVolumeAccessDenied) + return nil, errVolumeAccessDenied } return fi, nil @@ -173,52 +202,55 @@ func osErrToFSFileErr(err error) error { } err = errors.Cause(err) if os.IsNotExist(err) { - return errors.Trace(errFileNotFound) + return errFileNotFound } if os.IsPermission(err) { - return errors.Trace(errFileAccessDenied) + return errFileAccessDenied } if isSysErrNotDir(err) { - return errors.Trace(errFileAccessDenied) + return errFileAccessDenied } if isSysErrPathNotFound(err) { - return errors.Trace(errFileNotFound) + return errFileNotFound } return err } // Lookup if directory exists, returns directory attributes upon success. -func fsStatDir(statDir string) (os.FileInfo, error) { - fi, err := fsStat(statDir) +func fsStatDir(ctx context.Context, statDir string) (os.FileInfo, error) { + fi, err := fsStat(ctx, statDir) if err != nil { return nil, osErrToFSFileErr(err) } if !fi.IsDir() { - return nil, errors.Trace(errFileAccessDenied) + return nil, errFileAccessDenied } return fi, nil } // Lookup if file exists, returns file attributes upon success. -func fsStatFile(statFile string) (os.FileInfo, error) { - fi, err := fsStat(statFile) +func fsStatFile(ctx context.Context, statFile string) (os.FileInfo, error) { + fi, err := fsStat(ctx, statFile) if err != nil { return nil, osErrToFSFileErr(err) } if fi.IsDir() { - return nil, errors.Trace(errFileAccessDenied) + logger.LogIf(ctx, errFileAccessDenied) + return nil, errFileAccessDenied } return fi, nil } // Opens the file at given path, optionally from an offset. Upon success returns // a readable stream and the size of the readable stream. -func fsOpenFile(readPath string, offset int64) (io.ReadCloser, int64, error) { +func fsOpenFile(ctx context.Context, readPath string, offset int64) (io.ReadCloser, int64, error) { if readPath == "" || offset < 0 { - return nil, 0, errors.Trace(errInvalidArgument) + logger.LogIf(ctx, errInvalidArgument) + return nil, 0, errInvalidArgument } if err := checkPathLength(readPath); err != nil { - return nil, 0, errors.Trace(err) + logger.LogIf(ctx, err) + return nil, 0, err } fr, err := os.Open((readPath)) @@ -229,19 +261,22 @@ func fsOpenFile(readPath string, offset int64) (io.ReadCloser, int64, error) { // Stat to get the size of the file at path. st, err := os.Stat((readPath)) if err != nil { - return nil, 0, errors.Trace(err) + logger.LogIf(ctx, err) + return nil, 0, err } // Verify if its not a regular file, since subsequent Seek is undefined. if !st.Mode().IsRegular() { - return nil, 0, errors.Trace(errIsNotRegular) + logger.LogIf(ctx, errIsNotRegular) + return nil, 0, errIsNotRegular } // Seek to the requested offset. if offset > 0 { _, err = fr.Seek(offset, os.SEEK_SET) if err != nil { - return nil, 0, errors.Trace(err) + logger.LogIf(ctx, err) + return nil, 0, err } } @@ -250,21 +285,25 @@ func fsOpenFile(readPath string, offset int64) (io.ReadCloser, int64, error) { } // Creates a file and copies data from incoming reader. Staging buffer is used by io.CopyBuffer. -func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int64) (int64, error) { +func fsCreateFile(ctx context.Context, filePath string, reader io.Reader, buf []byte, fallocSize int64) (int64, error) { if filePath == "" || reader == nil { - return 0, errors.Trace(errInvalidArgument) + logger.LogIf(ctx, errInvalidArgument) + return 0, errInvalidArgument } if err := checkPathLength(filePath); err != nil { - return 0, errors.Trace(err) + logger.LogIf(ctx, err) + return 0, err } if err := mkdirAll(pathutil.Dir(filePath), 0777); err != nil { - return 0, errors.Trace(err) + logger.LogIf(ctx, err) + return 0, err } if err := checkDiskFree(pathutil.Dir(filePath), fallocSize); err != nil { - return 0, errors.Trace(err) + logger.LogIf(ctx, err) + return 0, err } writer, err := lock.Open(filePath, os.O_CREATE|os.O_WRONLY, 0666) @@ -276,7 +315,8 @@ func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int6 // Fallocate only if the size is final object is known. if fallocSize > 0 { if err = fsFAllocate(int(writer.Fd()), 0, fallocSize); err != nil { - return 0, errors.Trace(err) + logger.LogIf(ctx, err) + return 0, err } } @@ -284,12 +324,14 @@ func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int6 if buf != nil { bytesWritten, err = io.CopyBuffer(writer, reader, buf) if err != nil { - return 0, errors.Trace(err) + logger.LogIf(ctx, err) + return 0, err } } else { bytesWritten, err = io.Copy(writer, reader) if err != nil { - return 0, errors.Trace(err) + logger.LogIf(ctx, err) + return 0, err } } @@ -320,12 +362,14 @@ func fsFAllocate(fd int, offset int64, len int64) (err error) { // Renames source path to destination path, creates all the // missing parents if they don't exist. -func fsRenameFile(sourcePath, destPath string) error { +func fsRenameFile(ctx context.Context, sourcePath, destPath string) error { if err := checkPathLength(sourcePath); err != nil { - return errors.Trace(err) + logger.LogIf(ctx, err) + return err } if err := checkPathLength(destPath); err != nil { - return errors.Trace(err) + logger.LogIf(ctx, err) + return err } // Verify if source path exists. @@ -334,27 +378,34 @@ func fsRenameFile(sourcePath, destPath string) error { } if err := renameAll(sourcePath, destPath); err != nil { - return errors.Trace(err) + logger.LogIf(ctx, err) + return err } return nil } // fsDeleteFile is a wrapper for deleteFile(), after checking the path length. -func fsDeleteFile(basePath, deletePath string) error { +func fsDeleteFile(ctx context.Context, basePath, deletePath string) error { if err := checkPathLength(basePath); err != nil { - return errors.Trace(err) + logger.LogIf(ctx, err) + return err } if err := checkPathLength(deletePath); err != nil { - return errors.Trace(err) + logger.LogIf(ctx, err) + return err } - return deleteFile(basePath, deletePath) + if err := deleteFile(basePath, deletePath); err != nil { + logger.LogIf(ctx, err) + return err + } + return nil } // fsRemoveMeta safely removes a locked file and takes care of Windows special case -func fsRemoveMeta(basePath, deletePath, tmpDir string) error { +func fsRemoveMeta(ctx context.Context, basePath, deletePath, tmpDir string) error { // Special case for windows please read through. if runtime.GOOS == globalWindowsOSName { // Ordinarily windows does not permit deletion or renaming of files still @@ -388,13 +439,13 @@ func fsRemoveMeta(basePath, deletePath, tmpDir string) error { tmpPath := pathJoin(tmpDir, mustGetUUID()) - fsRenameFile(deletePath, tmpPath) + fsRenameFile(ctx, deletePath, tmpPath) // Proceed to deleting the directory if empty - fsDeleteFile(basePath, pathutil.Dir(deletePath)) + fsDeleteFile(ctx, basePath, pathutil.Dir(deletePath)) // Finally delete the renamed file. - return fsDeleteFile(tmpDir, tmpPath) + return fsDeleteFile(ctx, tmpDir, tmpPath) } - return fsDeleteFile(basePath, deletePath) + return fsDeleteFile(ctx, basePath, deletePath) } diff --git a/cmd/fs-v1-helpers_test.go b/cmd/fs-v1-helpers_test.go index 0fc6177b0..ac44e744f 100644 --- a/cmd/fs-v1-helpers_test.go +++ b/cmd/fs-v1-helpers_test.go @@ -18,6 +18,7 @@ package cmd import ( "bytes" + "context" "io" "io/ioutil" "os" @@ -36,19 +37,19 @@ func TestFSRenameFile(t *testing.T) { } defer os.RemoveAll(path) - if err = fsMkdir(pathJoin(path, "testvolume1")); err != nil { + if err = fsMkdir(context.Background(), pathJoin(path, "testvolume1")); err != nil { t.Fatal(err) } - if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); err != nil { + if err = fsRenameFile(context.Background(), pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); err != nil { t.Fatal(err) } - if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); errors.Cause(err) != errFileNotFound { + if err = fsRenameFile(context.Background(), pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); errors.Cause(err) != errFileNotFound { t.Fatal(err) } - if err = fsRenameFile(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), pathJoin(path, "testvolume2")); errors.Cause(err) != errFileNameTooLong { + if err = fsRenameFile(context.Background(), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), pathJoin(path, "testvolume2")); errors.Cause(err) != errFileNameTooLong { t.Fatal("Unexpected error", err) } - if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errors.Cause(err) != errFileNameTooLong { + if err = fsRenameFile(context.Background(), pathJoin(path, "testvolume1"), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errors.Cause(err) != errFileNameTooLong { t.Fatal("Unexpected error", err) } } @@ -63,30 +64,30 @@ func TestFSStats(t *testing.T) { // Setup test environment. - if err = fsMkdir(""); errors.Cause(err) != errInvalidArgument { + if err = fsMkdir(context.Background(), ""); errors.Cause(err) != errInvalidArgument { t.Fatal("Unexpected error", err) } - if err = fsMkdir(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errors.Cause(err) != errFileNameTooLong { + if err = fsMkdir(context.Background(), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errors.Cause(err) != errFileNameTooLong { t.Fatal("Unexpected error", err) } - if err = fsMkdir(pathJoin(path, "success-vol")); err != nil { + if err = fsMkdir(context.Background(), pathJoin(path, "success-vol")); err != nil { t.Fatalf("Unable to create volume, %s", err) } var reader = bytes.NewReader([]byte("Hello, world")) - if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil { + if _, err = fsCreateFile(context.Background(), pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil { t.Fatalf("Unable to create file, %s", err) } // Seek back. reader.Seek(0, 0) - if err = fsMkdir(pathJoin(path, "success-vol", "success-file")); errors.Cause(err) != errVolumeExists { + if err = fsMkdir(context.Background(), pathJoin(path, "success-vol", "success-file")); errors.Cause(err) != errVolumeExists { t.Fatal("Unexpected error", err) } - if _, err = fsCreateFile(pathJoin(path, "success-vol", "path/to/success-file"), reader, nil, 0); err != nil { + if _, err = fsCreateFile(context.Background(), pathJoin(path, "success-vol", "path/to/success-file"), reader, nil, 0); err != nil { t.Fatalf("Unable to create file, %s", err) } // Seek back. @@ -169,12 +170,12 @@ func TestFSStats(t *testing.T) { for i, testCase := range testCases { if testCase.srcPath != "" { - if _, err := fsStatFile(pathJoin(testCase.srcFSPath, testCase.srcVol, + if _, err := fsStatFile(context.Background(), pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); errors.Cause(err) != testCase.expectedErr { t.Fatalf("TestPosix case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) } } else { - if _, err := fsStatVolume(pathJoin(testCase.srcFSPath, testCase.srcVol)); errors.Cause(err) != testCase.expectedErr { + if _, err := fsStatVolume(context.Background(), pathJoin(testCase.srcFSPath, testCase.srcVol)); errors.Cause(err) != testCase.expectedErr { t.Fatalf("TestPosix case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) } } @@ -189,20 +190,20 @@ func TestFSCreateAndOpen(t *testing.T) { } defer os.RemoveAll(path) - if err = fsMkdir(pathJoin(path, "success-vol")); err != nil { + if err = fsMkdir(context.Background(), pathJoin(path, "success-vol")); err != nil { t.Fatalf("Unable to create directory, %s", err) } - if _, err = fsCreateFile("", nil, nil, 0); errors.Cause(err) != errInvalidArgument { + if _, err = fsCreateFile(context.Background(), "", nil, nil, 0); errors.Cause(err) != errInvalidArgument { t.Fatal("Unexpected error", err) } - if _, _, err = fsOpenFile("", -1); errors.Cause(err) != errInvalidArgument { + if _, _, err = fsOpenFile(context.Background(), "", -1); errors.Cause(err) != errInvalidArgument { t.Fatal("Unexpected error", err) } var reader = bytes.NewReader([]byte("Hello, world")) - if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil { + if _, err = fsCreateFile(context.Background(), pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil { t.Fatalf("Unable to create file, %s", err) } // Seek back. @@ -230,18 +231,18 @@ func TestFSCreateAndOpen(t *testing.T) { } for i, testCase := range testCases { - _, err = fsCreateFile(pathJoin(path, testCase.srcVol, testCase.srcPath), reader, nil, 0) + _, err = fsCreateFile(context.Background(), pathJoin(path, testCase.srcVol, testCase.srcPath), reader, nil, 0) if errors.Cause(err) != testCase.expectedErr { t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) } - _, _, err = fsOpenFile(pathJoin(path, testCase.srcVol, testCase.srcPath), 0) + _, _, err = fsOpenFile(context.Background(), pathJoin(path, testCase.srcVol, testCase.srcPath), 0) if errors.Cause(err) != testCase.expectedErr { t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) } } // Attempt to open a directory. - if _, _, err = fsOpenFile(pathJoin(path), 0); errors.Cause(err) != errIsNotRegular { + if _, _, err = fsOpenFile(context.Background(), pathJoin(path), 0); errors.Cause(err) != errIsNotRegular { t.Fatal("Unexpected error", err) } } @@ -255,20 +256,20 @@ func TestFSDeletes(t *testing.T) { defer os.RemoveAll(path) // Setup test environment. - if err = fsMkdir(pathJoin(path, "success-vol")); err != nil { + if err = fsMkdir(context.Background(), pathJoin(path, "success-vol")); err != nil { t.Fatalf("Unable to create directory, %s", err) } var buf = make([]byte, 4096) var reader = bytes.NewReader([]byte("Hello, world")) - if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, buf, reader.Size()); err != nil { + if _, err = fsCreateFile(context.Background(), pathJoin(path, "success-vol", "success-file"), reader, buf, reader.Size()); err != nil { t.Fatalf("Unable to create file, %s", err) } // Seek back. reader.Seek(0, io.SeekStart) // folder is not empty - err = fsMkdir(pathJoin(path, "success-vol", "not-empty")) + err = fsMkdir(context.Background(), pathJoin(path, "success-vol", "not-empty")) if err != nil { t.Fatal(err) } @@ -278,10 +279,10 @@ func TestFSDeletes(t *testing.T) { } // recursive - if err = fsMkdir(pathJoin(path, "success-vol", "parent")); err != nil { + if err = fsMkdir(context.Background(), pathJoin(path, "success-vol", "parent")); err != nil { t.Fatal(err) } - if err = fsMkdir(pathJoin(path, "success-vol", "parent", "dir")); err != nil { + if err = fsMkdir(context.Background(), pathJoin(path, "success-vol", "parent", "dir")); err != nil { t.Fatal(err) } @@ -343,7 +344,7 @@ func TestFSDeletes(t *testing.T) { } for i, testCase := range testCases { - if err = fsDeleteFile(testCase.basePath, pathJoin(testCase.basePath, testCase.srcVol, testCase.srcPath)); errors.Cause(err) != testCase.expectedErr { + if err = fsDeleteFile(context.Background(), testCase.basePath, pathJoin(testCase.basePath, testCase.srcVol, testCase.srcPath)); errors.Cause(err) != testCase.expectedErr { t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) } } @@ -358,7 +359,7 @@ func BenchmarkFSDeleteFile(b *testing.B) { defer os.RemoveAll(path) // Setup test environment. - if err = fsMkdir(pathJoin(path, "benchmark")); err != nil { + if err = fsMkdir(context.Background(), pathJoin(path, "benchmark")); err != nil { b.Fatalf("Unable to create directory, %s", err) } @@ -375,7 +376,7 @@ func BenchmarkFSDeleteFile(b *testing.B) { } b.StartTimer() - err = fsDeleteFile(benchDir, filename) + err = fsDeleteFile(context.Background(), benchDir, filename) if err != nil { b.Fatal(err) } @@ -392,18 +393,18 @@ func TestFSRemoves(t *testing.T) { defer os.RemoveAll(path) // Setup test environment. - if err = fsMkdir(pathJoin(path, "success-vol")); err != nil { + if err = fsMkdir(context.Background(), pathJoin(path, "success-vol")); err != nil { t.Fatalf("Unable to create directory, %s", err) } var reader = bytes.NewReader([]byte("Hello, world")) - if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil { + if _, err = fsCreateFile(context.Background(), pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil { t.Fatalf("Unable to create file, %s", err) } // Seek back. reader.Seek(0, 0) - if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file-new"), reader, nil, 0); err != nil { + if _, err = fsCreateFile(context.Background(), pathJoin(path, "success-vol", "success-file-new"), reader, nil, 0); err != nil { t.Fatalf("Unable to create file, %s", err) } // Seek back. @@ -477,25 +478,25 @@ func TestFSRemoves(t *testing.T) { for i, testCase := range testCases { if testCase.srcPath != "" { - if err = fsRemoveFile(pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); errors.Cause(err) != testCase.expectedErr { + if err = fsRemoveFile(context.Background(), pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); errors.Cause(err) != testCase.expectedErr { t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) } } else { - if err = fsRemoveDir(pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); errors.Cause(err) != testCase.expectedErr { + if err = fsRemoveDir(context.Background(), pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); errors.Cause(err) != testCase.expectedErr { t.Error(err) } } } - if err = fsRemoveAll(pathJoin(path, "success-vol")); err != nil { + if err = fsRemoveAll(context.Background(), pathJoin(path, "success-vol")); err != nil { t.Fatal(err) } - if err = fsRemoveAll(""); errors.Cause(err) != errInvalidArgument { + if err = fsRemoveAll(context.Background(), ""); errors.Cause(err) != errInvalidArgument { t.Fatal(err) } - if err = fsRemoveAll("my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"); errors.Cause(err) != errFileNameTooLong { + if err = fsRemoveAll(context.Background(), "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"); errors.Cause(err) != errFileNameTooLong { t.Fatal(err) } } @@ -509,14 +510,14 @@ func TestFSRemoveMeta(t *testing.T) { defer os.RemoveAll(fsPath) // Setup test environment. - if err = fsMkdir(pathJoin(fsPath, "success-vol")); err != nil { + if err = fsMkdir(context.Background(), pathJoin(fsPath, "success-vol")); err != nil { t.Fatalf("Unable to create directory, %s", err) } filePath := pathJoin(fsPath, "success-vol", "success-file") var reader = bytes.NewReader([]byte("Hello, world")) - if _, err = fsCreateFile(filePath, reader, nil, 0); err != nil { + if _, err = fsCreateFile(context.Background(), filePath, reader, nil, 0); err != nil { t.Fatalf("Unable to create file, %s", err) } @@ -535,7 +536,7 @@ func TestFSRemoveMeta(t *testing.T) { t.Fatal(tmpErr) } - if err := fsRemoveMeta(fsPath, filePath, tmpDir); err != nil { + if err := fsRemoveMeta(context.Background(), fsPath, filePath, tmpDir); err != nil { t.Fatalf("Unable to remove file, %s", err) } diff --git a/cmd/fs-v1-metadata.go b/cmd/fs-v1-metadata.go index e23fc4ba3..4c78ca9e1 100644 --- a/cmd/fs-v1-metadata.go +++ b/cmd/fs-v1-metadata.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "encoding/hex" "encoding/json" "io" @@ -25,7 +26,7 @@ import ( pathutil "path" "strings" - "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/lock" "github.com/minio/minio/pkg/mimedb" "github.com/tidwall/gjson" @@ -237,20 +238,23 @@ func parseFSPartsArray(fsMetaBuf []byte) []objectPartInfo { return partsArray } -func (m *fsMetaV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) { +func (m *fsMetaV1) ReadFrom(ctx context.Context, lk *lock.LockedFile) (n int64, err error) { var fsMetaBuf []byte fi, err := lk.Stat() if err != nil { - return 0, errors.Trace(err) + logger.LogIf(ctx, err) + return 0, err } fsMetaBuf, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size())) if err != nil { - return 0, errors.Trace(err) + logger.LogIf(ctx, err) + return 0, err } if len(fsMetaBuf) == 0 { - return 0, errors.Trace(io.EOF) + logger.LogIf(ctx, io.EOF) + return 0, io.EOF } // obtain version. @@ -259,7 +263,9 @@ func (m *fsMetaV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) { // Verify if the format is valid, return corrupted format // for unrecognized formats. if !isFSMetaValid(m.Version) { - return 0, errors.Trace(errCorruptedFormat) + logger.GetReqInfo(ctx).AppendTags("file", lk.Name()) + logger.LogIf(ctx, errCorruptedFormat) + return 0, errCorruptedFormat } // obtain parts information diff --git a/cmd/fs-v1-metadata_test.go b/cmd/fs-v1-metadata_test.go index fdd720875..28c3eddba 100644 --- a/cmd/fs-v1-metadata_test.go +++ b/cmd/fs-v1-metadata_test.go @@ -70,7 +70,7 @@ func TestReadFSMetadata(t *testing.T) { // Regular fs metadata reading, no errors expected fsMeta := fsMetaV1{} - if _, err = fsMeta.ReadFrom(rlk.LockedFile); err != nil { + if _, err = fsMeta.ReadFrom(context.Background(), rlk.LockedFile); err != nil { t.Fatal("Unexpected error ", err) } } @@ -105,7 +105,7 @@ func TestWriteFSMetadata(t *testing.T) { // FS metadata reading, no errors expected (healthy disk) fsMeta := fsMetaV1{} - _, err = fsMeta.ReadFrom(rlk.LockedFile) + _, err = fsMeta.ReadFrom(context.Background(), rlk.LockedFile) if err != nil { t.Fatal("Unexpected error ", err) } diff --git a/cmd/fs-v1-multipart.go b/cmd/fs-v1-multipart.go index 6ba3d2b8b..2153a6a71 100644 --- a/cmd/fs-v1-multipart.go +++ b/cmd/fs-v1-multipart.go @@ -29,6 +29,7 @@ import ( "strings" "time" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/errors" mioutil "github.com/minio/minio/pkg/ioutil" @@ -64,8 +65,9 @@ func (fs *FSObjects) decodePartFile(name string) (partNumber int, etag string, e } // Appends parts to an appendFile sequentially. -func (fs *FSObjects) backgroundAppend(bucket, object, uploadID string) { +func (fs *FSObjects) backgroundAppend(ctx context.Context, bucket, object, uploadID string) { fs.appendFileMapMu.Lock() + logger.GetReqInfo(ctx).AppendTags("uploadID", uploadID) file := fs.appendFileMap[uploadID] if file == nil { file = &fsAppendFile{ @@ -84,7 +86,8 @@ func (fs *FSObjects) backgroundAppend(bucket, object, uploadID string) { entries, err := readDir(uploadIDDir) if err != nil { - errorIf(err, "error reading directory %s", uploadIDDir) + logger.GetReqInfo(ctx).AppendTags("uploadIDDir", uploadIDDir) + logger.LogIf(ctx, err) return } sort.Strings(entries) @@ -95,7 +98,8 @@ func (fs *FSObjects) backgroundAppend(bucket, object, uploadID string) { } partNumber, etag, err := fs.decodePartFile(entry) if err != nil { - errorIf(err, "unable to split the file name into partNumber and etag: %s", entry) + logger.GetReqInfo(ctx).AppendTags("entry", entry) + logger.LogIf(ctx, err) return } if partNumber < nextPartNumber { @@ -110,7 +114,9 @@ func (fs *FSObjects) backgroundAppend(bucket, object, uploadID string) { partPath := pathJoin(uploadIDDir, entry) err = mioutil.AppendFile(file.filePath, partPath) if err != nil { - errorIf(err, "Unable to append %s to %s", partPath, file.filePath) + reqInfo := logger.GetReqInfo(ctx).AppendTags("partPath", partPath) + reqInfo.AppendTags("filepath", file.filePath) + logger.LogIf(ctx, err) return } @@ -122,12 +128,12 @@ func (fs *FSObjects) backgroundAppend(bucket, object, uploadID string) { // ListMultipartUploads - lists all the uploadIDs for the specified object. // We do not support prefix based listing. func (fs *FSObjects) ListMultipartUploads(ctx context.Context, bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, e error) { - if err := checkListMultipartArgs(bucket, object, keyMarker, uploadIDMarker, delimiter, fs); err != nil { - return result, toObjectErr(errors.Trace(err)) + if err := checkListMultipartArgs(ctx, bucket, object, keyMarker, uploadIDMarker, delimiter, fs); err != nil { + return result, toObjectErr(err) } - if _, err := fs.statBucketDir(bucket); err != nil { - return result, toObjectErr(errors.Trace(err), bucket) + if _, err := fs.statBucketDir(ctx, bucket); err != nil { + return result, toObjectErr(err, bucket) } result.MaxUploads = maxUploads @@ -143,7 +149,8 @@ func (fs *FSObjects) ListMultipartUploads(ctx context.Context, bucket, object, k result.IsTruncated = false return result, nil } - return result, toObjectErr(errors.Trace(err)) + logger.LogIf(ctx, err) + return result, toObjectErr(err) } // S3 spec says uploaIDs should be sorted based on initiated time. ModTime of fs.json @@ -151,7 +158,7 @@ func (fs *FSObjects) ListMultipartUploads(ctx context.Context, bucket, object, k var uploads []MultipartInfo for _, uploadID := range uploadIDs { metaFilePath := pathJoin(fs.getMultipartSHADir(bucket, object), uploadID, fs.metaJSONFile) - fi, err := fsStatFile(metaFilePath) + fi, err := fsStatFile(ctx, metaFilePath) if err != nil { return result, toObjectErr(err, bucket, object) } @@ -204,11 +211,11 @@ func (fs *FSObjects) ListMultipartUploads(ctx context.Context, bucket, object, k // // Implements S3 compatible initiate multipart API. func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string) (string, error) { - if err := checkNewMultipartArgs(bucket, object, fs); err != nil { + if err := checkNewMultipartArgs(ctx, bucket, object, fs); err != nil { return "", toObjectErr(err, bucket) } - if _, err := fs.statBucketDir(bucket); err != nil { + if _, err := fs.statBucketDir(ctx, bucket); err != nil { return "", toObjectErr(err, bucket) } @@ -217,7 +224,8 @@ func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object stri err := mkdirAll(uploadIDDir, 0755) if err != nil { - return "", errors.Trace(err) + logger.LogIf(ctx, err) + return "", err } // Initialize fs.json values. @@ -226,11 +234,13 @@ func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object stri fsMetaBytes, err := json.Marshal(fsMeta) if err != nil { - return "", errors.Trace(err) + logger.LogIf(ctx, err) + return "", err } if err = ioutil.WriteFile(pathJoin(uploadIDDir, fs.metaJSONFile), fsMetaBytes, 0644); err != nil { - return "", errors.Trace(err) + logger.LogIf(ctx, err) + return "", err } return uploadID, nil @@ -242,22 +252,22 @@ func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object stri func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo) (pi PartInfo, e error) { - if err := checkNewMultipartArgs(srcBucket, srcObject, fs); err != nil { - return pi, toObjectErr(errors.Trace(err)) + if err := checkNewMultipartArgs(ctx, srcBucket, srcObject, fs); err != nil { + return pi, toObjectErr(err) } // Initialize pipe. go func() { if gerr := fs.GetObject(ctx, srcBucket, srcObject, startOffset, length, srcInfo.Writer, srcInfo.ETag); gerr != nil { if gerr = srcInfo.Writer.Close(); gerr != nil { - errorIf(gerr, "Unable to read %s/%s.", srcBucket, srcObject) + logger.LogIf(ctx, gerr) return } return } // Close writer explicitly signalling we wrote all data. if gerr := srcInfo.Writer.Close(); gerr != nil { - errorIf(gerr, "Unable to read %s/%s.", srcBucket, srcObject) + logger.LogIf(ctx, gerr) return } }() @@ -275,26 +285,27 @@ func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d // written to '.minio.sys/tmp' location and safely renamed to // '.minio.sys/multipart' for reach parts. func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (pi PartInfo, e error) { - if err := checkPutObjectPartArgs(bucket, object, fs); err != nil { - return pi, toObjectErr(errors.Trace(err), bucket) + if err := checkPutObjectPartArgs(ctx, bucket, object, fs); err != nil { + return pi, toObjectErr(err, bucket) } - if _, err := fs.statBucketDir(bucket); err != nil { - return pi, toObjectErr(errors.Trace(err), bucket) + if _, err := fs.statBucketDir(ctx, bucket); err != nil { + return pi, toObjectErr(err, bucket) } // Validate input data size and it can never be less than zero. if data.Size() < 0 { - return pi, toObjectErr(errors.Trace(errInvalidArgument)) + logger.LogIf(ctx, errInvalidArgument) + return pi, toObjectErr(errInvalidArgument) } uploadIDDir := fs.getUploadIDDir(bucket, object, uploadID) // Just check if the uploadID exists to avoid copy if it doesn't. - _, err := fsStatFile(pathJoin(uploadIDDir, fs.metaJSONFile)) + _, err := fsStatFile(ctx, pathJoin(uploadIDDir, fs.metaJSONFile)) if err != nil { if errors.Cause(err) == errFileNotFound || errors.Cause(err) == errFileAccessDenied { - return pi, errors.Trace(InvalidUploadID{UploadID: uploadID}) + return pi, InvalidUploadID{UploadID: uploadID} } return pi, toObjectErr(err, bucket, object) } @@ -306,23 +317,23 @@ func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID buf := make([]byte, bufSize) tmpPartPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, uploadID+"."+mustGetUUID()+"."+strconv.Itoa(partID)) - bytesWritten, err := fsCreateFile(tmpPartPath, data, buf, data.Size()) + bytesWritten, err := fsCreateFile(ctx, tmpPartPath, data, buf, data.Size()) if err != nil { - fsRemoveFile(tmpPartPath) + fsRemoveFile(ctx, tmpPartPath) return pi, toObjectErr(err, minioMetaTmpBucket, tmpPartPath) } // Should return IncompleteBody{} error when reader has fewer // bytes than specified in request header. if bytesWritten < data.Size() { - fsRemoveFile(tmpPartPath) - return pi, errors.Trace(IncompleteBody{}) + fsRemoveFile(ctx, tmpPartPath) + return pi, IncompleteBody{} } // Delete temporary part in case of failure. If // PutObjectPart succeeds then there would be nothing to // delete in which case we just ignore the error. - defer fsRemoveFile(tmpPartPath) + defer fsRemoveFile(ctx, tmpPartPath) etag := hex.EncodeToString(data.MD5Current()) if etag == "" { @@ -330,13 +341,13 @@ func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID } partPath := pathJoin(uploadIDDir, fs.encodePartFile(partID, etag)) - if err = fsRenameFile(tmpPartPath, partPath); err != nil { + if err = fsRenameFile(ctx, tmpPartPath, partPath); err != nil { return pi, toObjectErr(err, minioMetaMultipartBucket, partPath) } - go fs.backgroundAppend(bucket, object, uploadID) + go fs.backgroundAppend(ctx, bucket, object, uploadID) - fi, err := fsStatFile(partPath) + fi, err := fsStatFile(ctx, partPath) if err != nil { return pi, toObjectErr(err, minioMetaMultipartBucket, partPath) } @@ -356,8 +367,8 @@ func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID // ListPartsInfo structure is unmarshalled directly into XML and // replied back to the client. func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int) (result ListPartsInfo, e error) { - if err := checkListPartsArgs(bucket, object, fs); err != nil { - return result, toObjectErr(errors.Trace(err)) + if err := checkListPartsArgs(ctx, bucket, object, fs); err != nil { + return result, toObjectErr(err) } result.Bucket = bucket result.Object = object @@ -366,22 +377,23 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload result.PartNumberMarker = partNumberMarker // Check if bucket exists - if _, err := fs.statBucketDir(bucket); err != nil { - return result, toObjectErr(errors.Trace(err), bucket) + if _, err := fs.statBucketDir(ctx, bucket); err != nil { + return result, toObjectErr(err, bucket) } uploadIDDir := fs.getUploadIDDir(bucket, object, uploadID) - _, err := fsStatFile(pathJoin(uploadIDDir, fs.metaJSONFile)) + _, err := fsStatFile(ctx, pathJoin(uploadIDDir, fs.metaJSONFile)) if err != nil { if errors.Cause(err) == errFileNotFound || errors.Cause(err) == errFileAccessDenied { - return result, errors.Trace(InvalidUploadID{UploadID: uploadID}) + return result, InvalidUploadID{UploadID: uploadID} } - return result, toObjectErr(errors.Trace(err), bucket, object) + return result, toObjectErr(err, bucket, object) } entries, err := readDir(uploadIDDir) if err != nil { - return result, toObjectErr(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return result, toObjectErr(err, bucket) } partsMap := make(map[int]string) @@ -391,20 +403,21 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload } partNumber, etag1, derr := fs.decodePartFile(entry) if derr != nil { - return result, toObjectErr(errors.Trace(derr)) + logger.LogIf(ctx, derr) + return result, toObjectErr(derr) } etag2, ok := partsMap[partNumber] if !ok { partsMap[partNumber] = etag1 continue } - stat1, serr := fsStatFile(pathJoin(uploadIDDir, fs.encodePartFile(partNumber, etag1))) + stat1, serr := fsStatFile(ctx, pathJoin(uploadIDDir, fs.encodePartFile(partNumber, etag1))) if serr != nil { - return result, toObjectErr(errors.Trace(serr)) + return result, toObjectErr(serr) } - stat2, serr := fsStatFile(pathJoin(uploadIDDir, fs.encodePartFile(partNumber, etag2))) + stat2, serr := fsStatFile(ctx, pathJoin(uploadIDDir, fs.encodePartFile(partNumber, etag2))) if serr != nil { - return result, toObjectErr(errors.Trace(serr)) + return result, toObjectErr(serr) } if stat1.ModTime().After(stat2.ModTime()) { partsMap[partNumber] = etag1 @@ -443,9 +456,9 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload } for i, part := range result.Parts { var stat os.FileInfo - stat, err = fsStatFile(pathJoin(uploadIDDir, fs.encodePartFile(part.PartNumber, part.ETag))) + stat, err = fsStatFile(ctx, pathJoin(uploadIDDir, fs.encodePartFile(part.PartNumber, part.ETag))) if err != nil { - return result, toObjectErr(errors.Trace(err)) + return result, toObjectErr(err) } result.Parts[i].LastModified = stat.ModTime() result.Parts[i].Size = stat.Size() @@ -453,7 +466,8 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload fsMetaBytes, err := ioutil.ReadFile(pathJoin(uploadIDDir, fs.metaJSONFile)) if err != nil { - return result, errors.Trace(err) + logger.LogIf(ctx, err) + return result, err } result.UserDefined = parseFSMetaMap(fsMetaBytes) @@ -467,31 +481,31 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload // // Implements S3 compatible Complete multipart API. func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart) (oi ObjectInfo, e error) { - if err := checkCompleteMultipartArgs(bucket, object, fs); err != nil { + if err := checkCompleteMultipartArgs(ctx, bucket, object, fs); err != nil { return oi, toObjectErr(err) } // Check if an object is present as one of the parent dir. - if fs.parentDirIsObject(bucket, pathutil.Dir(object)) { - return oi, toObjectErr(errors.Trace(errFileAccessDenied), bucket, object) + if fs.parentDirIsObject(ctx, bucket, pathutil.Dir(object)) { + return oi, toObjectErr(errFileAccessDenied, bucket, object) } - if _, err := fs.statBucketDir(bucket); err != nil { + if _, err := fs.statBucketDir(ctx, bucket); err != nil { return oi, toObjectErr(err, bucket) } uploadIDDir := fs.getUploadIDDir(bucket, object, uploadID) // Just check if the uploadID exists to avoid copy if it doesn't. - _, err := fsStatFile(pathJoin(uploadIDDir, fs.metaJSONFile)) + _, err := fsStatFile(ctx, pathJoin(uploadIDDir, fs.metaJSONFile)) if err != nil { if errors.Cause(err) == errFileNotFound || errors.Cause(err) == errFileAccessDenied { - return oi, errors.Trace(InvalidUploadID{UploadID: uploadID}) + return oi, InvalidUploadID{UploadID: uploadID} } return oi, toObjectErr(err, bucket, object) } // Calculate s3 compatible md5sum for complete multipart. - s3MD5, err := getCompleteMultipartMD5(parts) + s3MD5, err := getCompleteMultipartMD5(ctx, parts) if err != nil { return oi, err } @@ -507,12 +521,12 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string, for i, part := range parts { partPath := pathJoin(uploadIDDir, fs.encodePartFile(part.PartNumber, part.ETag)) var fi os.FileInfo - fi, err = fsStatFile(partPath) + fi, err = fsStatFile(ctx, partPath) if err != nil { if errors.Cause(err) == errFileNotFound || errors.Cause(err) == errFileAccessDenied { - return oi, errors.Trace(InvalidPart{}) + return oi, InvalidPart{} } - return oi, errors.Trace(err) + return oi, err } if partSize == -1 { partSize = fi.Size() @@ -530,11 +544,13 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string, // All parts except the last part has to be atleast 5MB. if !isMinAllowedPartSize(fi.Size()) { - return oi, errors.Trace(PartTooSmall{ + err = PartTooSmall{ PartNumber: part.PartNumber, PartSize: fi.Size(), PartETag: part.ETag, - }) + } + logger.LogIf(ctx, err) + return oi, err } // TODO: Make necessary changes in future as explained in the below comment. @@ -545,7 +561,8 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string, // CompleteMultipartUpload we already have the full file available which can be // renamed to the main name-space. if partSize != fi.Size() { - return oi, errors.Trace(PartsSizeUnequal{}) + logger.LogIf(ctx, PartsSizeUnequal{}) + return oi, PartsSizeUnequal{} } } @@ -557,7 +574,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string, // 1. The last PutObjectPart triggers go-routine fs.backgroundAppend, this go-routine has not started yet. // 2. Now CompleteMultipartUpload gets called which sees that lastPart is not appended and starts appending // from the beginning - fs.backgroundAppend(bucket, object, uploadID) + fs.backgroundAppend(ctx, bucket, object, uploadID) fs.appendFileMapMu.Lock() file := fs.appendFileMap[uploadID] @@ -585,12 +602,13 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string, } if appendFallback { - fsRemoveFile(file.filePath) + fsRemoveFile(ctx, file.filePath) for _, part := range parts { partPath := pathJoin(uploadIDDir, fs.encodePartFile(part.PartNumber, part.ETag)) err = mioutil.AppendFile(appendFilePath, partPath) if err != nil { - return oi, toObjectErr(errors.Trace(err)) + logger.LogIf(ctx, err) + return oi, toObjectErr(err) } } } @@ -604,18 +622,21 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string, fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile) metaFile, err := fs.rwPool.Create(fsMetaPath) if err != nil { - return oi, toObjectErr(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return oi, toObjectErr(err, bucket, object) } defer metaFile.Close() // Read saved fs metadata for ongoing multipart. fsMetaBuf, err := ioutil.ReadFile(pathJoin(uploadIDDir, fs.metaJSONFile)) if err != nil { - return oi, toObjectErr(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return oi, toObjectErr(err, bucket, object) } err = json.Unmarshal(fsMetaBuf, &fsMeta) if err != nil { - return oi, toObjectErr(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return oi, toObjectErr(err, bucket, object) } // Save additional metadata. if len(fsMeta.Meta) == 0 { @@ -623,24 +644,26 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string, } fsMeta.Meta["etag"] = s3MD5 if _, err = fsMeta.WriteTo(metaFile); err != nil { - return oi, toObjectErr(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return oi, toObjectErr(err, bucket, object) } // Deny if WORM is enabled if globalWORMEnabled { - if _, err = fsStatFile(pathJoin(fs.fsPath, bucket, object)); err == nil { - return ObjectInfo{}, errors.Trace(ObjectAlreadyExists{Bucket: bucket, Object: object}) + if _, err = fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object)); err == nil { + return ObjectInfo{}, ObjectAlreadyExists{Bucket: bucket, Object: object} } } - err = fsRenameFile(appendFilePath, pathJoin(fs.fsPath, bucket, object)) + err = fsRenameFile(ctx, appendFilePath, pathJoin(fs.fsPath, bucket, object)) if err != nil { - return oi, toObjectErr(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return oi, toObjectErr(err, bucket, object) } - fsRemoveAll(uploadIDDir) - fi, err := fsStatFile(pathJoin(fs.fsPath, bucket, object)) + fsRemoveAll(ctx, uploadIDDir) + fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object)) if err != nil { - return oi, toObjectErr(errors.Trace(err), bucket, object) + return oi, toObjectErr(err, bucket, object) } return fsMeta.ToObjectInfo(bucket, object, fi), nil @@ -659,12 +682,12 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string, // no affect and further requests to the same uploadID would not be // honored. func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error { - if err := checkAbortMultipartArgs(bucket, object, fs); err != nil { + if err := checkAbortMultipartArgs(ctx, bucket, object, fs); err != nil { return err } - if _, err := fs.statBucketDir(bucket); err != nil { - return toObjectErr(errors.Trace(err), bucket) + if _, err := fs.statBucketDir(ctx, bucket); err != nil { + return toObjectErr(err, bucket) } fs.appendFileMapMu.Lock() @@ -673,16 +696,16 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u uploadIDDir := fs.getUploadIDDir(bucket, object, uploadID) // Just check if the uploadID exists to avoid copy if it doesn't. - _, err := fsStatFile(pathJoin(uploadIDDir, fs.metaJSONFile)) + _, err := fsStatFile(ctx, pathJoin(uploadIDDir, fs.metaJSONFile)) if err != nil { if errors.Cause(err) == errFileNotFound || errors.Cause(err) == errFileAccessDenied { - return errors.Trace(InvalidUploadID{UploadID: uploadID}) + return InvalidUploadID{UploadID: uploadID} } - return toObjectErr(errors.Trace(err), bucket, object) + return toObjectErr(err, bucket, object) } // Ignore the error returned as Windows fails to remove directory if a file in it // is Open()ed by the backgroundAppend() - fsRemoveAll(uploadIDDir) + fsRemoveAll(ctx, uploadIDDir) return nil } @@ -690,7 +713,7 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u // Removes multipart uploads if any older than `expiry` duration // on all buckets for every `cleanupInterval`, this function is // blocking and should be run in a go-routine. -func (fs *FSObjects) cleanupStaleMultipartUploads(cleanupInterval, expiry time.Duration, doneCh chan struct{}) { +func (fs *FSObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh chan struct{}) { ticker := time.NewTicker(cleanupInterval) for { select { @@ -710,12 +733,12 @@ func (fs *FSObjects) cleanupStaleMultipartUploads(cleanupInterval, expiry time.D continue } for _, uploadID := range uploadIDs { - fi, err := fsStatDir(pathJoin(fs.fsPath, minioMetaMultipartBucket, entry, uploadID)) + fi, err := fsStatDir(ctx, pathJoin(fs.fsPath, minioMetaMultipartBucket, entry, uploadID)) if err != nil { continue } if now.Sub(fi.ModTime()) > expiry { - fsRemoveAll(pathJoin(fs.fsPath, minioMetaMultipartBucket, entry, uploadID)) + fsRemoveAll(ctx, pathJoin(fs.fsPath, minioMetaMultipartBucket, entry, uploadID)) } } } diff --git a/cmd/fs-v1-multipart_test.go b/cmd/fs-v1-multipart_test.go index 29c30b2ac..6edc85f93 100644 --- a/cmd/fs-v1-multipart_test.go +++ b/cmd/fs-v1-multipart_test.go @@ -49,7 +49,7 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) { t.Fatal("Unexpected err: ", err) } - go fs.cleanupStaleMultipartUploads(20*time.Millisecond, 0, globalServiceDoneCh) + go fs.cleanupStaleMultipartUploads(context.Background(), 20*time.Millisecond, 0, globalServiceDoneCh) // Wait for 40ms such that - we have given enough time for // cleanup routine to kick in. diff --git a/cmd/fs-v1-rwpool.go b/cmd/fs-v1-rwpool.go index b516232fb..bf31518b2 100644 --- a/cmd/fs-v1-rwpool.go +++ b/cmd/fs-v1-rwpool.go @@ -17,10 +17,12 @@ package cmd import ( + "context" "os" pathutil "path" "sync" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/lock" ) @@ -48,7 +50,9 @@ func (fsi *fsIOPool) lookupToRead(path string) (*lock.RLockedFile, bool) { // If the file is closed and not removed from map is a bug. if rlkFile.IsClosed() { // Log this as an error. - errorIf(errUnexpected, "Unexpected entry found on the map %s", path) + reqInfo := (&logger.ReqInfo{}).AppendTags("path", path) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, errUnexpected) // Purge the cached lock path from map. delete(fsi.readersMap, path) diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index 6dd6e7afe..98a23cb3d 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -32,6 +32,7 @@ import ( "time" "github.com/minio/minio-go/pkg/policy" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/hash" "github.com/minio/minio/pkg/lock" @@ -97,6 +98,7 @@ func initMetaVolumeFS(fsPath, fsUUID string) error { // NewFSObjectLayer - initialize new fs object layer. func NewFSObjectLayer(fsPath string) (ObjectLayer, error) { + ctx := context.Background() if fsPath == "" { return nil, errInvalidArgument } @@ -142,7 +144,7 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) { } // Initialize `format.json`, this function also returns. - rlk, err := initFormatFS(fsPath) + rlk, err := initFormatFS(ctx, fsPath) if err != nil { return nil, err } @@ -177,7 +179,7 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) { return nil, fmt.Errorf("Unable to initialize event notification. %s", err) } - go fs.cleanupStaleMultipartUploads(globalMultipartCleanupInterval, globalMultipartExpiry, globalServiceDoneCh) + go fs.cleanupStaleMultipartUploads(ctx, globalMultipartCleanupInterval, globalMultipartExpiry, globalServiceDoneCh) // Return successfully initialized object layer. return fs, nil @@ -188,13 +190,14 @@ func (fs *FSObjects) Shutdown(ctx context.Context) error { fs.fsFormatRlk.Close() // Cleanup and delete tmp uuid. - return fsRemoveAll(pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID)) + return fsRemoveAll(ctx, pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID)) } // StorageInfo - returns underlying storage statistics. func (fs *FSObjects) StorageInfo(ctx context.Context) StorageInfo { info, err := getDiskInfo((fs.fsPath)) - errorIf(err, "Unable to get disk info %#v", fs.fsPath) + logger.GetReqInfo(ctx).AppendTags("path", fs.fsPath) + logger.LogIf(ctx, err) storageInfo := StorageInfo{ Total: info.Total, Free: info.Free, @@ -220,22 +223,24 @@ func (fs *FSObjects) ClearLocks(ctx context.Context, info []VolumeLockInfo) erro // getBucketDir - will convert incoming bucket names to // corresponding valid bucket names on the backend in a platform // compatible way for all operating systems. -func (fs *FSObjects) getBucketDir(bucket string) (string, error) { +func (fs *FSObjects) getBucketDir(ctx context.Context, bucket string) (string, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return "", errors.Trace(BucketNameInvalid{Bucket: bucket}) + err := BucketNameInvalid{Bucket: bucket} + logger.LogIf(ctx, err) + return "", err } bucketDir := pathJoin(fs.fsPath, bucket) return bucketDir, nil } -func (fs *FSObjects) statBucketDir(bucket string) (os.FileInfo, error) { - bucketDir, err := fs.getBucketDir(bucket) +func (fs *FSObjects) statBucketDir(ctx context.Context, bucket string) (os.FileInfo, error) { + bucketDir, err := fs.getBucketDir(ctx, bucket) if err != nil { return nil, err } - st, err := fsStatVolume(bucketDir) + st, err := fsStatVolume(ctx, bucketDir) if err != nil { return nil, err } @@ -250,12 +255,12 @@ func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket, locatio return err } defer bucketLock.Unlock() - bucketDir, err := fs.getBucketDir(bucket) + bucketDir, err := fs.getBucketDir(ctx, bucket) if err != nil { return toObjectErr(err, bucket) } - if err = fsMkdir(bucketDir); err != nil { + if err = fsMkdir(ctx, bucketDir); err != nil { return toObjectErr(err, bucket) } @@ -269,7 +274,7 @@ func (fs *FSObjects) GetBucketInfo(ctx context.Context, bucket string) (bi Bucke return bi, e } defer bucketLock.RUnlock() - st, err := fs.statBucketDir(bucket) + st, err := fs.statBucketDir(ctx, bucket) if err != nil { return bi, toObjectErr(err, bucket) } @@ -285,12 +290,14 @@ func (fs *FSObjects) GetBucketInfo(ctx context.Context, bucket string) (bi Bucke // ListBuckets - list all s3 compatible buckets (directories) at fsPath. func (fs *FSObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) { if err := checkPathLength(fs.fsPath); err != nil { - return nil, errors.Trace(err) + logger.LogIf(ctx, err) + return nil, err } var bucketInfos []BucketInfo entries, err := readDir((fs.fsPath)) if err != nil { - return nil, toObjectErr(errors.Trace(errDiskNotFound)) + logger.LogIf(ctx, errDiskNotFound) + return nil, toObjectErr(errDiskNotFound) } for _, entry := range entries { @@ -299,7 +306,7 @@ func (fs *FSObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) { continue } var fi os.FileInfo - fi, err = fsStatVolume(pathJoin(fs.fsPath, entry)) + fi, err = fsStatVolume(ctx, pathJoin(fs.fsPath, entry)) // There seems like no practical reason to check for errors // at this point, if there are indeed errors we can simply // just ignore such buckets and list only those which @@ -327,27 +334,28 @@ func (fs *FSObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) { func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string) error { bucketLock := fs.nsMutex.NewNSLock(bucket, "") if err := bucketLock.GetLock(globalObjectTimeout); err != nil { + logger.LogIf(ctx, err) return err } defer bucketLock.Unlock() - bucketDir, err := fs.getBucketDir(bucket) + bucketDir, err := fs.getBucketDir(ctx, bucket) if err != nil { return toObjectErr(err, bucket) } // Attempt to delete regular bucket. - if err = fsRemoveDir(bucketDir); err != nil { + if err = fsRemoveDir(ctx, bucketDir); err != nil { return toObjectErr(err, bucket) } // Cleanup all the bucket metadata. minioMetadataBucketDir := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket) - if err = fsRemoveAll(minioMetadataBucketDir); err != nil { + if err = fsRemoveAll(ctx, minioMetadataBucketDir); err != nil { return toObjectErr(err, bucket) } // Delete all bucket metadata. - deleteBucketMetadata(bucket, fs) + deleteBucketMetadata(ctx, bucket, fs) return nil } @@ -380,7 +388,7 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu } defer objectSRLock.RUnlock() } - if _, err := fs.statBucketDir(srcBucket); err != nil { + if _, err := fs.statBucketDir(ctx, srcBucket); err != nil { return oi, toObjectErr(err, srcBucket) } @@ -391,14 +399,15 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, srcBucket, srcObject, fs.metaJSONFile) wlk, err := fs.rwPool.Write(fsMetaPath) if err != nil { - return oi, toObjectErr(errors.Trace(err), srcBucket, srcObject) + logger.LogIf(ctx, err) + return oi, toObjectErr(err, srcBucket, srcObject) } // This close will allow for locks to be synchronized on `fs.json`. defer wlk.Close() // Save objects' metadata in `fs.json`. fsMeta := newFSMetaV1() - if _, err = fsMeta.ReadFrom(wlk); err != nil { + if _, err = fsMeta.ReadFrom(ctx, wlk); err != nil { return oi, toObjectErr(err, srcBucket, srcObject) } @@ -409,7 +418,7 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu } // Stat the file to get file size. - fi, err := fsStatFile(pathJoin(fs.fsPath, srcBucket, srcObject)) + fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, srcBucket, srcObject)) if err != nil { return oi, toObjectErr(err, srcBucket, srcObject) } @@ -419,20 +428,20 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu } go func() { - if gerr := fs.getObject(srcBucket, srcObject, 0, srcInfo.Size, srcInfo.Writer, srcInfo.ETag, !cpSrcDstSame); gerr != nil { + if gerr := fs.getObject(ctx, srcBucket, srcObject, 0, srcInfo.Size, srcInfo.Writer, srcInfo.ETag, !cpSrcDstSame); gerr != nil { if gerr = srcInfo.Writer.Close(); gerr != nil { - errorIf(gerr, "Unable to read the object %s/%s.", srcBucket, srcObject) + logger.LogIf(ctx, gerr) } return } // Close writer explicitly signalling we wrote all data. if gerr := srcInfo.Writer.Close(); gerr != nil { - errorIf(gerr, "Unable to read the object %s/%s.", srcBucket, srcObject) + logger.LogIf(ctx, gerr) return } }() - objInfo, err := fs.putObject(dstBucket, dstObject, srcInfo.Reader, srcInfo.UserDefined) + objInfo, err := fs.putObject(ctx, dstBucket, dstObject, srcInfo.Reader, srcInfo.UserDefined) if err != nil { return oi, toObjectErr(err, dstBucket, dstObject) } @@ -447,39 +456,43 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu // startOffset indicates the starting read location of the object. // length indicates the total length of the object. func (fs *FSObjects) GetObject(ctx context.Context, bucket, object string, offset int64, length int64, writer io.Writer, etag string) (err error) { - if err = checkGetObjArgs(bucket, object); err != nil { + if err = checkGetObjArgs(ctx, bucket, object); err != nil { return err } // Lock the object before reading. objectLock := fs.nsMutex.NewNSLock(bucket, object) if err := objectLock.GetRLock(globalObjectTimeout); err != nil { + logger.LogIf(ctx, err) return err } defer objectLock.RUnlock() - return fs.getObject(bucket, object, offset, length, writer, etag, true) + return fs.getObject(ctx, bucket, object, offset, length, writer, etag, true) } // getObject - wrapper for GetObject -func (fs *FSObjects) getObject(bucket, object string, offset int64, length int64, writer io.Writer, etag string, lock bool) (err error) { - if _, err = fs.statBucketDir(bucket); err != nil { +func (fs *FSObjects) getObject(ctx context.Context, bucket, object string, offset int64, length int64, writer io.Writer, etag string, lock bool) (err error) { + if _, err = fs.statBucketDir(ctx, bucket); err != nil { return toObjectErr(err, bucket) } // Offset cannot be negative. if offset < 0 { - return toObjectErr(errors.Trace(errUnexpected), bucket, object) + logger.LogIf(ctx, errUnexpected) + return toObjectErr(errUnexpected, bucket, object) } // Writer cannot be nil. if writer == nil { - return toObjectErr(errors.Trace(errUnexpected), bucket, object) + logger.LogIf(ctx, errUnexpected) + return toObjectErr(errUnexpected, bucket, object) } // If its a directory request, we return an empty body. if hasSuffix(object, slashSeparator) { _, err = writer.Write([]byte("")) - return toObjectErr(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return toObjectErr(err, bucket, object) } if bucket != minioMetaBucket { @@ -487,25 +500,27 @@ func (fs *FSObjects) getObject(bucket, object string, offset int64, length int64 if lock { _, err = fs.rwPool.Open(fsMetaPath) if err != nil && err != errFileNotFound { - return toObjectErr(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return toObjectErr(err, bucket, object) } defer fs.rwPool.Close(fsMetaPath) } } if etag != "" { - objEtag, perr := fs.getObjectETag(bucket, object, lock) + objEtag, perr := fs.getObjectETag(ctx, bucket, object, lock) if perr != nil { - return toObjectErr(errors.Trace(perr), bucket, object) + return toObjectErr(perr, bucket, object) } if objEtag != etag { - return toObjectErr(errors.Trace(InvalidETag{}), bucket, object) + logger.LogIf(ctx, InvalidETag{}) + return toObjectErr(InvalidETag{}, bucket, object) } } // Read the object, doesn't exist returns an s3 compatible error. fsObjPath := pathJoin(fs.fsPath, bucket, object) - reader, size, err := fsOpenFile(fsObjPath, offset) + reader, size, err := fsOpenFile(ctx, fsObjPath, offset) if err != nil { return toObjectErr(err, bucket, object) } @@ -523,21 +538,23 @@ func (fs *FSObjects) getObject(bucket, object string, offset int64, length int64 // Reply back invalid range if the input offset and length fall out of range. if offset > size || offset+length > size { - return errors.Trace(InvalidRange{offset, length, size}) + err = InvalidRange{offset, length, size} + logger.LogIf(ctx, err) + return err } // Allocate a staging buffer. buf := make([]byte, int(bufSize)) _, err = io.CopyBuffer(writer, io.LimitReader(reader, length), buf) - - return toObjectErr(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return toObjectErr(err, bucket, object) } // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo. -func (fs *FSObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e error) { +func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) { fsMeta := fsMetaV1{} - fi, err := fsStatDir(pathJoin(fs.fsPath, bucket, object)) + fi, err := fsStatDir(ctx, pathJoin(fs.fsPath, bucket, object)) if err != nil && errors.Cause(err) != errFileAccessDenied { return oi, toObjectErr(err, bucket, object) } @@ -547,6 +564,7 @@ func (fs *FSObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e erro if hasSuffix(object, slashSeparator) { return fsMeta.ToObjectInfo(bucket, object, fi), nil } + logger.LogIf(ctx, errFileNotFound) return oi, toObjectErr(errFileNotFound, bucket, object) } @@ -558,7 +576,7 @@ func (fs *FSObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e erro if err == nil { // Read from fs metadata only if it exists. defer fs.rwPool.Close(fsMetaPath) - if _, rerr := fsMeta.ReadFrom(rlk.LockedFile); rerr != nil { + if _, rerr := fsMeta.ReadFrom(ctx, rlk.LockedFile); rerr != nil { // `fs.json` can be empty due to previously failed // PutObject() transaction, if we arrive at such // a situation we just ignore and continue. @@ -570,11 +588,12 @@ func (fs *FSObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e erro // Ignore if `fs.json` is not available, this is true for pre-existing data. if err != nil && err != errFileNotFound { - return oi, toObjectErr(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return oi, toObjectErr(err, bucket, object) } // Stat the file to get file size. - fi, err = fsStatFile(pathJoin(fs.fsPath, bucket, object)) + fi, err = fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object)) if err != nil { return oi, toObjectErr(err, bucket, object) } @@ -591,27 +610,27 @@ func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string) ( } defer objectLock.RUnlock() - if err := checkGetObjArgs(bucket, object); err != nil { + if err := checkGetObjArgs(ctx, bucket, object); err != nil { return oi, err } - if _, err := fs.statBucketDir(bucket); err != nil { + if _, err := fs.statBucketDir(ctx, bucket); err != nil { return oi, toObjectErr(err, bucket) } - return fs.getObjectInfo(bucket, object) + return fs.getObjectInfo(ctx, bucket, object) } // This function does the following check, suppose // object is "a/b/c/d", stat makes sure that objects ""a/b/c"" // "a/b" and "a" do not exist. -func (fs *FSObjects) parentDirIsObject(bucket, parent string) bool { +func (fs *FSObjects) parentDirIsObject(ctx context.Context, bucket, parent string) bool { var isParentDirObject func(string) bool isParentDirObject = func(p string) bool { if p == "." || p == "/" { return false } - if _, err := fsStatFile(pathJoin(fs.fsPath, bucket, p)); err == nil { + if _, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, p)); err == nil { // If there is already a file at prefix "p", return true. return true } @@ -627,20 +646,21 @@ func (fs *FSObjects) parentDirIsObject(bucket, parent string) bool { // Additionally writes `fs.json` which carries the necessary metadata // for future object operations. func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) { - if err := checkPutObjectArgs(bucket, object, fs, data.Size()); err != nil { + if err := checkPutObjectArgs(ctx, bucket, object, fs, data.Size()); err != nil { return ObjectInfo{}, err } // Lock the object. objectLock := fs.nsMutex.NewNSLock(bucket, object) if err := objectLock.GetLock(globalObjectTimeout); err != nil { + logger.LogIf(ctx, err) return objInfo, err } defer objectLock.Unlock() - return fs.putObject(bucket, object, data, metadata) + return fs.putObject(ctx, bucket, object, data, metadata) } // putObject - wrapper for PutObject -func (fs *FSObjects) putObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) { +func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) { // No metadata is set, allocate a new one. meta := make(map[string]string) for k, v := range metadata { @@ -649,7 +669,7 @@ func (fs *FSObjects) putObject(bucket string, object string, data *hash.Reader, var err error // Validate if bucket name is valid and exists. - if _, err = fs.statBucketDir(bucket); err != nil { + if _, err = fs.statBucketDir(ctx, bucket); err != nil { return ObjectInfo{}, toObjectErr(err, bucket) } @@ -661,31 +681,35 @@ func (fs *FSObjects) putObject(bucket string, object string, data *hash.Reader, // and return success. if isObjectDir(object, data.Size()) { // Check if an object is present as one of the parent dir. - if fs.parentDirIsObject(bucket, path.Dir(object)) { - return ObjectInfo{}, toObjectErr(errors.Trace(errFileAccessDenied), bucket, object) + if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) { + logger.LogIf(ctx, errFileAccessDenied) + return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object) } if err = mkdirAll(pathJoin(fs.fsPath, bucket, object), 0777); err != nil { + logger.LogIf(ctx, err) return ObjectInfo{}, toObjectErr(err, bucket, object) } var fi os.FileInfo - if fi, err = fsStatDir(pathJoin(fs.fsPath, bucket, object)); err != nil { + if fi, err = fsStatDir(ctx, pathJoin(fs.fsPath, bucket, object)); err != nil { return ObjectInfo{}, toObjectErr(err, bucket, object) } return fsMeta.ToObjectInfo(bucket, object, fi), nil } - if err = checkPutObjectArgs(bucket, object, fs, data.Size()); err != nil { + if err = checkPutObjectArgs(ctx, bucket, object, fs, data.Size()); err != nil { return ObjectInfo{}, err } // Check if an object is present as one of the parent dir. - if fs.parentDirIsObject(bucket, path.Dir(object)) { - return ObjectInfo{}, toObjectErr(errors.Trace(errFileAccessDenied), bucket, object) + if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) { + logger.LogIf(ctx, errFileAccessDenied) + return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object) } // Validate input data size and it can never be less than zero. if data.Size() < 0 { - return ObjectInfo{}, errors.Trace(errInvalidArgument) + logger.LogIf(ctx, errInvalidArgument) + return ObjectInfo{}, errInvalidArgument } var wlk *lock.LockedFile @@ -695,7 +719,8 @@ func (fs *FSObjects) putObject(bucket string, object string, data *hash.Reader, fsMetaPath := pathJoin(bucketMetaDir, bucket, object, fs.metaJSONFile) wlk, err = fs.rwPool.Create(fsMetaPath) if err != nil { - return ObjectInfo{}, toObjectErr(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return ObjectInfo{}, toObjectErr(err, bucket, object) } // This close will allow for locks to be synchronized on `fs.json`. defer wlk.Close() @@ -703,7 +728,7 @@ func (fs *FSObjects) putObject(bucket string, object string, data *hash.Reader, // Remove meta file when PutObject encounters any error if retErr != nil { tmpDir := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID) - fsRemoveMeta(bucketMetaDir, fsMetaPath, tmpDir) + fsRemoveMeta(ctx, bucketMetaDir, fsMetaPath, tmpDir) } }() } @@ -721,9 +746,9 @@ func (fs *FSObjects) putObject(bucket string, object string, data *hash.Reader, buf := make([]byte, int(bufSize)) fsTmpObjPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, tempObj) - bytesWritten, err := fsCreateFile(fsTmpObjPath, data, buf, data.Size()) + bytesWritten, err := fsCreateFile(ctx, fsTmpObjPath, data, buf, data.Size()) if err != nil { - fsRemoveFile(fsTmpObjPath) + fsRemoveFile(ctx, fsTmpObjPath) return ObjectInfo{}, toObjectErr(err, bucket, object) } @@ -732,24 +757,24 @@ func (fs *FSObjects) putObject(bucket string, object string, data *hash.Reader, // Should return IncompleteBody{} error when reader has fewer // bytes than specified in request header. if bytesWritten < data.Size() { - fsRemoveFile(fsTmpObjPath) - return ObjectInfo{}, errors.Trace(IncompleteBody{}) + fsRemoveFile(ctx, fsTmpObjPath) + return ObjectInfo{}, IncompleteBody{} } // Delete the temporary object in the case of a // failure. If PutObject succeeds, then there would be // nothing to delete. - defer fsRemoveFile(fsTmpObjPath) + defer fsRemoveFile(ctx, fsTmpObjPath) // Entire object was written to the temp location, now it's safe to rename it to the actual location. fsNSObjPath := pathJoin(fs.fsPath, bucket, object) // Deny if WORM is enabled if globalWORMEnabled { - if _, err = fsStatFile(fsNSObjPath); err == nil { - return ObjectInfo{}, errors.Trace(ObjectAlreadyExists{Bucket: bucket, Object: object}) + if _, err = fsStatFile(ctx, fsNSObjPath); err == nil { + return ObjectInfo{}, ObjectAlreadyExists{Bucket: bucket, Object: object} } } - if err = fsRenameFile(fsTmpObjPath, fsNSObjPath); err != nil { + if err = fsRenameFile(ctx, fsTmpObjPath, fsNSObjPath); err != nil { return ObjectInfo{}, toObjectErr(err, bucket, object) } @@ -761,7 +786,7 @@ func (fs *FSObjects) putObject(bucket string, object string, data *hash.Reader, } // Stat the file to fetch timestamp, size. - fi, err := fsStatFile(pathJoin(fs.fsPath, bucket, object)) + fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object)) if err != nil { return ObjectInfo{}, toObjectErr(err, bucket, object) } @@ -780,11 +805,11 @@ func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string) er } defer objectLock.Unlock() - if err := checkDelObjArgs(bucket, object); err != nil { + if err := checkDelObjArgs(ctx, bucket, object); err != nil { return err } - if _, err := fs.statBucketDir(bucket); err != nil { + if _, err := fs.statBucketDir(ctx, bucket); err != nil { return toObjectErr(err, bucket) } @@ -797,18 +822,19 @@ func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string) er defer rwlk.Close() } if lerr != nil && lerr != errFileNotFound { - return toObjectErr(errors.Trace(lerr), bucket, object) + logger.LogIf(ctx, lerr) + return toObjectErr(lerr, bucket, object) } } // Delete the object. - if err := fsDeleteFile(pathJoin(fs.fsPath, bucket), pathJoin(fs.fsPath, bucket, object)); err != nil { + if err := fsDeleteFile(ctx, pathJoin(fs.fsPath, bucket), pathJoin(fs.fsPath, bucket, object)); err != nil { return toObjectErr(err, bucket, object) } if bucket != minioMetaBucket { // Delete the metadata object. - err := fsDeleteFile(minioMetaBucketDir, fsMetaPath) + err := fsDeleteFile(ctx, minioMetaBucketDir, fsMetaPath) if err != nil && errors.Cause(err) != errFileNotFound { return toObjectErr(err, bucket, object) } @@ -836,7 +862,7 @@ func (fs *FSObjects) listDirFactory(isLeaf isLeafFunc) listDirFunc { // getObjectETag is a helper function, which returns only the md5sum // of the file on the disk. -func (fs *FSObjects) getObjectETag(bucket, entry string, lock bool) (string, error) { +func (fs *FSObjects) getObjectETag(ctx context.Context, bucket, entry string, lock bool) (string, error) { fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, entry, fs.metaJSONFile) var reader io.Reader @@ -848,7 +874,8 @@ func (fs *FSObjects) getObjectETag(bucket, entry string, lock bool) (string, err rlk, err := fs.rwPool.Open(fsMetaPath) // Ignore if `fs.json` is not available, this is true for pre-existing data. if err != nil && err != errFileNotFound { - return "", toObjectErr(errors.Trace(err), bucket, entry) + logger.LogIf(ctx, err) + return "", toObjectErr(err, bucket, entry) } // If file is not found, we don't need to proceed forward. @@ -862,16 +889,17 @@ func (fs *FSObjects) getObjectETag(bucket, entry string, lock bool) (string, err // Fetch the size of the underlying file. fi, err = rlk.LockedFile.Stat() if err != nil { - return "", toObjectErr(errors.Trace(err), bucket, entry) + logger.LogIf(ctx, err) + return "", toObjectErr(err, bucket, entry) } size = fi.Size() reader = io.NewSectionReader(rlk.LockedFile, 0, fi.Size()) } else { var err error - reader, size, err = fsOpenFile(fsMetaPath, 0) + reader, size, err = fsOpenFile(ctx, fsMetaPath, 0) if err != nil { - return "", toObjectErr(errors.Trace(err), bucket, entry) + return "", toObjectErr(err, bucket, entry) } } @@ -884,12 +912,14 @@ func (fs *FSObjects) getObjectETag(bucket, entry string, lock bool) (string, err fsMetaBuf, err := ioutil.ReadAll(reader) if err != nil { - return "", toObjectErr(errors.Trace(err), bucket, entry) + logger.LogIf(ctx, err) + return "", toObjectErr(err, bucket, entry) } // Check if FS metadata is valid, if not return error. if !isFSMetaValid(parseFSVersion(fsMetaBuf)) { - return "", toObjectErr(errors.Trace(errCorruptedFormat), bucket, entry) + logger.LogIf(ctx, errCorruptedFormat) + return "", toObjectErr(errCorruptedFormat, bucket, entry) } return extractETag(parseFSMetaMap(fsMetaBuf)), nil @@ -898,7 +928,7 @@ func (fs *FSObjects) getObjectETag(bucket, entry string, lock bool) (string, err // ListObjects - list all objects at prefix upto maxKeys., optionally delimited by '/'. Maintains the list pool // state for future re-entrant list requests. func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) { - if err := checkListObjsArgs(bucket, prefix, marker, delimiter, fs); err != nil { + if err := checkListObjsArgs(ctx, bucket, prefix, marker, delimiter, fs); err != nil { return loi, err } // Marker is set validate pre-condition. @@ -908,7 +938,7 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de return ListObjectsInfo{}, e } } - if _, err := fs.statBucketDir(bucket); err != nil { + if _, err := fs.statBucketDir(ctx, bucket); err != nil { return loi, err } @@ -942,10 +972,11 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de // Protect the entry from concurrent deletes, or renames. objectLock := fs.nsMutex.NewNSLock(bucket, entry) if err = objectLock.GetRLock(globalListingTimeout); err != nil { + logger.LogIf(ctx, err) return ObjectInfo{}, err } defer objectLock.RUnlock() - return fs.getObjectInfo(bucket, entry) + return fs.getObjectInfo(ctx, bucket, entry) } heal := false // true only for xl.ListObjectsHeal() @@ -959,7 +990,7 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de return !hasSuffix(object, slashSeparator) } listDir := fs.listDirFactory(isLeaf) - walkResultCh = startTreeWalk(bucket, prefix, marker, recursive, listDir, isLeaf, endWalkCh) + walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, endWalkCh) } var objInfos []ObjectInfo @@ -984,7 +1015,6 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de } objInfo, err := entryToObjectInfo(walkResult.entry) if err != nil { - errorIf(err, "Unable to fetch object info for %s", walkResult.entry) return loi, nil } nextMarker = objInfo.Name @@ -1018,34 +1048,39 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de // HealFormat - no-op for fs, Valid only for XL. func (fs *FSObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) { - return madmin.HealResultItem{}, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return madmin.HealResultItem{}, NotImplemented{} } // HealObject - no-op for fs. Valid only for XL. func (fs *FSObjects) HealObject(ctx context.Context, bucket, object string, dryRun bool) ( res madmin.HealResultItem, err error) { - return res, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return res, NotImplemented{} } // HealBucket - no-op for fs, Valid only for XL. func (fs *FSObjects) HealBucket(ctx context.Context, bucket string, dryRun bool) ([]madmin.HealResultItem, error) { - return nil, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return nil, NotImplemented{} } // ListObjectsHeal - list all objects to be healed. Valid only for XL func (fs *FSObjects) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) { - return loi, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return loi, NotImplemented{} } // ListBucketsHeal - list all buckets to be healed. Valid only for XL func (fs *FSObjects) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error) { - return []BucketInfo{}, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return []BucketInfo{}, NotImplemented{} } // SetBucketPolicy sets policy on bucket func (fs *FSObjects) SetBucketPolicy(ctx context.Context, bucket string, policy policy.BucketAccessPolicy) error { - return persistAndNotifyBucketPolicyChange(bucket, false, policy, fs) + return persistAndNotifyBucketPolicyChange(ctx, bucket, false, policy, fs) } // GetBucketPolicy will get policy on bucket @@ -1059,7 +1094,7 @@ func (fs *FSObjects) GetBucketPolicy(ctx context.Context, bucket string) (policy // DeleteBucketPolicy deletes all policies on bucket func (fs *FSObjects) DeleteBucketPolicy(ctx context.Context, bucket string) error { - return persistAndNotifyBucketPolicyChange(bucket, true, emptyBucketPolicy, fs) + return persistAndNotifyBucketPolicyChange(ctx, bucket, true, emptyBucketPolicy, fs) } // ListObjectsV2 lists all blobs in bucket filtered by prefix diff --git a/cmd/fs-v1_test.go b/cmd/fs-v1_test.go index 3c9ada128..30e836909 100644 --- a/cmd/fs-v1_test.go +++ b/cmd/fs-v1_test.go @@ -91,7 +91,7 @@ func TestFSParentDirIsObject(t *testing.T) { }, } for i, testCase := range testCases { - gotValue := fs.parentDirIsObject(bucketName, testCase.objectName) + gotValue := fs.parentDirIsObject(context.Background(), bucketName, testCase.objectName) if testCase.parentIsObject != gotValue { t.Errorf("Test %d: Unexpected value returned got %t, expected %t", i+1, gotValue, testCase.parentIsObject) } diff --git a/cmd/gateway-common.go b/cmd/gateway-common.go index e7f742e65..2d15abc6f 100644 --- a/cmd/gateway-common.go +++ b/cmd/gateway-common.go @@ -19,7 +19,6 @@ package cmd import ( "net/http" - "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/hash" minio "github.com/minio/minio-go" @@ -31,12 +30,6 @@ var ( // MustGetUUID function alias. MustGetUUID = mustGetUUID - - // ErrorIf provides errorIf function alias. - ErrorIf = errorIf - - // FatalIf provides fatalIf function alias. - FatalIf = fatalIf ) // AnonErrToObjectErr - converts standard http codes into meaningful object layer errors. @@ -262,16 +255,6 @@ func ErrorRespToObjectError(err error, params ...string) error { return nil } - e, ok := err.(*errors.Error) - if !ok { - // Code should be fixed if this function is called without doing traceError() - // Else handling different situations in this function makes this function complicated. - errorIf(err, "Expected type *Error") - return err - } - - err = e.Cause - bucket := "" object := "" if len(params) >= 1 { @@ -282,15 +265,14 @@ func ErrorRespToObjectError(err error, params ...string) error { } if isNetworkOrHostDown(err) { - e.Cause = BackendDown{} - return e + return BackendDown{} } minioErr, ok := err.(minio.ErrorResponse) if !ok { // We don't interpret non Minio errors. As minio errors will // have StatusCode to help to convert to object errors. - return e + return err } switch minioErr.Code { @@ -325,6 +307,5 @@ func ErrorRespToObjectError(err error, params ...string) error { err = PartTooSmall{} } - e.Cause = err - return e + return err } diff --git a/cmd/gateway-main.go b/cmd/gateway-main.go index 305a5d008..b7048c9a2 100644 --- a/cmd/gateway-main.go +++ b/cmd/gateway-main.go @@ -17,6 +17,8 @@ package cmd import ( + "context" + "errors" "fmt" "net/url" "os" @@ -28,7 +30,7 @@ import ( "github.com/gorilla/mux" "github.com/minio/cli" miniohttp "github.com/minio/minio/cmd/http" - "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" ) var ( @@ -100,10 +102,14 @@ func ValidateGatewayArguments(serverAddr, endpointAddr string) error { return nil } +func init() { + logger.Init(GOPATH) +} + // StartGateway - handler for 'minio gateway '. func StartGateway(ctx *cli.Context, gw Gateway) { if gw == nil { - fatalIf(errUnexpected, "Gateway implementation not initialized, exiting.") + logger.FatalIf(errUnexpected, "Gateway implementation not initialized, exiting.") } // Validate if we have access, secret set through environment. @@ -116,13 +122,13 @@ func StartGateway(ctx *cli.Context, gw Gateway) { // enable json and quite modes if jason flag is turned on. jsonFlag := ctx.IsSet("json") || ctx.GlobalIsSet("json") if jsonFlag { - log.EnableJSON() + logger.EnableJSON() } // Get quiet flag from command line argument. quietFlag := ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet") if quietFlag { - log.EnableQuiet() + logger.EnableQuiet() } // Fetch address option @@ -139,35 +145,34 @@ func StartGateway(ctx *cli.Context, gw Gateway) { // Validate if we have access, secret set through environment. if !globalIsEnvCreds { - errorIf(fmt.Errorf("Access and secret keys not set"), "Access and Secret keys should be set through ENVs for backend [%s]", gatewayName) + reqInfo := (&logger.ReqInfo{}).AppendTags("gatewayName", gatewayName) + contxt := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(contxt, errors.New("Access and Secret keys should be set through ENVs for backend")) cli.ShowCommandHelpAndExit(ctx, gatewayName, 1) } // Create certs path. - fatalIf(createConfigDir(), "Unable to create configuration directories.") + logger.FatalIf(createConfigDir(), "Unable to create configuration directories.") // Initialize gateway config. initConfig() - // Init the error tracing module. - errors.Init(GOPATH, "github.com/minio/minio") - // Check and load SSL certificates. var err error globalPublicCerts, globalRootCAs, globalTLSCertificate, globalIsSSL, err = getSSLConfig() - fatalIf(err, "Invalid SSL certificate file") + logger.FatalIf(err, "Invalid SSL certificate file") // Set system resources to maximum. - errorIf(setMaxResources(), "Unable to change resource limit") + logger.LogIf(context.Background(), setMaxResources()) initNSLock(false) // Enable local namespace lock. // Initialize notification system. globalNotificationSys, err = NewNotificationSys(globalServerConfig, EndpointList{}) - fatalIf(err, "Unable to initialize notification system.") + logger.FatalIf(err, "Unable to initialize notification system.") newObject, err := gw.NewGatewayLayer(globalServerConfig.GetCredential()) - fatalIf(err, "Unable to initialize gateway layer") + logger.FatalIf(err, "Unable to initialize gateway layer") router := mux.NewRouter().SkipClean(true) @@ -176,7 +181,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) { // Register web router when its enabled. if globalIsBrowserEnabled { - fatalIf(registerWebRouter(router), "Unable to configure web browser") + logger.FatalIf(registerWebRouter(router), "Unable to configure web browser") } // Add API router. @@ -204,7 +209,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) { // Print a warning message if gateway is not ready for production before the startup banner. if !gw.Production() { - log.Println(colorYellow("\n *** Warning: Not Ready for Production ***")) + logger.Println(colorYellow("\n *** Warning: Not Ready for Production ***")) } // Print gateway startup message. diff --git a/cmd/gateway-startup-msg.go b/cmd/gateway-startup-msg.go index ac76ad6b4..6e000b590 100644 --- a/cmd/gateway-startup-msg.go +++ b/cmd/gateway-startup-msg.go @@ -20,6 +20,8 @@ import ( "context" "fmt" "strings" + + "github.com/minio/minio/cmd/logger" ) // Prints the formatted startup message. @@ -54,12 +56,12 @@ func printGatewayCommonMsg(apiEndpoints []string) { apiEndpointStr := strings.Join(apiEndpoints, " ") // Colorize the message and print. - log.Println(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr))) - log.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey))) - log.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey))) + logger.Println(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr))) + logger.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey))) + logger.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey))) if globalIsBrowserEnabled { - log.Println(colorBlue("\nBrowser Access:")) - log.Println(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr)) + logger.Println(colorBlue("\nBrowser Access:")) + logger.Println(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr)) } } diff --git a/cmd/gateway-unsupported.go b/cmd/gateway-unsupported.go index 5af4e968f..6f1f760f9 100644 --- a/cmd/gateway-unsupported.go +++ b/cmd/gateway-unsupported.go @@ -21,7 +21,7 @@ import ( "time" "github.com/minio/minio-go/pkg/policy" - "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/hash" "github.com/minio/minio/pkg/madmin" ) @@ -31,105 +31,125 @@ type GatewayUnsupported struct{} // ListMultipartUploads lists all multipart uploads. func (a GatewayUnsupported) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi ListMultipartsInfo, err error) { - return lmi, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return lmi, NotImplemented{} } // NewMultipartUpload upload object in multiple parts func (a GatewayUnsupported) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string) (uploadID string, err error) { - return "", errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return "", NotImplemented{} } // CopyObjectPart copy part of object to uploadID for another object func (a GatewayUnsupported) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string, partID int, startOffset, length int64, srcInfo ObjectInfo) (pi PartInfo, err error) { - return pi, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return pi, NotImplemented{} } // PutObjectPart puts a part of object in bucket func (a GatewayUnsupported) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader) (pi PartInfo, err error) { - return pi, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return pi, NotImplemented{} } // ListObjectParts returns all object parts for specified object in specified bucket func (a GatewayUnsupported) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi ListPartsInfo, err error) { - return lpi, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return lpi, NotImplemented{} } // AbortMultipartUpload aborts a ongoing multipart upload func (a GatewayUnsupported) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string) error { - return errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return NotImplemented{} } // CompleteMultipartUpload completes ongoing multipart upload and finalizes object func (a GatewayUnsupported) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []CompletePart) (oi ObjectInfo, err error) { - return oi, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return oi, NotImplemented{} } // SetBucketPolicy sets policy on bucket func (a GatewayUnsupported) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error { - return errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return NotImplemented{} } // GetBucketPolicy will get policy on bucket func (a GatewayUnsupported) GetBucketPolicy(ctx context.Context, bucket string) (bal policy.BucketAccessPolicy, err error) { - return bal, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return bal, NotImplemented{} } // DeleteBucketPolicy deletes all policies on bucket func (a GatewayUnsupported) DeleteBucketPolicy(ctx context.Context, bucket string) error { - return errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return NotImplemented{} } // HealFormat - Not implemented stub func (a GatewayUnsupported) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) { - return madmin.HealResultItem{}, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return madmin.HealResultItem{}, NotImplemented{} } // HealBucket - Not implemented stub func (a GatewayUnsupported) HealBucket(ctx context.Context, bucket string, dryRun bool) ([]madmin.HealResultItem, error) { - return nil, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return nil, NotImplemented{} } // ListBucketsHeal - Not implemented stub func (a GatewayUnsupported) ListBucketsHeal(ctx context.Context) (buckets []BucketInfo, err error) { - return nil, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return nil, NotImplemented{} } // HealObject - Not implemented stub func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, dryRun bool) (h madmin.HealResultItem, e error) { - return h, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return h, NotImplemented{} } // ListObjectsV2 - Not implemented stub func (a GatewayUnsupported) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) { - return result, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return result, NotImplemented{} } // ListObjectsHeal - Not implemented stub func (a GatewayUnsupported) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) { - return loi, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return loi, NotImplemented{} } // CopyObject copies a blob from source container to destination container. func (a GatewayUnsupported) CopyObject(ctx context.Context, srcBucket string, srcObject string, destBucket string, destObject string, srcInfo ObjectInfo) (objInfo ObjectInfo, err error) { - return objInfo, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return objInfo, NotImplemented{} } // Locking operations // ListLocks lists namespace locks held in object layer func (a GatewayUnsupported) ListLocks(ctx context.Context, bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) { - return []VolumeLockInfo{}, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return []VolumeLockInfo{}, NotImplemented{} } // ClearLocks clears namespace locks held in object layer func (a GatewayUnsupported) ClearLocks(ctx context.Context, info []VolumeLockInfo) error { - return errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return NotImplemented{} } // RefreshBucketPolicy refreshes cache policy with what's on disk. func (a GatewayUnsupported) RefreshBucketPolicy(ctx context.Context, bucket string) error { - return errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return NotImplemented{} } // IsNotificationSupported returns whether bucket notification is applicable for this layer. diff --git a/cmd/gateway/azure/gateway-azure.go b/cmd/gateway/azure/gateway-azure.go index d7b48afe5..eecfe70e3 100644 --- a/cmd/gateway/azure/gateway-azure.go +++ b/cmd/gateway/azure/gateway-azure.go @@ -35,6 +35,7 @@ import ( humanize "github.com/dustin/go-humanize" "github.com/minio/cli" "github.com/minio/minio-go/pkg/policy" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/hash" @@ -118,7 +119,7 @@ func azureGatewayMain(ctx *cli.Context) { // Validate gateway arguments. host := ctx.Args().First() // Validate gateway arguments. - minio.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument") + logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument") minio.StartGateway(ctx, &Azure{host}) } @@ -181,11 +182,12 @@ func (g *Azure) Production() bool { // copied into BlobProperties. // // Header names are canonicalized as in http.Header. -func s3MetaToAzureProperties(s3Metadata map[string]string) (storage.BlobMetadata, +func s3MetaToAzureProperties(ctx context.Context, s3Metadata map[string]string) (storage.BlobMetadata, storage.BlobProperties, error) { for k := range s3Metadata { if strings.Contains(k, "--") { - return storage.BlobMetadata{}, storage.BlobProperties{}, errors.Trace(minio.UnsupportedMetadata{}) + logger.LogIf(ctx, minio.UnsupportedMetadata{}) + return storage.BlobMetadata{}, storage.BlobProperties{}, minio.UnsupportedMetadata{} } } @@ -300,15 +302,6 @@ func azureToObjectError(err error, params ...string) error { return nil } - e, ok := err.(*errors.Error) - if !ok { - // Code should be fixed if this function is called without doing errors.Trace() - // Else handling different situations in this function makes this function complicated. - minio.ErrorIf(err, "Expected type *Error") - return err - } - - err = e.Cause bucket := "" object := "" if len(params) >= 1 { @@ -322,7 +315,7 @@ func azureToObjectError(err error, params ...string) error { if !ok { // We don't interpret non Azure errors. As azure errors will // have StatusCode to help to convert to object errors. - return e + return err } switch azureErr.Code { @@ -349,8 +342,7 @@ func azureToObjectError(err error, params ...string) error { err = minio.BucketNameInvalid{Bucket: bucket} } } - e.Cause = err - return e + return err } // mustGetAzureUploadID - returns new upload ID which is hex encoded 8 bytes random value. @@ -371,17 +363,23 @@ func mustGetAzureUploadID() string { } // checkAzureUploadID - returns error in case of given string is upload ID. -func checkAzureUploadID(uploadID string) (err error) { +func checkAzureUploadID(ctx context.Context, uploadID string) (err error) { if len(uploadID) != 16 { - return errors.Trace(minio.MalformedUploadID{ + logger.LogIf(ctx, minio.MalformedUploadID{ UploadID: uploadID, }) + return minio.MalformedUploadID{ + UploadID: uploadID, + } } if _, err = hex.DecodeString(uploadID); err != nil { - return errors.Trace(minio.MalformedUploadID{ + logger.LogIf(ctx, minio.MalformedUploadID{ UploadID: uploadID, }) + return minio.MalformedUploadID{ + UploadID: uploadID, + } } return nil @@ -438,7 +436,8 @@ func (a *azureObjects) MakeBucketWithLocation(ctx context.Context, bucket, locat err := container.Create(&storage.CreateContainerOptions{ Access: storage.ContainerAccessTypePrivate, }) - return azureToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return azureToObjectError(err, bucket) } // GetBucketInfo - Get bucket metadata.. @@ -448,7 +447,8 @@ func (a *azureObjects) GetBucketInfo(ctx context.Context, bucket string) (bi min // in azure documentation, so we will simply use the same function here. // Ref - https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata if !minio.IsValidBucketName(bucket) { - return bi, errors.Trace(minio.BucketNameInvalid{Bucket: bucket}) + logger.LogIf(ctx, minio.BucketNameInvalid{Bucket: bucket}) + return bi, minio.BucketNameInvalid{Bucket: bucket} } // Azure does not have an equivalent call, hence use @@ -457,7 +457,8 @@ func (a *azureObjects) GetBucketInfo(ctx context.Context, bucket string) (bi min Prefix: bucket, }) if err != nil { - return bi, azureToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return bi, azureToObjectError(err, bucket) } for _, container := range resp.Containers { if container.Name == bucket { @@ -470,19 +471,22 @@ func (a *azureObjects) GetBucketInfo(ctx context.Context, bucket string) (bi min } // else continue } } - return bi, errors.Trace(minio.BucketNotFound{Bucket: bucket}) + logger.LogIf(ctx, minio.BucketNotFound{Bucket: bucket}) + return bi, minio.BucketNotFound{Bucket: bucket} } // ListBuckets - Lists all azure containers, uses Azure equivalent ListContainers. func (a *azureObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketInfo, err error) { resp, err := a.client.ListContainers(storage.ListContainersParameters{}) if err != nil { - return nil, azureToObjectError(errors.Trace(err)) + logger.LogIf(ctx, err) + return nil, azureToObjectError(err) } for _, container := range resp.Containers { t, e := time.Parse(time.RFC1123, container.Properties.LastModified) if e != nil { - return nil, errors.Trace(e) + logger.LogIf(ctx, e) + return nil, e } buckets = append(buckets, minio.BucketInfo{ Name: container.Name, @@ -495,7 +499,9 @@ func (a *azureObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketI // DeleteBucket - delete a container on azure, uses Azure equivalent DeleteContainer. func (a *azureObjects) DeleteBucket(ctx context.Context, bucket string) error { container := a.client.GetContainerReference(bucket) - return azureToObjectError(errors.Trace(container.Delete(nil)), bucket) + err := container.Delete(nil) + logger.LogIf(ctx, err) + return azureToObjectError(err, bucket) } // ListObjects - lists all blobs on azure with in a container filtered by prefix @@ -512,7 +518,8 @@ func (a *azureObjects) ListObjects(ctx context.Context, bucket, prefix, marker, MaxResults: uint(maxKeys), }) if err != nil { - return result, azureToObjectError(errors.Trace(err), bucket, prefix) + logger.LogIf(ctx, err) + return result, azureToObjectError(err, bucket, prefix) } for _, object := range resp.Blobs { @@ -580,7 +587,8 @@ func (a *azureObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin func (a *azureObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error { // startOffset cannot be negative. if startOffset < 0 { - return azureToObjectError(errors.Trace(minio.InvalidRange{}), bucket, object) + logger.LogIf(ctx, minio.InvalidRange{}) + return azureToObjectError(minio.InvalidRange{}, bucket, object) } blobRange := &storage.BlobRange{Start: uint64(startOffset)} @@ -599,11 +607,13 @@ func (a *azureObjects) GetObject(ctx context.Context, bucket, object string, sta }) } if err != nil { - return azureToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return azureToObjectError(err, bucket, object) } _, err = io.Copy(writer, rc) rc.Close() - return errors.Trace(err) + logger.LogIf(ctx, err) + return err } // GetObjectInfo - reads blob metadata properties and replies back minio.ObjectInfo, @@ -612,7 +622,8 @@ func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string) blob := a.client.GetContainerReference(bucket).GetBlobReference(object) err = blob.GetProperties(nil) if err != nil { - return objInfo, azureToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return objInfo, azureToObjectError(err, bucket, object) } return minio.ObjectInfo{ @@ -631,13 +642,14 @@ func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string) // uses Azure equivalent CreateBlockBlobFromReader. func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { blob := a.client.GetContainerReference(bucket).GetBlobReference(object) - blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(metadata) + blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(ctx, metadata) if err != nil { return objInfo, azureToObjectError(err, bucket, object) } err = blob.CreateBlockBlobFromReader(data, nil) if err != nil { - return objInfo, azureToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return objInfo, azureToObjectError(err, bucket, object) } return a.GetObjectInfo(ctx, bucket, object) } @@ -647,19 +659,21 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, dat func (a *azureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) { srcBlobURL := a.client.GetContainerReference(srcBucket).GetBlobReference(srcObject).GetURL() destBlob := a.client.GetContainerReference(destBucket).GetBlobReference(destObject) - azureMeta, props, err := s3MetaToAzureProperties(srcInfo.UserDefined) + azureMeta, props, err := s3MetaToAzureProperties(ctx, srcInfo.UserDefined) if err != nil { return objInfo, azureToObjectError(err, srcBucket, srcObject) } destBlob.Metadata = azureMeta err = destBlob.Copy(srcBlobURL, nil) if err != nil { - return objInfo, azureToObjectError(errors.Trace(err), srcBucket, srcObject) + logger.LogIf(ctx, err) + return objInfo, azureToObjectError(err, srcBucket, srcObject) } destBlob.Properties = props err = destBlob.SetProperties(nil) if err != nil { - return objInfo, azureToObjectError(errors.Trace(err), srcBucket, srcObject) + logger.LogIf(ctx, err) + return objInfo, azureToObjectError(err, srcBucket, srcObject) } return a.GetObjectInfo(ctx, destBucket, destObject) } @@ -670,7 +684,7 @@ func (a *azureObjects) DeleteObject(ctx context.Context, bucket, object string) blob := a.client.GetContainerReference(bucket).GetBlobReference(object) err := blob.Delete(nil) if err != nil { - return azureToObjectError(errors.Trace(err), bucket, object) + return azureToObjectError(err, bucket, object) } return nil } @@ -690,19 +704,21 @@ func getAzureMetadataObjectName(objectName, uploadID string) string { return fmt.Sprintf(metadataObjectNameTemplate, uploadID, sha256.Sum256([]byte(objectName))) } -func (a *azureObjects) checkUploadIDExists(bucketName, objectName, uploadID string) (err error) { +func (a *azureObjects) checkUploadIDExists(ctx context.Context, bucketName, objectName, uploadID string) (err error) { blob := a.client.GetContainerReference(bucketName).GetBlobReference( getAzureMetadataObjectName(objectName, uploadID)) err = blob.GetMetadata(nil) - err = azureToObjectError(errors.Trace(err), bucketName, objectName) + logger.LogIf(ctx, err) + err = azureToObjectError(err, bucketName, objectName) oerr := minio.ObjectNotFound{ Bucket: bucketName, Object: objectName, } if errors.Cause(err) == oerr { - err = errors.Trace(minio.InvalidUploadID{ + logger.LogIf(ctx, minio.InvalidUploadID{UploadID: uploadID}) + err = minio.InvalidUploadID{ UploadID: uploadID, - }) + } } return err } @@ -714,13 +730,15 @@ func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object st var jsonData []byte if jsonData, err = json.Marshal(azureMultipartMetadata{Name: object, Metadata: metadata}); err != nil { - return "", errors.Trace(err) + logger.LogIf(ctx, err) + return "", err } blob := a.client.GetContainerReference(bucket).GetBlobReference(metadataObject) err = blob.CreateBlockBlobFromReader(bytes.NewBuffer(jsonData), nil) if err != nil { - return "", azureToObjectError(errors.Trace(err), bucket, metadataObject) + logger.LogIf(ctx, err) + return "", azureToObjectError(err, bucket, metadataObject) } return uploadID, nil @@ -728,11 +746,11 @@ func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object st // PutObjectPart - Use Azure equivalent PutBlockWithLength. func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info minio.PartInfo, err error) { - if err = a.checkUploadIDExists(bucket, object, uploadID); err != nil { + if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil { return info, err } - if err = checkAzureUploadID(uploadID); err != nil { + if err = checkAzureUploadID(ctx, uploadID); err != nil { return info, err } @@ -756,7 +774,8 @@ func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, upload blob := a.client.GetContainerReference(bucket).GetBlobReference(object) err = blob.PutBlockWithLength(id, uint64(subPartSize), io.LimitReader(data, subPartSize), nil) if err != nil { - return info, azureToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return info, azureToObjectError(err, bucket, object) } subPartNumber++ } @@ -770,7 +789,7 @@ func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, upload // ListObjectParts - Use Azure equivalent GetBlockList. func (a *azureObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int) (result minio.ListPartsInfo, err error) { - if err = a.checkUploadIDExists(bucket, object, uploadID); err != nil { + if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil { return result, err } @@ -787,7 +806,8 @@ func (a *azureObjects) ListObjectParts(ctx context.Context, bucket, object, uplo return result, nil } if err != nil { - return result, azureToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return result, azureToObjectError(err, bucket, object) } // Build a sorted list of parts and return the requested entries. partsMap := make(map[int]minio.PartInfo) @@ -796,7 +816,8 @@ func (a *azureObjects) ListObjectParts(ctx context.Context, bucket, object, uplo var parsedUploadID string var md5Hex string if partNumber, _, parsedUploadID, md5Hex, err = azureParseBlockID(block.Name); err != nil { - return result, azureToObjectError(errors.Trace(fmt.Errorf("Unexpected error")), bucket, object) + logger.LogIf(ctx, fmt.Errorf("Unexpected error")) + return result, azureToObjectError(fmt.Errorf("Unexpected error"), bucket, object) } if parsedUploadID != uploadID { continue @@ -813,7 +834,8 @@ func (a *azureObjects) ListObjectParts(ctx context.Context, bucket, object, uplo if part.ETag != md5Hex { // If two parts of same partNumber were uploaded with different contents // return error as we won't be able to decide which the latest part is. - return result, azureToObjectError(errors.Trace(fmt.Errorf("Unexpected error")), bucket, object) + logger.LogIf(ctx, fmt.Errorf("Unexpected error")) + return result, azureToObjectError(fmt.Errorf("Unexpected error"), bucket, object) } part.Size += block.Size partsMap[partNumber] = part @@ -856,7 +878,7 @@ func (a *azureObjects) ListObjectParts(ctx context.Context, bucket, object, uplo // There is no corresponding API in azure to abort an incomplete upload. The uncommmitted blocks // gets deleted after one week. func (a *azureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) (err error) { - if err = a.checkUploadIDExists(bucket, object, uploadID); err != nil { + if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil { return err } @@ -868,23 +890,25 @@ func (a *azureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, // CompleteMultipartUpload - Use Azure equivalent PutBlockList. func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []minio.CompletePart) (objInfo minio.ObjectInfo, err error) { metadataObject := getAzureMetadataObjectName(object, uploadID) - if err = a.checkUploadIDExists(bucket, object, uploadID); err != nil { + if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil { return objInfo, err } - if err = checkAzureUploadID(uploadID); err != nil { + if err = checkAzureUploadID(ctx, uploadID); err != nil { return objInfo, err } var metadataReader io.Reader blob := a.client.GetContainerReference(bucket).GetBlobReference(metadataObject) if metadataReader, err = blob.Get(nil); err != nil { - return objInfo, azureToObjectError(errors.Trace(err), bucket, metadataObject) + logger.LogIf(ctx, err) + return objInfo, azureToObjectError(err, bucket, metadataObject) } var metadata azureMultipartMetadata if err = json.NewDecoder(metadataReader).Decode(&metadata); err != nil { - return objInfo, azureToObjectError(errors.Trace(err), bucket, metadataObject) + logger.LogIf(ctx, err) + return objInfo, azureToObjectError(err, bucket, metadataObject) } defer func() { @@ -894,13 +918,15 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje blob := a.client.GetContainerReference(bucket).GetBlobReference(metadataObject) derr := blob.Delete(nil) - minio.ErrorIf(derr, "unable to remove meta data object for upload ID %s", uploadID) + logger.GetReqInfo(ctx).AppendTags("uploadID", uploadID) + logger.LogIf(ctx, derr) }() objBlob := a.client.GetContainerReference(bucket).GetBlobReference(object) resp, err := objBlob.GetBlockList(storage.BlockListTypeUncommitted, nil) if err != nil { - return objInfo, azureToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return objInfo, azureToObjectError(err, bucket, object) } getBlocks := func(partNumber int, etag string) (blocks []storage.Block, size int64, err error) { @@ -936,7 +962,8 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje var size int64 blocks, size, err = getBlocks(part.PartNumber, part.ETag) if err != nil { - return objInfo, errors.Trace(err) + logger.LogIf(ctx, err) + return objInfo, err } allBlocks = append(allBlocks, blocks...) @@ -946,30 +973,39 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje // Error out if parts except last part sizing < 5MiB. for i, size := range partSizes[:len(partSizes)-1] { if size < azureS3MinPartSize { - return objInfo, errors.Trace(minio.PartTooSmall{ + logger.LogIf(ctx, minio.PartTooSmall{ PartNumber: uploadedParts[i].PartNumber, PartSize: size, PartETag: uploadedParts[i].ETag, }) + return objInfo, minio.PartTooSmall{ + PartNumber: uploadedParts[i].PartNumber, + PartSize: size, + PartETag: uploadedParts[i].ETag, + } } } err = objBlob.PutBlockList(allBlocks, nil) if err != nil { - return objInfo, azureToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return objInfo, azureToObjectError(err, bucket, object) } if len(metadata.Metadata) > 0 { - objBlob.Metadata, objBlob.Properties, err = s3MetaToAzureProperties(metadata.Metadata) + objBlob.Metadata, objBlob.Properties, err = s3MetaToAzureProperties(ctx, metadata.Metadata) if err != nil { + logger.LogIf(ctx, err) return objInfo, azureToObjectError(err, bucket, object) } err = objBlob.SetProperties(nil) if err != nil { - return objInfo, azureToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return objInfo, azureToObjectError(err, bucket, object) } err = objBlob.SetMetadata(nil) if err != nil { - return objInfo, azureToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return objInfo, azureToObjectError(err, bucket, object) } } return a.GetObjectInfo(ctx, bucket, object) @@ -992,13 +1028,16 @@ func (a *azureObjects) SetBucketPolicy(ctx context.Context, bucket string, polic } prefix := bucket + "/*" // For all objects inside the bucket. if len(policies) != 1 { - return errors.Trace(minio.NotImplemented{}) + logger.LogIf(ctx, minio.NotImplemented{}) + return minio.NotImplemented{} } if policies[0].Prefix != prefix { - return errors.Trace(minio.NotImplemented{}) + logger.LogIf(ctx, minio.NotImplemented{}) + return minio.NotImplemented{} } if policies[0].Policy != policy.BucketPolicyReadOnly { - return errors.Trace(minio.NotImplemented{}) + logger.LogIf(ctx, minio.NotImplemented{}) + return minio.NotImplemented{} } perm := storage.ContainerPermissions{ AccessType: storage.ContainerAccessTypeContainer, @@ -1006,7 +1045,8 @@ func (a *azureObjects) SetBucketPolicy(ctx context.Context, bucket string, polic } container := a.client.GetContainerReference(bucket) err := container.SetPermissions(perm, nil) - return azureToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return azureToObjectError(err, bucket) } // GetBucketPolicy - Get the container ACL and convert it to canonical []bucketAccessPolicy @@ -1015,15 +1055,18 @@ func (a *azureObjects) GetBucketPolicy(ctx context.Context, bucket string) (poli container := a.client.GetContainerReference(bucket) perm, err := container.GetPermissions(nil) if err != nil { - return policy.BucketAccessPolicy{}, azureToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return policy.BucketAccessPolicy{}, azureToObjectError(err, bucket) } switch perm.AccessType { case storage.ContainerAccessTypePrivate: - return policy.BucketAccessPolicy{}, errors.Trace(minio.PolicyNotFound{Bucket: bucket}) + logger.LogIf(ctx, minio.PolicyNotFound{Bucket: bucket}) + return policy.BucketAccessPolicy{}, minio.PolicyNotFound{Bucket: bucket} case storage.ContainerAccessTypeContainer: policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadOnly, bucket, "") default: - return policy.BucketAccessPolicy{}, azureToObjectError(errors.Trace(minio.NotImplemented{})) + logger.LogIf(ctx, minio.NotImplemented{}) + return policy.BucketAccessPolicy{}, azureToObjectError(minio.NotImplemented{}) } return policyInfo, nil } @@ -1036,5 +1079,6 @@ func (a *azureObjects) DeleteBucketPolicy(ctx context.Context, bucket string) er } container := a.client.GetContainerReference(bucket) err := container.SetPermissions(perm, nil) - return azureToObjectError(errors.Trace(err)) + logger.LogIf(ctx, err) + return azureToObjectError(err) } diff --git a/cmd/gateway/azure/gateway-azure_test.go b/cmd/gateway/azure/gateway-azure_test.go index c7ebce3c3..957882e51 100644 --- a/cmd/gateway/azure/gateway-azure_test.go +++ b/cmd/gateway/azure/gateway-azure_test.go @@ -17,6 +17,7 @@ package azure import ( + "context" "fmt" "net/http" "reflect" @@ -55,7 +56,7 @@ func TestS3MetaToAzureProperties(t *testing.T) { "X_Amz_Matdesc": "{}", "X_Amz_Iv": "eWmyryl8kq+EVnnsE7jpOg==", } - meta, _, err := s3MetaToAzureProperties(headers) + meta, _, err := s3MetaToAzureProperties(context.Background(), headers) if err != nil { t.Fatalf("Test failed, with %s", err) } @@ -65,7 +66,7 @@ func TestS3MetaToAzureProperties(t *testing.T) { headers = map[string]string{ "invalid--meta": "value", } - _, _, err = s3MetaToAzureProperties(headers) + _, _, err = s3MetaToAzureProperties(context.Background(), headers) if err = errors.Cause(err); err != nil { if _, ok := err.(minio.UnsupportedMetadata); !ok { t.Fatalf("Test failed with unexpected error %s, expected UnsupportedMetadata", err) @@ -75,7 +76,7 @@ func TestS3MetaToAzureProperties(t *testing.T) { headers = map[string]string{ "content-md5": "Dce7bmCX61zvxzP5QmfelQ==", } - _, props, err := s3MetaToAzureProperties(headers) + _, props, err := s3MetaToAzureProperties(context.Background(), headers) if err != nil { t.Fatalf("Test failed, with %s", err) } @@ -137,53 +138,46 @@ func TestAzureToObjectError(t *testing.T) { nil, nil, "", "", }, { - errors.Trace(fmt.Errorf("Non azure error")), + fmt.Errorf("Non azure error"), fmt.Errorf("Non azure error"), "", "", }, { storage.AzureStorageServiceError{ Code: "ContainerAlreadyExists", - }, storage.AzureStorageServiceError{ - Code: "ContainerAlreadyExists", - }, "bucket", "", + }, minio.BucketExists{Bucket: "bucket"}, "bucket", "", }, { - errors.Trace(storage.AzureStorageServiceError{ - Code: "ContainerAlreadyExists", - }), minio.BucketExists{Bucket: "bucket"}, "bucket", "", - }, - { - errors.Trace(storage.AzureStorageServiceError{ + storage.AzureStorageServiceError{ Code: "InvalidResourceName", - }), minio.BucketNameInvalid{Bucket: "bucket."}, "bucket.", "", + }, minio.BucketNameInvalid{Bucket: "bucket."}, "bucket.", "", }, { - errors.Trace(storage.AzureStorageServiceError{ + storage.AzureStorageServiceError{ Code: "RequestBodyTooLarge", - }), minio.PartTooBig{}, "", "", + }, minio.PartTooBig{}, "", "", }, { - errors.Trace(storage.AzureStorageServiceError{ + storage.AzureStorageServiceError{ Code: "InvalidMetadata", - }), minio.UnsupportedMetadata{}, "", "", + }, minio.UnsupportedMetadata{}, "", "", }, { - errors.Trace(storage.AzureStorageServiceError{ + storage.AzureStorageServiceError{ StatusCode: http.StatusNotFound, - }), minio.ObjectNotFound{ + }, minio.ObjectNotFound{ Bucket: "bucket", Object: "object", }, "bucket", "object", }, { - errors.Trace(storage.AzureStorageServiceError{ + storage.AzureStorageServiceError{ StatusCode: http.StatusNotFound, - }), minio.BucketNotFound{Bucket: "bucket"}, "bucket", "", + }, minio.BucketNotFound{Bucket: "bucket"}, "bucket", "", }, { - errors.Trace(storage.AzureStorageServiceError{ + storage.AzureStorageServiceError{ StatusCode: http.StatusBadRequest, - }), minio.BucketNameInvalid{Bucket: "bucket."}, "bucket.", "", + }, minio.BucketNameInvalid{Bucket: "bucket."}, "bucket.", "", }, } for i, testCase := range testCases { @@ -307,7 +301,7 @@ func TestCheckAzureUploadID(t *testing.T) { } for _, uploadID := range invalidUploadIDs { - if err := checkAzureUploadID(uploadID); err == nil { + if err := checkAzureUploadID(context.Background(), uploadID); err == nil { t.Fatalf("%s: expected: , got: ", uploadID) } } @@ -318,7 +312,7 @@ func TestCheckAzureUploadID(t *testing.T) { } for _, uploadID := range validUploadIDs { - if err := checkAzureUploadID(uploadID); err != nil { + if err := checkAzureUploadID(context.Background(), uploadID); err != nil { t.Fatalf("%s: expected: , got: %s", uploadID, err) } } diff --git a/cmd/gateway/b2/gateway-b2.go b/cmd/gateway/b2/gateway-b2.go index 8fbcefb0c..4fd53ab79 100644 --- a/cmd/gateway/b2/gateway-b2.go +++ b/cmd/gateway/b2/gateway-b2.go @@ -31,8 +31,8 @@ import ( b2 "github.com/minio/blazer/base" "github.com/minio/cli" "github.com/minio/minio-go/pkg/policy" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" - "github.com/minio/minio/pkg/errors" h2 "github.com/minio/minio/pkg/hash" minio "github.com/minio/minio/cmd" @@ -146,16 +146,6 @@ func b2ToObjectError(err error, params ...string) error { if err == nil { return nil } - - e, ok := err.(*errors.Error) - if !ok { - // Code should be fixed if this function is called without doing errors.Trace() - // Else handling different situations in this function makes this function complicated. - minio.ErrorIf(err, "Expected type *Error") - return err - } - - err = e.Cause bucket := "" object := "" uploadID := "" @@ -177,7 +167,7 @@ func b2ToObjectError(err error, params ...string) error { if statusCode == 0 { // We don't interpret non B2 errors. B2 errors have statusCode // to help us convert them to S3 object errors. - return e + return err } switch code { @@ -208,8 +198,7 @@ func b2ToObjectError(err error, params ...string) error { err = minio.InvalidUploadID{UploadID: uploadID} } - e.Cause = err - return e + return err } // Shutdown saves any gateway metadata to disk @@ -230,7 +219,8 @@ func (l *b2Objects) MakeBucketWithLocation(ctx context.Context, bucket, location // All buckets are set to private by default. _, err := l.b2Client.CreateBucket(l.ctx, bucket, bucketTypePrivate, nil, nil) - return b2ToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return b2ToObjectError(err, bucket) } func (l *b2Objects) reAuthorizeAccount(ctx context.Context) error { @@ -271,14 +261,16 @@ func (l *b2Objects) listBuckets(ctx context.Context, err error) ([]*b2.Bucket, e func (l *b2Objects) Bucket(ctx context.Context, bucket string) (*b2.Bucket, error) { bktList, err := l.listBuckets(ctx, nil) if err != nil { - return nil, b2ToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return nil, b2ToObjectError(err, bucket) } for _, bkt := range bktList { if bkt.Name == bucket { return bkt, nil } } - return nil, errors.Trace(minio.BucketNotFound{Bucket: bucket}) + logger.LogIf(ctx, minio.BucketNotFound{Bucket: bucket}) + return nil, minio.BucketNotFound{Bucket: bucket} } // GetBucketInfo gets bucket metadata.. @@ -315,7 +307,8 @@ func (l *b2Objects) DeleteBucket(ctx context.Context, bucket string) error { return err } err = bkt.DeleteBucket(l.ctx) - return b2ToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return b2ToObjectError(err, bucket) } // ListObjects lists all objects in B2 bucket filtered by prefix, returns upto at max 1000 entries at a time. @@ -326,7 +319,8 @@ func (l *b2Objects) ListObjects(ctx context.Context, bucket string, prefix strin } files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter) if lerr != nil { - return loi, b2ToObjectError(errors.Trace(lerr), bucket) + logger.LogIf(ctx, lerr) + return loi, b2ToObjectError(lerr, bucket) } loi.IsTruncated = next != "" loi.NextMarker = next @@ -359,7 +353,8 @@ func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat } files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, continuationToken, prefix, delimiter) if lerr != nil { - return loi, b2ToObjectError(errors.Trace(lerr), bucket) + logger.LogIf(ctx, lerr) + return loi, b2ToObjectError(lerr, bucket) } loi.IsTruncated = next != "" loi.ContinuationToken = continuationToken @@ -396,11 +391,13 @@ func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string, } reader, err := bkt.DownloadFileByName(l.ctx, object, startOffset, length) if err != nil { - return b2ToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return b2ToObjectError(err, bucket, object) } defer reader.Close() _, err = io.Copy(writer, reader) - return b2ToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return b2ToObjectError(err, bucket, object) } // GetObjectInfo reads object info and replies back ObjectInfo @@ -411,12 +408,14 @@ func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object str } f, err := bkt.DownloadFileByName(l.ctx, object, 0, 1) if err != nil { - return objInfo, b2ToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return objInfo, b2ToObjectError(err, bucket, object) } f.Close() fi, err := bkt.File(f.ID, object).GetFileInfo(l.ctx) if err != nil { - return objInfo, b2ToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return objInfo, b2ToObjectError(err, bucket, object) } return minio.ObjectInfo{ Bucket: bucket, @@ -504,20 +503,23 @@ func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string, var u *b2.URL u, err = bkt.GetUploadURL(l.ctx) if err != nil { - return objInfo, b2ToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return objInfo, b2ToObjectError(err, bucket, object) } hr := newB2Reader(data, data.Size()) var f *b2.File f, err = u.UploadFile(l.ctx, hr, int(hr.Size()), object, contentType, sha1AtEOF, metadata) if err != nil { - return objInfo, b2ToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return objInfo, b2ToObjectError(err, bucket, object) } var fi *b2.FileInfo fi, err = f.GetFileInfo(l.ctx) if err != nil { - return objInfo, b2ToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return objInfo, b2ToObjectError(err, bucket, object) } return minio.ObjectInfo{ @@ -539,12 +541,14 @@ func (l *b2Objects) DeleteObject(ctx context.Context, bucket string, object stri } reader, err := bkt.DownloadFileByName(l.ctx, object, 0, 1) if err != nil { - return b2ToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return b2ToObjectError(err, bucket, object) } io.Copy(ioutil.Discard, reader) reader.Close() err = bkt.File(reader.ID, object).DeleteFileVersion(l.ctx) - return b2ToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return b2ToObjectError(err, bucket, object) } // ListMultipartUploads lists all multipart uploads. @@ -563,7 +567,8 @@ func (l *b2Objects) ListMultipartUploads(ctx context.Context, bucket string, pre } largeFiles, nextMarker, err := bkt.ListUnfinishedLargeFiles(l.ctx, uploadIDMarker, maxUploads) if err != nil { - return lmi, b2ToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return lmi, b2ToObjectError(err, bucket) } lmi = minio.ListMultipartsInfo{ MaxUploads: maxUploads, @@ -598,7 +603,8 @@ func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, objec delete(metadata, "content-type") lf, err := bkt.StartLargeFile(l.ctx, object, contentType, metadata) if err != nil { - return uploadID, b2ToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return uploadID, b2ToObjectError(err, bucket, object) } return lf.ID, nil @@ -613,13 +619,15 @@ func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object str fc, err := bkt.File(uploadID, object).CompileParts(0, nil).GetUploadPartURL(l.ctx) if err != nil { - return pi, b2ToObjectError(errors.Trace(err), bucket, object, uploadID) + logger.LogIf(ctx, err) + return pi, b2ToObjectError(err, bucket, object, uploadID) } hr := newB2Reader(data, data.Size()) sha1, err := fc.UploadPart(l.ctx, hr, sha1AtEOF, int(hr.Size()), partID) if err != nil { - return pi, b2ToObjectError(errors.Trace(err), bucket, object, uploadID) + logger.LogIf(ctx, err) + return pi, b2ToObjectError(err, bucket, object, uploadID) } return minio.PartInfo{ @@ -647,7 +655,8 @@ func (l *b2Objects) ListObjectParts(ctx context.Context, bucket string, object s partNumberMarker++ partsList, next, err := bkt.File(uploadID, object).ListParts(l.ctx, partNumberMarker, maxParts) if err != nil { - return lpi, b2ToObjectError(errors.Trace(err), bucket, object, uploadID) + logger.LogIf(ctx, err) + return lpi, b2ToObjectError(err, bucket, object, uploadID) } if next != 0 { lpi.IsTruncated = true @@ -670,7 +679,8 @@ func (l *b2Objects) AbortMultipartUpload(ctx context.Context, bucket string, obj return err } err = bkt.File(uploadID, object).CompileParts(0, nil).CancelLargeFile(l.ctx) - return b2ToObjectError(errors.Trace(err), bucket, object, uploadID) + logger.LogIf(ctx, err) + return b2ToObjectError(err, bucket, object, uploadID) } // CompleteMultipartUpload completes ongoing multipart upload and finalizes object, uses B2's LargeFile upload API. @@ -684,7 +694,8 @@ func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string, // B2 requires contigous part numbers starting with 1, they do not support // hand picking part numbers, we return an S3 compatible error instead. if i+1 != uploadedPart.PartNumber { - return oi, b2ToObjectError(errors.Trace(minio.InvalidPart{}), bucket, object, uploadID) + logger.LogIf(ctx, minio.InvalidPart{}) + return oi, b2ToObjectError(minio.InvalidPart{}, bucket, object, uploadID) } // Trim "-1" suffix in ETag as PutObjectPart() treats B2 returned SHA1 as ETag. @@ -692,7 +703,8 @@ func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string, } if _, err = bkt.File(uploadID, object).CompileParts(0, hashes).FinishLargeFile(l.ctx); err != nil { - return oi, b2ToObjectError(errors.Trace(err), bucket, object, uploadID) + logger.LogIf(ctx, err) + return oi, b2ToObjectError(err, bucket, object, uploadID) } return l.GetObjectInfo(ctx, bucket, object) @@ -713,13 +725,16 @@ func (l *b2Objects) SetBucketPolicy(ctx context.Context, bucket string, policyIn } prefix := bucket + "/*" // For all objects inside the bucket. if len(policies) != 1 { - return errors.Trace(minio.NotImplemented{}) + logger.LogIf(ctx, minio.NotImplemented{}) + return minio.NotImplemented{} } if policies[0].Prefix != prefix { - return errors.Trace(minio.NotImplemented{}) + logger.LogIf(ctx, minio.NotImplemented{}) + return minio.NotImplemented{} } if policies[0].Policy != policy.BucketPolicyReadOnly { - return errors.Trace(minio.NotImplemented{}) + logger.LogIf(ctx, minio.NotImplemented{}) + return minio.NotImplemented{} } bkt, err := l.Bucket(ctx, bucket) if err != nil { @@ -727,7 +742,8 @@ func (l *b2Objects) SetBucketPolicy(ctx context.Context, bucket string, policyIn } bkt.Type = bucketTypeReadOnly _, err = bkt.Update(l.ctx) - return b2ToObjectError(errors.Trace(err)) + logger.LogIf(ctx, err) + return b2ToObjectError(err) } // GetBucketPolicy, returns the current bucketType from B2 backend and convert @@ -745,7 +761,8 @@ func (l *b2Objects) GetBucketPolicy(ctx context.Context, bucket string) (policy. // bkt.Type can also be snapshot, but it is only allowed through B2 browser console, // just return back as policy not found for all cases. // CreateBucket always sets the value to allPrivate by default. - return policy.BucketAccessPolicy{}, errors.Trace(minio.PolicyNotFound{Bucket: bucket}) + logger.LogIf(ctx, minio.PolicyNotFound{Bucket: bucket}) + return policy.BucketAccessPolicy{}, minio.PolicyNotFound{Bucket: bucket} } // DeleteBucketPolicy - resets the bucketType of bucket on B2 to 'allPrivate'. @@ -756,5 +773,6 @@ func (l *b2Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error } bkt.Type = bucketTypePrivate _, err = bkt.Update(l.ctx) - return b2ToObjectError(errors.Trace(err)) + logger.LogIf(ctx, err) + return b2ToObjectError(err) } diff --git a/cmd/gateway/b2/gateway-b2_test.go b/cmd/gateway/b2/gateway-b2_test.go index 8c2c1e613..349127616 100644 --- a/cmd/gateway/b2/gateway-b2_test.go +++ b/cmd/gateway/b2/gateway-b2_test.go @@ -21,7 +21,6 @@ import ( "testing" b2 "github.com/minio/blazer/base" - "github.com/minio/minio/pkg/errors" minio "github.com/minio/minio/cmd" ) @@ -40,70 +39,70 @@ func TestB2ObjectError(t *testing.T) { []string{}, fmt.Errorf("Not *Error"), fmt.Errorf("Not *Error"), }, { - []string{}, errors.Trace(fmt.Errorf("Non B2 Error")), fmt.Errorf("Non B2 Error"), + []string{}, fmt.Errorf("Non B2 Error"), fmt.Errorf("Non B2 Error"), }, { - []string{"bucket"}, errors.Trace(b2.Error{ + []string{"bucket"}, b2.Error{ StatusCode: 1, Code: "duplicate_bucket_name", - }), minio.BucketAlreadyOwnedByYou{ + }, minio.BucketAlreadyOwnedByYou{ Bucket: "bucket", }, }, { - []string{"bucket"}, errors.Trace(b2.Error{ + []string{"bucket"}, b2.Error{ StatusCode: 1, Code: "bad_request", - }), minio.BucketNotFound{ + }, minio.BucketNotFound{ Bucket: "bucket", }, }, { - []string{"bucket", "object"}, errors.Trace(b2.Error{ + []string{"bucket", "object"}, b2.Error{ StatusCode: 1, Code: "bad_request", - }), minio.ObjectNameInvalid{ + }, minio.ObjectNameInvalid{ Bucket: "bucket", Object: "object", }, }, { - []string{"bucket"}, errors.Trace(b2.Error{ + []string{"bucket"}, b2.Error{ StatusCode: 1, Code: "bad_bucket_id", - }), minio.BucketNotFound{Bucket: "bucket"}, + }, minio.BucketNotFound{Bucket: "bucket"}, }, { - []string{"bucket", "object"}, errors.Trace(b2.Error{ + []string{"bucket", "object"}, b2.Error{ StatusCode: 1, Code: "file_not_present", - }), minio.ObjectNotFound{ + }, minio.ObjectNotFound{ Bucket: "bucket", Object: "object", }, }, { - []string{"bucket", "object"}, errors.Trace(b2.Error{ + []string{"bucket", "object"}, b2.Error{ StatusCode: 1, Code: "not_found", - }), minio.ObjectNotFound{ + }, minio.ObjectNotFound{ Bucket: "bucket", Object: "object", }, }, { - []string{"bucket"}, errors.Trace(b2.Error{ + []string{"bucket"}, b2.Error{ StatusCode: 1, Code: "cannot_delete_non_empty_bucket", - }), minio.BucketNotEmpty{ + }, minio.BucketNotEmpty{ Bucket: "bucket", }, }, { - []string{"bucket", "object", "uploadID"}, errors.Trace(b2.Error{ + []string{"bucket", "object", "uploadID"}, b2.Error{ StatusCode: 1, Message: "No active upload for", - }), minio.InvalidUploadID{ + }, minio.InvalidUploadID{ UploadID: "uploadID", }, }, diff --git a/cmd/gateway/gcs/gateway-gcs.go b/cmd/gateway/gcs/gateway-gcs.go index a5b803b9d..98025c115 100644 --- a/cmd/gateway/gcs/gateway-gcs.go +++ b/cmd/gateway/gcs/gateway-gcs.go @@ -34,8 +34,8 @@ import ( humanize "github.com/dustin/go-humanize" "github.com/minio/cli" "github.com/minio/minio-go/pkg/policy" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" - "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/hash" "google.golang.org/api/googleapi" @@ -154,11 +154,13 @@ EXAMPLES: func gcsGatewayMain(ctx *cli.Context) { projectID := ctx.Args().First() if projectID == "" && os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" { - minio.ErrorIf(errGCSProjectIDNotFound, "project-id should be provided as argument or GOOGLE_APPLICATION_CREDENTIALS should be set with path to credentials.json") + logger.LogIf(context.Background(), errGCSProjectIDNotFound) cli.ShowCommandHelpAndExit(ctx, "gcs", 1) } if projectID != "" && !isValidGCSProjectIDFormat(projectID) { - minio.ErrorIf(errGCSInvalidProjectID, "Unable to start GCS gateway with %s", ctx.Args().First()) + reqInfo := (&logger.ReqInfo{}).AppendTags("projectID", ctx.Args().First()) + contxt := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(contxt, errGCSInvalidProjectID) cli.ShowCommandHelpAndExit(ctx, "gcs", 1) } @@ -237,16 +239,6 @@ func gcsToObjectError(err error, params ...string) error { return nil } - e, ok := err.(*errors.Error) - if !ok { - // Code should be fixed if this function is called without doing errors.Trace() - // Else handling different situations in this function makes this function complicated. - minio.ErrorIf(err, "Expected type *Error") - return err - } - - err = e.Cause - bucket := "" object := "" uploadID := "" @@ -266,8 +258,7 @@ func gcsToObjectError(err error, params ...string) error { err = minio.BucketNotFound{ Bucket: bucket, } - e.Cause = err - return e + return err case "storage: object doesn't exist": if uploadID != "" { err = minio.InvalidUploadID{ @@ -279,21 +270,18 @@ func gcsToObjectError(err error, params ...string) error { Object: object, } } - e.Cause = err - return e + return err } googleAPIErr, ok := err.(*googleapi.Error) if !ok { // We don't interpret non Minio errors. As minio errors will // have StatusCode to help to convert to object errors. - e.Cause = err - return e + return err } if len(googleAPIErr.Errors) == 0 { - e.Cause = err - return e + return err } reason := googleAPIErr.Errors[0].Reason @@ -337,8 +325,7 @@ func gcsToObjectError(err error, params ...string) error { err = fmt.Errorf("Unsupported error reason: %s", reason) } - e.Cause = err - return e + return err } // gcsProjectIDRegex defines a valid gcs project id format @@ -381,7 +368,9 @@ func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(bucket string) { attrs, err := it.Next() if err != nil { if err != iterator.Done { - minio.ErrorIf(err, "Object listing error on bucket %s during purging of old files in minio.sys.tmp", bucket) + reqInfo := &logger.ReqInfo{BucketName: bucket} + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) } return } @@ -389,7 +378,9 @@ func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(bucket string) { // Delete files older than 2 weeks. err := l.client.Bucket(bucket).Object(attrs.Name).Delete(l.ctx) if err != nil { - minio.ErrorIf(err, "Unable to delete %s/%s during purging of old files in minio.sys.tmp", bucket, attrs.Name) + reqInfo := &logger.ReqInfo{BucketName: bucket, ObjectName: attrs.Name} + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) return } } @@ -404,7 +395,8 @@ func (l *gcsGateway) CleanupGCSMinioSysTmp() { attrs, err := it.Next() if err != nil { if err != iterator.Done { - minio.ErrorIf(err, "Bucket listing error during purging of old files in minio.sys.tmp") + ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{}) + logger.LogIf(ctx, err) } break } @@ -438,15 +430,16 @@ func (l *gcsGateway) MakeBucketWithLocation(ctx context.Context, bucket, locatio err := bkt.Create(l.ctx, l.projectID, &storage.BucketAttrs{ Location: location, }) - - return gcsToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return gcsToObjectError(err, bucket) } // GetBucketInfo - Get bucket metadata.. func (l *gcsGateway) GetBucketInfo(ctx context.Context, bucket string) (minio.BucketInfo, error) { attrs, err := l.client.Bucket(bucket).Attrs(l.ctx) if err != nil { - return minio.BucketInfo{}, gcsToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return minio.BucketInfo{}, gcsToObjectError(err, bucket) } return minio.BucketInfo{ @@ -467,7 +460,8 @@ func (l *gcsGateway) ListBuckets(ctx context.Context) (buckets []minio.BucketInf } if ierr != nil { - return buckets, gcsToObjectError(errors.Trace(ierr)) + logger.LogIf(ctx, ierr) + return buckets, gcsToObjectError(ierr) } buckets = append(buckets, minio.BucketInfo{ @@ -495,7 +489,8 @@ func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error { break } if err != nil { - return gcsToObjectError(errors.Trace(err)) + logger.LogIf(ctx, err) + return gcsToObjectError(err) } if objAttrs.Prefix == minio.GatewayMinioSysTmp { gcsMinioPathFound = true @@ -505,7 +500,8 @@ func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error { break } if nonGCSMinioPathFound { - return gcsToObjectError(errors.Trace(minio.BucketNotEmpty{})) + logger.LogIf(ctx, minio.BucketNotEmpty{}) + return gcsToObjectError(minio.BucketNotEmpty{}) } if gcsMinioPathFound { // Remove minio.sys.tmp before deleting the bucket. @@ -516,16 +512,19 @@ func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error { break } if err != nil { - return gcsToObjectError(errors.Trace(err)) + logger.LogIf(ctx, err) + return gcsToObjectError(err) } err = l.client.Bucket(bucket).Object(objAttrs.Name).Delete(l.ctx) if err != nil { - return gcsToObjectError(errors.Trace(err)) + logger.LogIf(ctx, err) + return gcsToObjectError(err) } } } err := l.client.Bucket(bucket).Delete(l.ctx) - return gcsToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return gcsToObjectError(err, bucket) } func toGCSPageToken(name string) string { @@ -607,7 +606,8 @@ func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix stri break } if err != nil { - return minio.ListObjectsInfo{}, gcsToObjectError(errors.Trace(err), bucket, prefix) + logger.LogIf(ctx, err) + return minio.ListObjectsInfo{}, gcsToObjectError(err, bucket, prefix) } nextMarker = toGCSPageToken(attrs.Name) @@ -689,7 +689,8 @@ func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continua } if err != nil { - return minio.ListObjectsV2Info{}, gcsToObjectError(errors.Trace(err), bucket, prefix) + logger.LogIf(ctx, err) + return minio.ListObjectsV2Info{}, gcsToObjectError(err, bucket, prefix) } if attrs.Prefix == minio.GatewayMinioSysTmp { @@ -733,18 +734,21 @@ func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, s // if we want to mimic S3 behavior exactly, we need to verify if bucket exists first, // otherwise gcs will just return object not exist in case of non-existing bucket if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil { - return gcsToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return gcsToObjectError(err, bucket) } object := l.client.Bucket(bucket).Object(key) r, err := object.NewRangeReader(l.ctx, startOffset, length) if err != nil { - return gcsToObjectError(errors.Trace(err), bucket, key) + logger.LogIf(ctx, err) + return gcsToObjectError(err, bucket, key) } defer r.Close() if _, err := io.Copy(writer, r); err != nil { - return gcsToObjectError(errors.Trace(err), bucket, key) + logger.LogIf(ctx, err) + return gcsToObjectError(err, bucket, key) } return nil @@ -771,12 +775,14 @@ func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object st // if we want to mimic S3 behavior exactly, we need to verify if bucket exists first, // otherwise gcs will just return object not exist in case of non-existing bucket if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil { - return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return minio.ObjectInfo{}, gcsToObjectError(err, bucket) } attrs, err := l.client.Bucket(bucket).Object(object).Attrs(l.ctx) if err != nil { - return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return minio.ObjectInfo{}, gcsToObjectError(err, bucket, object) } return fromGCSAttrsToObjectInfo(attrs), nil @@ -787,7 +793,8 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, d // if we want to mimic S3 behavior exactly, we need to verify if bucket exists first, // otherwise gcs will just return object not exist in case of non-existing bucket if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil { - return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return minio.ObjectInfo{}, gcsToObjectError(err, bucket) } object := l.client.Bucket(bucket).Object(key) @@ -801,7 +808,8 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, d if _, err := io.Copy(w, data); err != nil { // Close the object writer upon error. w.CloseWithError(err) - return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket, key) + logger.LogIf(ctx, err) + return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key) } // Close the object writer upon success. @@ -809,7 +817,8 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, d attrs, err := object.Attrs(l.ctx) if err != nil { - return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket, key) + logger.LogIf(ctx, err) + return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key) } return fromGCSAttrsToObjectInfo(attrs), nil @@ -827,7 +836,8 @@ func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject attrs, err := copier.Run(l.ctx) if err != nil { - return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), destBucket, destObject) + logger.LogIf(ctx, err) + return minio.ObjectInfo{}, gcsToObjectError(err, destBucket, destObject) } return fromGCSAttrsToObjectInfo(attrs), nil @@ -837,7 +847,8 @@ func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject func (l *gcsGateway) DeleteObject(ctx context.Context, bucket string, object string) error { err := l.client.Bucket(bucket).Object(object).Delete(l.ctx) if err != nil { - return gcsToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return gcsToObjectError(err, bucket, object) } return nil @@ -863,7 +874,8 @@ func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key bucket, key, }); err != nil { - return "", gcsToObjectError(errors.Trace(err), bucket, key) + logger.LogIf(ctx, err) + return "", gcsToObjectError(err, bucket, key) } return uploadID, nil } @@ -883,7 +895,8 @@ func (l *gcsGateway) ListMultipartUploads(ctx context.Context, bucket string, pr // an object layer compatible error upon any error. func (l *gcsGateway) checkUploadIDExists(ctx context.Context, bucket string, key string, uploadID string) error { _, err := l.client.Bucket(bucket).Object(gcsMultipartMetaName(uploadID)).Attrs(l.ctx) - return gcsToObjectError(errors.Trace(err), bucket, key, uploadID) + logger.LogIf(ctx, err) + return gcsToObjectError(err, bucket, key, uploadID) } // PutObjectPart puts a part of object in bucket @@ -904,7 +917,8 @@ func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key strin if _, err := io.Copy(w, data); err != nil { // Make sure to close object writer upon error. w.Close() - return minio.PartInfo{}, gcsToObjectError(errors.Trace(err), bucket, key) + logger.LogIf(ctx, err) + return minio.PartInfo{}, gcsToObjectError(err, bucket, key) } // Make sure to close the object writer upon success. w.Close() @@ -923,7 +937,7 @@ func (l *gcsGateway) ListObjectParts(ctx context.Context, bucket string, key str } // Called by AbortMultipartUpload and CompleteMultipartUpload for cleaning up. -func (l *gcsGateway) cleanupMultipartUpload(bucket, key, uploadID string) error { +func (l *gcsGateway) cleanupMultipartUpload(ctx context.Context, bucket, key, uploadID string) error { prefix := fmt.Sprintf("%s/%s/", gcsMinioMultipartPathV1, uploadID) // iterate through all parts and delete them @@ -935,7 +949,8 @@ func (l *gcsGateway) cleanupMultipartUpload(bucket, key, uploadID string) error break } if err != nil { - return gcsToObjectError(errors.Trace(err), bucket, key) + logger.LogIf(ctx, err) + return gcsToObjectError(err, bucket, key) } object := l.client.Bucket(bucket).Object(attrs.Name) @@ -951,7 +966,7 @@ func (l *gcsGateway) AbortMultipartUpload(ctx context.Context, bucket string, ke if err := l.checkUploadIDExists(ctx, bucket, key, uploadID); err != nil { return err } - return l.cleanupMultipartUpload(bucket, key, uploadID) + return l.cleanupMultipartUpload(ctx, bucket, key, uploadID) } // CompleteMultipartUpload completes ongoing multipart upload and finalizes object @@ -968,23 +983,27 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string, partZeroAttrs, err := object.Attrs(l.ctx) if err != nil { - return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket, key, uploadID) + logger.LogIf(ctx, err) + return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key, uploadID) } r, err := object.NewReader(l.ctx) if err != nil { - return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket, key) + logger.LogIf(ctx, err) + return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key) } defer r.Close() // Check version compatibility of the meta file before compose() multipartMeta := gcsMultipartMetaV1{} if err = json.NewDecoder(r).Decode(&multipartMeta); err != nil { - return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket, key) + logger.LogIf(ctx, err) + return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key) } if multipartMeta.Version != gcsMinioMultipartMetaCurrentVersion { - return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(errGCSFormat), bucket, key) + logger.LogIf(ctx, errGCSFormat) + return minio.ObjectInfo{}, gcsToObjectError(errGCSFormat, bucket, key) } // Validate if the gcs.json stores valid entries for the bucket and key. @@ -1001,7 +1020,8 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string, uploadedPart.PartNumber, uploadedPart.ETag))) partAttr, pErr := l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID, uploadedPart.PartNumber, uploadedPart.ETag)).Attrs(l.ctx) if pErr != nil { - return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(pErr), bucket, key, uploadID) + logger.LogIf(ctx, pErr) + return minio.ObjectInfo{}, gcsToObjectError(pErr, bucket, key, uploadID) } partSizes[i] = partAttr.Size } @@ -1009,11 +1029,16 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string, // Error out if parts except last part sizing < 5MiB. for i, size := range partSizes[:len(partSizes)-1] { if size < 5*humanize.MiByte { - return minio.ObjectInfo{}, errors.Trace(minio.PartTooSmall{ + logger.LogIf(ctx, minio.PartTooSmall{ PartNumber: uploadedParts[i].PartNumber, PartSize: size, PartETag: uploadedParts[i].ETag, }) + return minio.ObjectInfo{}, minio.PartTooSmall{ + PartNumber: uploadedParts[i].PartNumber, + PartSize: size, + PartETag: uploadedParts[i].ETag, + } } } @@ -1040,7 +1065,8 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string, composer.Metadata = partZeroAttrs.Metadata if _, err = composer.Run(l.ctx); err != nil { - return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket, key) + logger.LogIf(ctx, err) + return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key) } } @@ -1053,10 +1079,11 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string, composer.Metadata = partZeroAttrs.Metadata attrs, err := composer.Run(l.ctx) if err != nil { - return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket, key) + logger.LogIf(ctx, err) + return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key) } - if err = l.cleanupMultipartUpload(bucket, key, uploadID); err != nil { - return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket, key) + if err = l.cleanupMultipartUpload(ctx, bucket, key, uploadID); err != nil { + return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key) } return fromGCSAttrsToObjectInfo(attrs), nil } @@ -1075,16 +1102,19 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, policyI prefix := bucket + "/*" // For all objects inside the bucket. if len(policies) != 1 { - return errors.Trace(minio.NotImplemented{}) + logger.LogIf(ctx, minio.NotImplemented{}) + return minio.NotImplemented{} } if policies[0].Prefix != prefix { - return errors.Trace(minio.NotImplemented{}) + logger.LogIf(ctx, minio.NotImplemented{}) + return minio.NotImplemented{} } acl := l.client.Bucket(bucket).ACL() if policies[0].Policy == policy.BucketPolicyNone { if err := acl.Delete(l.ctx, storage.AllUsers); err != nil { - return gcsToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return gcsToObjectError(err, bucket) } return nil } @@ -1096,11 +1126,13 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, policyI case policy.BucketPolicyWriteOnly: role = storage.RoleWriter default: - return errors.Trace(minio.NotImplemented{}) + logger.LogIf(ctx, minio.NotImplemented{}) + return minio.NotImplemented{} } if err := acl.Set(l.ctx, storage.AllUsers, role); err != nil { - return gcsToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return gcsToObjectError(err, bucket) } return nil @@ -1110,7 +1142,8 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, policyI func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) { rules, err := l.client.Bucket(bucket).ACL().List(l.ctx) if err != nil { - return policy.BucketAccessPolicy{}, gcsToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return policy.BucketAccessPolicy{}, gcsToObjectError(err, bucket) } policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"} for _, r := range rules { @@ -1126,7 +1159,8 @@ func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (policy } // Return NoSuchBucketPolicy error, when policy is not set if len(policyInfo.Statements) == 0 { - return policy.BucketAccessPolicy{}, gcsToObjectError(errors.Trace(minio.PolicyNotFound{}), bucket) + logger.LogIf(ctx, minio.PolicyNotFound{}) + return policy.BucketAccessPolicy{}, gcsToObjectError(minio.PolicyNotFound{}, bucket) } return policyInfo, nil } @@ -1135,7 +1169,8 @@ func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (policy func (l *gcsGateway) DeleteBucketPolicy(ctx context.Context, bucket string) error { // This only removes the storage.AllUsers policies if err := l.client.Bucket(bucket).ACL().Delete(l.ctx, storage.AllUsers); err != nil { - return gcsToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return gcsToObjectError(err, bucket) } return nil diff --git a/cmd/gateway/gcs/gateway-gcs_test.go b/cmd/gateway/gcs/gateway-gcs_test.go index deef5c302..24064ee43 100644 --- a/cmd/gateway/gcs/gateway-gcs_test.go +++ b/cmd/gateway/gcs/gateway-gcs_test.go @@ -24,7 +24,6 @@ import ( "reflect" "testing" - "github.com/minio/minio/pkg/errors" "google.golang.org/api/googleapi" miniogo "github.com/minio/minio-go" @@ -237,14 +236,14 @@ func TestGCSToObjectError(t *testing.T) { }, { []string{"bucket"}, - errors.Trace(fmt.Errorf("storage: bucket doesn't exist")), + fmt.Errorf("storage: bucket doesn't exist"), minio.BucketNotFound{ Bucket: "bucket", }, }, { []string{"bucket", "object"}, - errors.Trace(fmt.Errorf("storage: object doesn't exist")), + fmt.Errorf("storage: object doesn't exist"), minio.ObjectNotFound{ Bucket: "bucket", Object: "object", @@ -252,76 +251,76 @@ func TestGCSToObjectError(t *testing.T) { }, { []string{"bucket", "object", "uploadID"}, - errors.Trace(fmt.Errorf("storage: object doesn't exist")), + fmt.Errorf("storage: object doesn't exist"), minio.InvalidUploadID{ UploadID: "uploadID", }, }, { []string{}, - errors.Trace(fmt.Errorf("Unknown error")), + fmt.Errorf("Unknown error"), fmt.Errorf("Unknown error"), }, { []string{"bucket", "object"}, - errors.Trace(&googleapi.Error{ + &googleapi.Error{ Message: "No list of errors", - }), + }, &googleapi.Error{ Message: "No list of errors", }, }, { []string{"bucket", "object"}, - errors.Trace(&googleapi.Error{ + &googleapi.Error{ Errors: []googleapi.ErrorItem{{ Reason: "conflict", Message: "You already own this bucket. Please select another name.", }}, - }), + }, minio.BucketAlreadyOwnedByYou{ Bucket: "bucket", }, }, { []string{"bucket", "object"}, - errors.Trace(&googleapi.Error{ + &googleapi.Error{ Errors: []googleapi.ErrorItem{{ Reason: "conflict", Message: "Sorry, that name is not available. Please try a different one.", }}, - }), + }, minio.BucketAlreadyExists{ Bucket: "bucket", }, }, { []string{"bucket", "object"}, - errors.Trace(&googleapi.Error{ + &googleapi.Error{ Errors: []googleapi.ErrorItem{{ Reason: "conflict", }}, - }), + }, minio.BucketNotEmpty{Bucket: "bucket"}, }, { []string{"bucket"}, - errors.Trace(&googleapi.Error{ + &googleapi.Error{ Errors: []googleapi.ErrorItem{{ Reason: "notFound", }}, - }), + }, minio.BucketNotFound{ Bucket: "bucket", }, }, { []string{"bucket", "object"}, - errors.Trace(&googleapi.Error{ + &googleapi.Error{ Errors: []googleapi.ErrorItem{{ Reason: "notFound", }}, - }), + }, minio.ObjectNotFound{ Bucket: "bucket", Object: "object", @@ -329,22 +328,22 @@ func TestGCSToObjectError(t *testing.T) { }, { []string{"bucket"}, - errors.Trace(&googleapi.Error{ + &googleapi.Error{ Errors: []googleapi.ErrorItem{{ Reason: "invalid", }}, - }), + }, minio.BucketNameInvalid{ Bucket: "bucket", }, }, { []string{"bucket", "object"}, - errors.Trace(&googleapi.Error{ + &googleapi.Error{ Errors: []googleapi.ErrorItem{{ Reason: "forbidden", }}, - }), + }, minio.PrefixAccessDenied{ Bucket: "bucket", Object: "object", @@ -352,11 +351,11 @@ func TestGCSToObjectError(t *testing.T) { }, { []string{"bucket", "object"}, - errors.Trace(&googleapi.Error{ + &googleapi.Error{ Errors: []googleapi.ErrorItem{{ Reason: "keyInvalid", }}, - }), + }, minio.PrefixAccessDenied{ Bucket: "bucket", Object: "object", @@ -364,11 +363,11 @@ func TestGCSToObjectError(t *testing.T) { }, { []string{"bucket", "object"}, - errors.Trace(&googleapi.Error{ + &googleapi.Error{ Errors: []googleapi.ErrorItem{{ Reason: "required", }}, - }), + }, minio.PrefixAccessDenied{ Bucket: "bucket", Object: "object", @@ -376,11 +375,11 @@ func TestGCSToObjectError(t *testing.T) { }, { []string{"bucket", "object"}, - errors.Trace(&googleapi.Error{ + &googleapi.Error{ Errors: []googleapi.ErrorItem{{ Reason: "unknown", }}, - }), + }, fmt.Errorf("Unsupported error reason: unknown"), }, } diff --git a/cmd/gateway/manta/gateway-manta.go b/cmd/gateway/manta/gateway-manta.go index 131b9da8b..7da0e0ae3 100644 --- a/cmd/gateway/manta/gateway-manta.go +++ b/cmd/gateway/manta/gateway-manta.go @@ -33,8 +33,8 @@ import ( "github.com/joyent/triton-go/storage" "github.com/minio/cli" minio "github.com/minio/minio/cmd" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" - "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/hash" ) @@ -118,7 +118,7 @@ func mantaGatewayMain(ctx *cli.Context) { // Validate gateway arguments. host := ctx.Args().First() // Validate gateway arguments. - minio.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument") + logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument") minio.StartGateway(ctx, &Manta{host}) } @@ -139,6 +139,7 @@ func (g *Manta) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro var err error var signer authentication.Signer var endpoint = defaultMantaURL + ctx := context.Background() if g.host != "" { endpoint, _, err = minio.ParseGatewayEndpoint(g.host) @@ -163,7 +164,8 @@ func (g *Manta) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro } signer, err = authentication.NewSSHAgentSigner(input) if err != nil { - return nil, errors.Trace(err) + logger.LogIf(ctx, err) + return nil, err } } else { var keyBytes []byte @@ -200,7 +202,8 @@ func (g *Manta) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro signer, err = authentication.NewPrivateKeySigner(input) if err != nil { - return nil, errors.Trace(err) + logger.LogIf(ctx, err) + return nil, err } } @@ -352,7 +355,8 @@ func (t *tritonObjects) ListObjects(ctx context.Context, bucket, prefix, marker, if terrors.IsResourceNotFoundError(err) { return result, nil } - return result, errors.Trace(err) + logger.LogIf(ctx, err) + return result, err } for _, obj := range objs.Entries { @@ -362,7 +366,8 @@ func (t *tritonObjects) ListObjects(ctx context.Context, bucket, prefix, marker, input.DirectoryName = path.Join(mantaRoot, bucket, prefix) objs, err = t.client.Dir().List(ctx, input) if err != nil { - return result, errors.Trace(err) + logger.LogIf(ctx, err) + return result, err } break } @@ -428,7 +433,8 @@ func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, conti if terrors.IsResourceNotFoundError(err) { return result, nil } - return result, errors.Trace(err) + logger.LogIf(ctx, err) + return result, err } for _, obj := range objs.Entries { @@ -436,7 +442,8 @@ func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, conti input.DirectoryName = path.Join(mantaRoot, bucket, prefix) objs, err = t.client.Dir().List(ctx, input) if err != nil { - return result, errors.Trace(err) + logger.LogIf(ctx, err) + return result, err } break } @@ -479,7 +486,8 @@ func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, conti func (t *tritonObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error { // Start offset cannot be negative. if startOffset < 0 { - return errors.Trace(fmt.Errorf("Unexpected error")) + logger.LogIf(ctx, fmt.Errorf("Unexpected error")) + return fmt.Errorf("Unexpected error") } output, err := t.client.Objects().Get(ctx, &storage.GetObjectInput{ @@ -555,11 +563,13 @@ func (t *tritonObjects) PutObject(ctx context.Context, bucket, object string, da ObjectReader: dummySeeker{data}, ForceInsert: true, }); err != nil { - return objInfo, errors.Trace(err) + logger.LogIf(ctx, err) + return objInfo, err } if err = data.Verify(); err != nil { t.DeleteObject(ctx, bucket, object) - return objInfo, errors.Trace(err) + logger.LogIf(ctx, err) + return objInfo, err } return t.GetObjectInfo(ctx, bucket, object) @@ -574,7 +584,8 @@ func (t *tritonObjects) CopyObject(ctx context.Context, srcBucket, srcObject, de SourcePath: path.Join(mantaRoot, srcBucket, srcObject), LinkPath: path.Join(mantaRoot, destBucket, destObject), }); err != nil { - return objInfo, errors.Trace(err) + logger.LogIf(ctx, err) + return objInfo, err } return t.GetObjectInfo(ctx, destBucket, destObject) @@ -587,7 +598,8 @@ func (t *tritonObjects) DeleteObject(ctx context.Context, bucket, object string) if err := t.client.Objects().Delete(ctx, &storage.DeleteObjectInput{ ObjectPath: path.Join(mantaRoot, bucket, object), }); err != nil { - return errors.Trace(err) + logger.LogIf(ctx, err) + return err } return nil diff --git a/cmd/gateway/oss/gateway-oss.go b/cmd/gateway/oss/gateway-oss.go index e1433586d..5e49828a0 100644 --- a/cmd/gateway/oss/gateway-oss.go +++ b/cmd/gateway/oss/gateway-oss.go @@ -32,8 +32,8 @@ import ( "github.com/minio/cli" "github.com/minio/minio-go/pkg/policy" minio "github.com/minio/minio/cmd" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" - "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/hash" ) @@ -113,7 +113,7 @@ func ossGatewayMain(ctx *cli.Context) { // Validate gateway arguments. host := ctx.Args().First() - minio.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument") + logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument") minio.StartGateway(ctx, &OSS{host}) } @@ -161,7 +161,7 @@ func (g *OSS) Production() bool { // `X-Amz-Meta-` prefix and converted into `X-Oss-Meta-`. // // Header names are canonicalized as in http.Header. -func appendS3MetaToOSSOptions(opts []oss.Option, s3Metadata map[string]string) ([]oss.Option, error) { +func appendS3MetaToOSSOptions(ctx context.Context, opts []oss.Option, s3Metadata map[string]string) ([]oss.Option, error) { if opts == nil { opts = make([]oss.Option, 0, len(s3Metadata)) } @@ -173,7 +173,8 @@ func appendS3MetaToOSSOptions(opts []oss.Option, s3Metadata map[string]string) ( metaKey := k[len("X-Amz-Meta-"):] // NOTE(timonwong): OSS won't allow headers with underscore(_). if strings.Contains(metaKey, "_") { - return nil, errors.Trace(minio.UnsupportedMetadata{}) + logger.LogIf(ctx, minio.UnsupportedMetadata{}) + return nil, minio.UnsupportedMetadata{} } opts = append(opts, oss.Meta(metaKey, v)) case k == "X-Amz-Acl": @@ -271,15 +272,6 @@ func ossToObjectError(err error, params ...string) error { return nil } - e, ok := err.(*errors.Error) - if !ok { - // Code should be fixed if this function is called without doing errors.Trace() - // Else handling different situations in this function makes this function complicated. - minio.ErrorIf(err, "Expected type *Error") - return err - } - - err = e.Cause bucket := "" object := "" uploadID := "" @@ -298,7 +290,7 @@ func ossToObjectError(err error, params ...string) error { if !ok { // We don't interpret non OSS errors. As oss errors will // have StatusCode to help to convert to object errors. - return e + return err } switch ossErr.Code { @@ -330,8 +322,7 @@ func ossToObjectError(err error, params ...string) error { err = minio.InvalidPart{} } - e.Cause = err - return e + return err } // ossObjects implements gateway for Aliyun Object Storage Service. @@ -366,22 +357,26 @@ func ossIsValidBucketName(bucket string) bool { // MakeBucketWithLocation creates a new container on OSS backend. func (l *ossObjects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error { if !ossIsValidBucketName(bucket) { - return errors.Trace(minio.BucketNameInvalid{Bucket: bucket}) + logger.LogIf(ctx, minio.BucketNameInvalid{Bucket: bucket}) + return minio.BucketNameInvalid{Bucket: bucket} } err := l.Client.CreateBucket(bucket) - return ossToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return ossToObjectError(err, bucket) } // ossGeBucketInfo gets bucket metadata. -func ossGeBucketInfo(client *oss.Client, bucket string) (bi minio.BucketInfo, err error) { +func ossGeBucketInfo(ctx context.Context, client *oss.Client, bucket string) (bi minio.BucketInfo, err error) { if !ossIsValidBucketName(bucket) { - return bi, errors.Trace(minio.BucketNameInvalid{Bucket: bucket}) + logger.LogIf(ctx, minio.BucketNameInvalid{Bucket: bucket}) + return bi, minio.BucketNameInvalid{Bucket: bucket} } bgir, err := client.GetBucketInfo(bucket) if err != nil { - return bi, ossToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return bi, ossToObjectError(err, bucket) } return minio.BucketInfo{ @@ -392,7 +387,7 @@ func ossGeBucketInfo(client *oss.Client, bucket string) (bi minio.BucketInfo, er // GetBucketInfo gets bucket metadata. func (l *ossObjects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, err error) { - return ossGeBucketInfo(l.Client, bucket) + return ossGeBucketInfo(ctx, l.Client, bucket) } // ListBuckets lists all OSS buckets. @@ -401,7 +396,8 @@ func (l *ossObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketInf for { lbr, err := l.Client.ListBuckets(marker) if err != nil { - return nil, ossToObjectError(errors.Trace(err)) + logger.LogIf(ctx, err) + return nil, ossToObjectError(err) } for _, bi := range lbr.Buckets { @@ -424,7 +420,8 @@ func (l *ossObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketInf func (l *ossObjects) DeleteBucket(ctx context.Context, bucket string) error { err := l.Client.DeleteBucket(bucket) if err != nil { - return ossToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return ossToObjectError(err, bucket) } return nil } @@ -462,10 +459,11 @@ func fromOSSClientListObjectsResult(bucket string, lor oss.ListObjectsResult) mi } // ossListObjects lists all blobs in OSS bucket filtered by prefix. -func ossListObjects(client *oss.Client, bucket, prefix, marker, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) { +func ossListObjects(ctx context.Context, client *oss.Client, bucket, prefix, marker, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) { buck, err := client.Bucket(bucket) if err != nil { - return loi, ossToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return loi, ossToObjectError(err, bucket) } // maxKeys should default to 1000 or less. @@ -475,19 +473,20 @@ func ossListObjects(client *oss.Client, bucket, prefix, marker, delimiter string lor, err := buck.ListObjects(oss.Prefix(prefix), oss.Marker(marker), oss.Delimiter(delimiter), oss.MaxKeys(maxKeys)) if err != nil { - return loi, ossToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return loi, ossToObjectError(err, bucket) } return fromOSSClientListObjectsResult(bucket, lor), nil } // ossListObjectsV2 lists all blobs in OSS bucket filtered by prefix. -func ossListObjectsV2(client *oss.Client, bucket, prefix, continuationToken, delimiter string, maxKeys int, +func ossListObjectsV2(ctx context.Context, client *oss.Client, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) { // fetchOwner and startAfter are not supported and unused. marker := continuationToken - resultV1, err := ossListObjects(client, bucket, prefix, marker, delimiter, maxKeys) + resultV1, err := ossListObjects(ctx, client, bucket, prefix, marker, delimiter, maxKeys) if err != nil { return loi, err } @@ -503,13 +502,13 @@ func ossListObjectsV2(client *oss.Client, bucket, prefix, continuationToken, del // ListObjects lists all blobs in OSS bucket filtered by prefix. func (l *ossObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) { - return ossListObjects(l.Client, bucket, prefix, marker, delimiter, maxKeys) + return ossListObjects(ctx, l.Client, bucket, prefix, marker, delimiter, maxKeys) } // ListObjectsV2 lists all blobs in OSS bucket filtered by prefix func (l *ossObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) { - return ossListObjectsV2(l.Client, bucket, prefix, continuationToken, delimiter, maxKeys, fetchOwner, startAfter) + return ossListObjectsV2(ctx, l.Client, bucket, prefix, continuationToken, delimiter, maxKeys, fetchOwner, startAfter) } // ossGetObject reads an object on OSS. Supports additional @@ -518,14 +517,16 @@ func (l *ossObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continua // // startOffset indicates the starting read location of the object. // length indicates the total length of the object. -func ossGetObject(client *oss.Client, bucket, key string, startOffset, length int64, writer io.Writer, etag string) error { +func ossGetObject(ctx context.Context, client *oss.Client, bucket, key string, startOffset, length int64, writer io.Writer, etag string) error { if length < 0 && length != -1 { - return ossToObjectError(errors.Trace(fmt.Errorf("Invalid argument")), bucket, key) + logger.LogIf(ctx, fmt.Errorf("Invalid argument")) + return ossToObjectError(fmt.Errorf("Invalid argument"), bucket, key) } bkt, err := client.Bucket(bucket) if err != nil { - return ossToObjectError(errors.Trace(err), bucket, key) + logger.LogIf(ctx, err) + return ossToObjectError(err, bucket, key) } var opts []oss.Option @@ -535,12 +536,14 @@ func ossGetObject(client *oss.Client, bucket, key string, startOffset, length in object, err := bkt.GetObject(key, opts...) if err != nil { - return ossToObjectError(errors.Trace(err), bucket, key) + logger.LogIf(ctx, err) + return ossToObjectError(err, bucket, key) } defer object.Close() if _, err := io.Copy(writer, object); err != nil { - return ossToObjectError(errors.Trace(err), bucket, key) + logger.LogIf(ctx, err) + return ossToObjectError(err, bucket, key) } return nil } @@ -552,7 +555,7 @@ func ossGetObject(client *oss.Client, bucket, key string, startOffset, length in // startOffset indicates the starting read location of the object. // length indicates the total length of the object. func (l *ossObjects) GetObject(ctx context.Context, bucket, key string, startOffset, length int64, writer io.Writer, etag string) error { - return ossGetObject(l.Client, bucket, key, startOffset, length, writer, etag) + return ossGetObject(ctx, l.Client, bucket, key, startOffset, length, writer, etag) } func translatePlainError(err error) error { @@ -569,15 +572,17 @@ func translatePlainError(err error) error { } // ossGetObjectInfo reads object info and replies back ObjectInfo. -func ossGetObjectInfo(client *oss.Client, bucket, object string) (objInfo minio.ObjectInfo, err error) { +func ossGetObjectInfo(ctx context.Context, client *oss.Client, bucket, object string) (objInfo minio.ObjectInfo, err error) { bkt, err := client.Bucket(bucket) if err != nil { - return objInfo, ossToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return objInfo, ossToObjectError(err, bucket, object) } header, err := bkt.GetObjectDetailedMeta(object) if err != nil { - return objInfo, ossToObjectError(errors.Trace(translatePlainError(err)), bucket, object) + logger.LogIf(ctx, translatePlainError(err)) + return objInfo, ossToObjectError(translatePlainError(err), bucket, object) } // Build S3 metadata from OSS metadata @@ -600,40 +605,43 @@ func ossGetObjectInfo(client *oss.Client, bucket, object string) (objInfo minio. // GetObjectInfo reads object info and replies back ObjectInfo. func (l *ossObjects) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo minio.ObjectInfo, err error) { - return ossGetObjectInfo(l.Client, bucket, object) + return ossGetObjectInfo(ctx, l.Client, bucket, object) } // ossPutObject creates a new object with the incoming data. -func ossPutObject(client *oss.Client, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { +func ossPutObject(ctx context.Context, client *oss.Client, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { bkt, err := client.Bucket(bucket) if err != nil { - return objInfo, ossToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return objInfo, ossToObjectError(err, bucket, object) } // Build OSS metadata - opts, err := appendS3MetaToOSSOptions(nil, metadata) + opts, err := appendS3MetaToOSSOptions(ctx, nil, metadata) if err != nil { return objInfo, ossToObjectError(err, bucket, object) } err = bkt.PutObject(object, data, opts...) if err != nil { - return objInfo, ossToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return objInfo, ossToObjectError(err, bucket, object) } - return ossGetObjectInfo(client, bucket, object) + return ossGetObjectInfo(ctx, client, bucket, object) } // PutObject creates a new object with the incoming data. func (l *ossObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { - return ossPutObject(l.Client, bucket, object, data, metadata) + return ossPutObject(ctx, l.Client, bucket, object, data, metadata) } // CopyObject copies an object from source bucket to a destination bucket. func (l *ossObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) { bkt, err := l.Client.Bucket(srcBucket) if err != nil { - return objInfo, ossToObjectError(errors.Trace(err), srcBucket, srcObject) + logger.LogIf(ctx, err) + return objInfo, ossToObjectError(err, srcBucket, srcObject) } opts := make([]oss.Option, 0, len(srcInfo.UserDefined)+1) @@ -644,13 +652,14 @@ func (l *ossObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu opts = append(opts, oss.MetadataDirective(oss.MetaReplace)) // Build OSS metadata - opts, err = appendS3MetaToOSSOptions(opts, srcInfo.UserDefined) + opts, err = appendS3MetaToOSSOptions(ctx, opts, srcInfo.UserDefined) if err != nil { return objInfo, ossToObjectError(err, srcBucket, srcObject) } if _, err = bkt.CopyObjectTo(dstBucket, dstObject, srcObject, opts...); err != nil { - return objInfo, ossToObjectError(errors.Trace(err), srcBucket, srcObject) + logger.LogIf(ctx, err) + return objInfo, ossToObjectError(err, srcBucket, srcObject) } return l.GetObjectInfo(ctx, dstBucket, dstObject) } @@ -659,12 +668,14 @@ func (l *ossObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu func (l *ossObjects) DeleteObject(ctx context.Context, bucket, object string) error { bkt, err := l.Client.Bucket(bucket) if err != nil { - return ossToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return ossToObjectError(err, bucket, object) } err = bkt.DeleteObject(object) if err != nil { - return ossToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return ossToObjectError(err, bucket, object) } return nil } @@ -701,13 +712,15 @@ func fromOSSClientListMultipartsInfo(lmur oss.ListMultipartUploadResult) minio.L func (l *ossObjects) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, err error) { bkt, err := l.Client.Bucket(bucket) if err != nil { - return lmi, ossToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return lmi, ossToObjectError(err, bucket) } lmur, err := bkt.ListMultipartUploads(oss.Prefix(prefix), oss.KeyMarker(keyMarker), oss.UploadIDMarker(uploadIDMarker), oss.Delimiter(delimiter), oss.MaxUploads(maxUploads)) if err != nil { - return lmi, ossToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return lmi, ossToObjectError(err, bucket) } return fromOSSClientListMultipartsInfo(lmur), nil @@ -717,18 +730,20 @@ func (l *ossObjects) ListMultipartUploads(ctx context.Context, bucket, prefix, k func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) { bkt, err := l.Client.Bucket(bucket) if err != nil { - return uploadID, ossToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return uploadID, ossToObjectError(err, bucket, object) } // Build OSS metadata - opts, err := appendS3MetaToOSSOptions(nil, metadata) + opts, err := appendS3MetaToOSSOptions(ctx, nil, metadata) if err != nil { return uploadID, ossToObjectError(err, bucket, object) } lmur, err := bkt.InitiateMultipartUpload(object, opts...) if err != nil { - return uploadID, ossToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return uploadID, ossToObjectError(err, bucket, object) } return lmur.UploadID, nil @@ -738,7 +753,8 @@ func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object stri func (l *ossObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (pi minio.PartInfo, err error) { bkt, err := l.Client.Bucket(bucket) if err != nil { - return pi, ossToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return pi, ossToObjectError(err, bucket, object) } imur := oss.InitiateMultipartUploadResult{ @@ -749,7 +765,8 @@ func (l *ossObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID size := data.Size() up, err := bkt.UploadPart(imur, data, size, partID) if err != nil { - return pi, ossToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return pi, ossToObjectError(err, bucket, object) } return minio.PartInfo{ @@ -820,11 +837,12 @@ func (l *ossObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d bkt, err := l.Client.Bucket(destBucket) if err != nil { - return p, ossToObjectError(errors.Trace(err), destBucket) + logger.LogIf(ctx, err) + return p, ossToObjectError(err, destBucket) } // Build OSS metadata - opts, err := appendS3MetaToOSSOptions(nil, srcInfo.UserDefined) + opts, err := appendS3MetaToOSSOptions(ctx, nil, srcInfo.UserDefined) if err != nil { return p, ossToObjectError(err, srcBucket, srcObject) } @@ -835,7 +853,8 @@ func (l *ossObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d }, srcBucket, srcObject, startOffset, length, partID, opts...) if err != nil { - return p, ossToObjectError(errors.Trace(err), srcBucket, srcObject) + logger.LogIf(ctx, err) + return p, ossToObjectError(err, srcBucket, srcObject) } p.PartNumber = completePart.PartNumber @@ -847,7 +866,8 @@ func (l *ossObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d func (l *ossObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int) (lpi minio.ListPartsInfo, err error) { lupr, err := ossListObjectParts(l.Client, bucket, object, uploadID, partNumberMarker, maxParts) if err != nil { - return lpi, ossToObjectError(errors.Trace(err), bucket, object, uploadID) + logger.LogIf(ctx, err) + return lpi, ossToObjectError(err, bucket, object, uploadID) } return fromOSSClientListPartsInfo(lupr, partNumberMarker), nil @@ -857,7 +877,8 @@ func (l *ossObjects) ListObjectParts(ctx context.Context, bucket, object, upload func (l *ossObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error { bkt, err := l.Client.Bucket(bucket) if err != nil { - return ossToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return ossToObjectError(err, bucket, object) } err = bkt.AbortMultipartUpload(oss.InitiateMultipartUploadResult{ @@ -866,7 +887,8 @@ func (l *ossObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u UploadID: uploadID, }) if err != nil { - return ossToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return ossToObjectError(err, bucket, object) } return nil } @@ -876,7 +898,8 @@ func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object client := l.Client bkt, err := client.Bucket(bucket) if err != nil { - return oi, ossToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return oi, ossToObjectError(err, bucket, object) } // Error out if uploadedParts except last part sizing < 5MiB. @@ -886,7 +909,8 @@ func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object for lupr.IsTruncated { lupr, err = ossListObjectParts(client, bucket, object, uploadID, partNumberMarker, ossMaxParts) if err != nil { - return oi, ossToObjectError(errors.Trace(err), bucket, object, uploadID) + logger.LogIf(ctx, err) + return oi, ossToObjectError(err, bucket, object, uploadID) } uploadedParts := lupr.UploadedParts @@ -900,11 +924,16 @@ func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object for _, part := range uploadedParts { if part.Size < ossS3MinPartSize { - return oi, errors.Trace(minio.PartTooSmall{ + logger.LogIf(ctx, minio.PartTooSmall{ PartNumber: part.PartNumber, PartSize: int64(part.Size), PartETag: minio.ToS3ETag(part.ETag), }) + return oi, minio.PartTooSmall{ + PartNumber: part.PartNumber, + PartSize: int64(part.Size), + PartETag: minio.ToS3ETag(part.ETag), + } } } @@ -926,7 +955,8 @@ func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object _, err = bkt.CompleteMultipartUpload(imur, parts) if err != nil { - return oi, ossToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return oi, ossToObjectError(err, bucket, object) } return l.GetObjectInfo(ctx, bucket, object) @@ -940,13 +970,15 @@ func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object func (l *ossObjects) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error { bucketPolicies := policy.GetPolicies(policyInfo.Statements, bucket, "") if len(bucketPolicies) != 1 { - return errors.Trace(minio.NotImplemented{}) + logger.LogIf(ctx, minio.NotImplemented{}) + return minio.NotImplemented{} } prefix := bucket + "/*" // For all objects inside the bucket. for policyPrefix, bucketPolicy := range bucketPolicies { if policyPrefix != prefix { - return errors.Trace(minio.NotImplemented{}) + logger.LogIf(ctx, minio.NotImplemented{}) + return minio.NotImplemented{} } var acl oss.ACLType @@ -958,12 +990,14 @@ func (l *ossObjects) SetBucketPolicy(ctx context.Context, bucket string, policyI case policy.BucketPolicyReadWrite: acl = oss.ACLPublicReadWrite default: - return errors.Trace(minio.NotImplemented{}) + logger.LogIf(ctx, minio.NotImplemented{}) + return minio.NotImplemented{} } err := l.Client.SetBucketACL(bucket, acl) if err != nil { - return ossToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return ossToObjectError(err, bucket) } } @@ -974,20 +1008,23 @@ func (l *ossObjects) SetBucketPolicy(ctx context.Context, bucket string, policyI func (l *ossObjects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) { result, err := l.Client.GetBucketACL(bucket) if err != nil { - return policy.BucketAccessPolicy{}, ossToObjectError(errors.Trace(err)) + logger.LogIf(ctx, err) + return policy.BucketAccessPolicy{}, ossToObjectError(err) } policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"} switch result.ACL { case string(oss.ACLPrivate): // By default, all buckets starts with a "private" policy. - return policy.BucketAccessPolicy{}, ossToObjectError(errors.Trace(minio.PolicyNotFound{}), bucket) + logger.LogIf(ctx, minio.PolicyNotFound{}) + return policy.BucketAccessPolicy{}, ossToObjectError(minio.PolicyNotFound{}, bucket) case string(oss.ACLPublicRead): policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadOnly, bucket, "") case string(oss.ACLPublicReadWrite): policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadWrite, bucket, "") default: - return policy.BucketAccessPolicy{}, errors.Trace(minio.NotImplemented{}) + logger.LogIf(ctx, minio.NotImplemented{}) + return policy.BucketAccessPolicy{}, minio.NotImplemented{} } return policyInfo, nil @@ -997,7 +1034,8 @@ func (l *ossObjects) GetBucketPolicy(ctx context.Context, bucket string) (policy func (l *ossObjects) DeleteBucketPolicy(ctx context.Context, bucket string) error { err := l.Client.SetBucketACL(bucket, oss.ACLPrivate) if err != nil { - return ossToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return ossToObjectError(err, bucket) } return nil } diff --git a/cmd/gateway/oss/gateway-oss_test.go b/cmd/gateway/oss/gateway-oss_test.go index 03318ae1d..f90c019b7 100644 --- a/cmd/gateway/oss/gateway-oss_test.go +++ b/cmd/gateway/oss/gateway-oss_test.go @@ -17,6 +17,7 @@ package oss import ( + "context" "fmt" "net/http" "reflect" @@ -29,9 +30,9 @@ import ( ) func ossErrResponse(code string) error { - return errors.Trace(oss.ServiceError{ + return oss.ServiceError{ Code: code, - }) + } } func TestOSSToObjectError(t *testing.T) { @@ -116,7 +117,7 @@ func TestS3MetaToOSSOptions(t *testing.T) { headers = map[string]string{ "x-amz-meta-invalid_meta": "value", } - _, err = appendS3MetaToOSSOptions(nil, headers) + _, err = appendS3MetaToOSSOptions(context.Background(), nil, headers) if err = errors.Cause(err); err != nil { if _, ok := err.(minio.UnsupportedMetadata); !ok { t.Fatalf("Test failed with unexpected error %s, expected UnsupportedMetadata", err) @@ -133,7 +134,7 @@ func TestS3MetaToOSSOptions(t *testing.T) { "X-Amz-Meta-X-Amz-Matdesc": "{}", "X-Amz-Meta-X-Amz-Iv": "eWmyryl8kq+EVnnsE7jpOg==", } - opts, err := appendS3MetaToOSSOptions(nil, headers) + opts, err := appendS3MetaToOSSOptions(context.Background(), nil, headers) if err != nil { t.Fatalf("Test failed, with %s", err) } diff --git a/cmd/gateway/s3/gateway-s3.go b/cmd/gateway/s3/gateway-s3.go index 6fedb0e15..03c145572 100644 --- a/cmd/gateway/s3/gateway-s3.go +++ b/cmd/gateway/s3/gateway-s3.go @@ -24,8 +24,8 @@ import ( miniogo "github.com/minio/minio-go" "github.com/minio/minio-go/pkg/policy" "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" - "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/hash" minio "github.com/minio/minio/cmd" @@ -101,7 +101,7 @@ func s3GatewayMain(ctx *cli.Context) { // Validate gateway arguments. host := ctx.Args().First() // Validate gateway arguments. - minio.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument") + logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument") minio.StartGateway(ctx, &S3{host}) } @@ -173,7 +173,8 @@ func (l *s3Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) { func (l *s3Objects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error { err := l.Client.MakeBucket(bucket, location) if err != nil { - return minio.ErrorRespToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return minio.ErrorRespToObjectError(err, bucket) } return err } @@ -188,12 +189,14 @@ func (l *s3Objects) GetBucketInfo(ctx context.Context, bucket string) (bi minio. // access to these buckets. // Ref - http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html if s3utils.CheckValidBucketName(bucket) != nil { - return bi, errors.Trace(minio.BucketNameInvalid{Bucket: bucket}) + logger.LogIf(ctx, minio.BucketNameInvalid{Bucket: bucket}) + return bi, minio.BucketNameInvalid{Bucket: bucket} } buckets, err := l.Client.ListBuckets() if err != nil { - return bi, minio.ErrorRespToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return bi, minio.ErrorRespToObjectError(err, bucket) } for _, bi := range buckets { @@ -207,14 +210,16 @@ func (l *s3Objects) GetBucketInfo(ctx context.Context, bucket string) (bi minio. }, nil } - return bi, errors.Trace(minio.BucketNotFound{Bucket: bucket}) + logger.LogIf(ctx, minio.BucketNotFound{Bucket: bucket}) + return bi, minio.BucketNotFound{Bucket: bucket} } // ListBuckets lists all S3 buckets func (l *s3Objects) ListBuckets(ctx context.Context) ([]minio.BucketInfo, error) { buckets, err := l.Client.ListBuckets() if err != nil { - return nil, minio.ErrorRespToObjectError(errors.Trace(err)) + logger.LogIf(ctx, err) + return nil, minio.ErrorRespToObjectError(err) } b := make([]minio.BucketInfo, len(buckets)) @@ -232,7 +237,8 @@ func (l *s3Objects) ListBuckets(ctx context.Context) ([]minio.BucketInfo, error) func (l *s3Objects) DeleteBucket(ctx context.Context, bucket string) error { err := l.Client.RemoveBucket(bucket) if err != nil { - return minio.ErrorRespToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return minio.ErrorRespToObjectError(err, bucket) } return nil } @@ -241,7 +247,8 @@ func (l *s3Objects) DeleteBucket(ctx context.Context, bucket string) error { func (l *s3Objects) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, e error) { result, err := l.Client.ListObjects(bucket, prefix, marker, delimiter, maxKeys) if err != nil { - return loi, minio.ErrorRespToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return loi, minio.ErrorRespToObjectError(err, bucket) } return minio.FromMinioClientListBucketResult(bucket, result), nil @@ -251,7 +258,8 @@ func (l *s3Objects) ListObjects(ctx context.Context, bucket string, prefix strin func (l *s3Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, e error) { result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys) if err != nil { - return loi, minio.ErrorRespToObjectError(errors.Trace(err), bucket) + logger.LogIf(ctx, err) + return loi, minio.ErrorRespToObjectError(err, bucket) } return minio.FromMinioClientListBucketV2Result(bucket, result), nil @@ -265,23 +273,27 @@ func (l *s3Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat // length indicates the total length of the object. func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string) error { if length < 0 && length != -1 { - return minio.ErrorRespToObjectError(errors.Trace(minio.InvalidRange{}), bucket, key) + logger.LogIf(ctx, minio.InvalidRange{}) + return minio.ErrorRespToObjectError(minio.InvalidRange{}, bucket, key) } opts := miniogo.GetObjectOptions{} if startOffset >= 0 && length >= 0 { if err := opts.SetRange(startOffset, startOffset+length-1); err != nil { - return minio.ErrorRespToObjectError(errors.Trace(err), bucket, key) + logger.LogIf(ctx, err) + return minio.ErrorRespToObjectError(err, bucket, key) } } object, _, err := l.Client.GetObject(bucket, key, opts) if err != nil { - return minio.ErrorRespToObjectError(errors.Trace(err), bucket, key) + logger.LogIf(ctx, err) + return minio.ErrorRespToObjectError(err, bucket, key) } defer object.Close() if _, err := io.Copy(writer, object); err != nil { - return minio.ErrorRespToObjectError(errors.Trace(err), bucket, key) + logger.LogIf(ctx, err) + return minio.ErrorRespToObjectError(err, bucket, key) } return nil } @@ -290,7 +302,8 @@ func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, st func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object string) (objInfo minio.ObjectInfo, err error) { oi, err := l.Client.StatObject(bucket, object, miniogo.StatObjectOptions{}) if err != nil { - return minio.ObjectInfo{}, minio.ErrorRespToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return minio.ObjectInfo{}, minio.ErrorRespToObjectError(err, bucket, object) } return minio.FromMinioClientObjectInfo(bucket, oi), nil @@ -300,7 +313,8 @@ func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object str func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { oi, err := l.Client.PutObject(bucket, object, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), minio.ToMinioClientMetadata(metadata)) if err != nil { - return objInfo, minio.ErrorRespToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return objInfo, minio.ErrorRespToObjectError(err, bucket, object) } return minio.FromMinioClientObjectInfo(bucket, oi), nil @@ -315,7 +329,8 @@ func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject srcInfo.UserDefined["x-amz-metadata-directive"] = "REPLACE" srcInfo.UserDefined["x-amz-copy-source-if-match"] = srcInfo.ETag if _, err = l.Client.CopyObject(srcBucket, srcObject, dstBucket, dstObject, srcInfo.UserDefined); err != nil { - return objInfo, minio.ErrorRespToObjectError(errors.Trace(err), srcBucket, srcObject) + logger.LogIf(ctx, err) + return objInfo, minio.ErrorRespToObjectError(err, srcBucket, srcObject) } return l.GetObjectInfo(ctx, dstBucket, dstObject) } @@ -324,7 +339,8 @@ func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject func (l *s3Objects) DeleteObject(ctx context.Context, bucket string, object string) error { err := l.Client.RemoveObject(bucket, object) if err != nil { - return minio.ErrorRespToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return minio.ErrorRespToObjectError(err, bucket, object) } return nil @@ -346,7 +362,8 @@ func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, objec opts := miniogo.PutObjectOptions{UserMetadata: metadata} uploadID, err = l.Client.NewMultipartUpload(bucket, object, opts) if err != nil { - return uploadID, minio.ErrorRespToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return uploadID, minio.ErrorRespToObjectError(err, bucket, object) } return uploadID, nil } @@ -355,7 +372,8 @@ func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, objec func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader) (pi minio.PartInfo, e error) { info, err := l.Client.PutObjectPart(bucket, object, uploadID, partID, data, data.Size(), data.MD5Base64String(), data.SHA256HexString()) if err != nil { - return pi, minio.ErrorRespToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return pi, minio.ErrorRespToObjectError(err, bucket, object) } return minio.FromMinioClientObjectPart(info), nil @@ -372,7 +390,8 @@ func (l *s3Objects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, de completePart, err := l.Client.CopyObjectPart(srcBucket, srcObject, destBucket, destObject, uploadID, partID, startOffset, length, srcInfo.UserDefined) if err != nil { - return p, minio.ErrorRespToObjectError(errors.Trace(err), srcBucket, srcObject) + logger.LogIf(ctx, err) + return p, minio.ErrorRespToObjectError(err, srcBucket, srcObject) } p.PartNumber = completePart.PartNumber p.ETag = completePart.ETag @@ -392,14 +411,16 @@ func (l *s3Objects) ListObjectParts(ctx context.Context, bucket string, object s // AbortMultipartUpload aborts a ongoing multipart upload func (l *s3Objects) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string) error { err := l.Client.AbortMultipartUpload(bucket, object, uploadID) - return minio.ErrorRespToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return minio.ErrorRespToObjectError(err, bucket, object) } // CompleteMultipartUpload completes ongoing multipart upload and finalizes object func (l *s3Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart) (oi minio.ObjectInfo, e error) { err := l.Client.CompleteMultipartUpload(bucket, object, uploadID, minio.ToMinioClientCompleteParts(uploadedParts)) if err != nil { - return oi, minio.ErrorRespToObjectError(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return oi, minio.ErrorRespToObjectError(err, bucket, object) } return l.GetObjectInfo(ctx, bucket, object) @@ -408,7 +429,8 @@ func (l *s3Objects) CompleteMultipartUpload(ctx context.Context, bucket string, // SetBucketPolicy sets policy on bucket func (l *s3Objects) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error { if err := l.Client.PutBucketPolicy(bucket, policyInfo); err != nil { - return minio.ErrorRespToObjectError(errors.Trace(err), bucket, "") + logger.LogIf(ctx, err) + return minio.ErrorRespToObjectError(err, bucket, "") } return nil @@ -418,7 +440,8 @@ func (l *s3Objects) SetBucketPolicy(ctx context.Context, bucket string, policyIn func (l *s3Objects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) { policyInfo, err := l.Client.GetBucketPolicy(bucket) if err != nil { - return policy.BucketAccessPolicy{}, minio.ErrorRespToObjectError(errors.Trace(err), bucket, "") + logger.LogIf(ctx, err) + return policy.BucketAccessPolicy{}, minio.ErrorRespToObjectError(err, bucket, "") } return policyInfo, nil } @@ -426,7 +449,8 @@ func (l *s3Objects) GetBucketPolicy(ctx context.Context, bucket string) (policy. // DeleteBucketPolicy deletes all policies on bucket func (l *s3Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error { if err := l.Client.PutBucketPolicy(bucket, policy.BucketAccessPolicy{}); err != nil { - return minio.ErrorRespToObjectError(errors.Trace(err), bucket, "") + logger.LogIf(ctx, err) + return minio.ErrorRespToObjectError(err, bucket, "") } return nil } diff --git a/cmd/gateway/s3/gateway-s3_test.go b/cmd/gateway/s3/gateway-s3_test.go index 45ad0d6d5..93c04f8d2 100644 --- a/cmd/gateway/s3/gateway-s3_test.go +++ b/cmd/gateway/s3/gateway-s3_test.go @@ -119,9 +119,9 @@ func TestS3ToObjectError(t *testing.T) { } for i, tc := range testCases { - actualErr := minio.ErrorRespToObjectError(errors.Trace(tc.inputErr), tc.bucket, tc.object) + actualErr := minio.ErrorRespToObjectError(tc.inputErr, tc.bucket, tc.object) if e, ok := actualErr.(*errors.Error); ok && e.Cause.Error() != tc.expectedErr.Error() { - t.Errorf("Test case %d: Expected error %v but received error %v", i+1, tc.expectedErr, e) + t.Errorf("Test case %d: Expected error %v but received error %v", i+1, tc.expectedErr, actualErr) } } } diff --git a/cmd/gateway/sia/gateway-sia.go b/cmd/gateway/sia/gateway-sia.go index cf4c4fd08..c8780ca16 100644 --- a/cmd/gateway/sia/gateway-sia.go +++ b/cmd/gateway/sia/gateway-sia.go @@ -24,7 +24,6 @@ import ( "encoding/json" "fmt" "io" - "log" "net/http" "net/url" "os" @@ -38,8 +37,8 @@ import ( "github.com/minio/cli" "github.com/minio/minio-go/pkg/set" minio "github.com/minio/minio/cmd" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" - "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/hash" ) @@ -112,7 +111,7 @@ func siaGatewayMain(ctx *cli.Context) { // Validate gateway arguments. host := ctx.Args().First() // Validate gateway arguments. - minio.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument") + logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument") minio.StartGateway(ctx, &Sia{host}) } @@ -164,9 +163,9 @@ func (g *Sia) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) colorBlue := color.New(color.FgBlue).SprintfFunc() colorBold := color.New(color.Bold).SprintFunc() - log.Println(colorBlue("\nSia Gateway Configuration:")) - log.Println(colorBlue(" Sia Daemon API Address:") + colorBold(fmt.Sprintf(" %s\n", sia.Address))) - log.Println(colorBlue(" Sia Temp Directory:") + colorBold(fmt.Sprintf(" %s\n", sia.TempDir))) + logger.Println(colorBlue("\nSia Gateway Configuration:")) + logger.Println(colorBlue(" Sia Daemon API Address:") + colorBold(fmt.Sprintf(" %s\n", sia.Address))) + logger.Println(colorBlue(" Sia Temp Directory:") + colorBold(fmt.Sprintf(" %s\n", sia.TempDir))) return sia, nil } @@ -217,10 +216,11 @@ func (s MethodNotSupported) Error() string { // apiGet wraps a GET request with a status code check, such that if the GET does // not return 2xx, the error will be read and returned. The response body is // not closed. -func apiGet(addr, call, apiPassword string) (*http.Response, error) { +func apiGet(ctx context.Context, addr, call, apiPassword string) (*http.Response, error) { req, err := http.NewRequest("GET", "http://"+addr+call, nil) if err != nil { - return nil, errors.Trace(err) + logger.LogIf(ctx, err) + return nil, err } req.Header.Set("User-Agent", "Sia-Agent") if apiPassword != "" { @@ -228,15 +228,18 @@ func apiGet(addr, call, apiPassword string) (*http.Response, error) { } resp, err := http.DefaultClient.Do(req) if err != nil { - return nil, errors.Trace(err) + logger.LogIf(ctx, err) + return nil, err } if resp.StatusCode == http.StatusNotFound { resp.Body.Close() + logger.LogIf(ctx, MethodNotSupported{call}) return nil, MethodNotSupported{call} } if non2xx(resp.StatusCode) { err := decodeError(resp) resp.Body.Close() + logger.LogIf(ctx, err) return nil, err } return resp, nil @@ -245,7 +248,7 @@ func apiGet(addr, call, apiPassword string) (*http.Response, error) { // apiPost wraps a POST request with a status code check, such that if the POST // does not return 2xx, the error will be read and returned. The response body // is not closed. -func apiPost(addr, call, vals, apiPassword string) (*http.Response, error) { +func apiPost(ctx context.Context, addr, call, vals, apiPassword string) (*http.Response, error) { req, err := http.NewRequest("POST", "http://"+addr+call, strings.NewReader(vals)) if err != nil { return nil, err @@ -257,7 +260,8 @@ func apiPost(addr, call, vals, apiPassword string) (*http.Response, error) { } resp, err := http.DefaultClient.Do(req) if err != nil { - return nil, errors.Trace(err) + logger.LogIf(ctx, err) + return nil, err } if resp.StatusCode == http.StatusNotFound { @@ -275,8 +279,8 @@ func apiPost(addr, call, vals, apiPassword string) (*http.Response, error) { // post makes an API call and discards the response. An error is returned if // the response status is not 2xx. -func post(addr, call, vals, apiPassword string) error { - resp, err := apiPost(addr, call, vals, apiPassword) +func post(ctx context.Context, addr, call, vals, apiPassword string) error { + resp, err := apiPost(ctx, addr, call, vals, apiPassword) if err != nil { return err } @@ -285,24 +289,26 @@ func post(addr, call, vals, apiPassword string) error { } // list makes a lists all the uploaded files, decodes the json response. -func list(addr string, apiPassword string, obj *renterFiles) error { - resp, err := apiGet(addr, "/renter/files", apiPassword) +func list(ctx context.Context, addr string, apiPassword string, obj *renterFiles) error { + resp, err := apiGet(ctx, addr, "/renter/files", apiPassword) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode == http.StatusNoContent { + logger.LogIf(ctx, fmt.Errorf("Expecting a response, but API returned %s", resp.Status)) return fmt.Errorf("Expecting a response, but API returned %s", resp.Status) } - - return json.NewDecoder(resp.Body).Decode(obj) + err = json.NewDecoder(resp.Body).Decode(obj) + logger.LogIf(ctx, err) + return err } // get makes an API call and discards the response. An error is returned if the // responsee status is not 2xx. -func get(addr, call, apiPassword string) error { - resp, err := apiGet(addr, call, apiPassword) +func get(ctx context.Context, addr, call, apiPassword string) error { + resp, err := apiGet(ctx, addr, call, apiPassword) if err != nil { return err } @@ -336,7 +342,7 @@ func (s *siaObjects) MakeBucketWithLocation(ctx context.Context, bucket, locatio sha256sum := sha256.Sum256([]byte(bucket)) var siaObj = path.Join(s.RootDir, bucket, hex.EncodeToString(sha256sum[:])) - return post(s.Address, "/renter/upload/"+siaObj, "source="+srcFile, s.password) + return post(ctx, s.Address, "/renter/upload/"+siaObj, "source="+srcFile, s.password) } // GetBucketInfo gets bucket metadata. @@ -347,7 +353,7 @@ func (s *siaObjects) GetBucketInfo(ctx context.Context, bucket string) (bi minio dstFile := path.Join(s.TempDir, minio.MustGetUUID()) defer os.Remove(dstFile) - if err := get(s.Address, "/renter/download/"+siaObj+"?destination="+url.QueryEscape(dstFile), s.password); err != nil { + if err := get(ctx, s.Address, "/renter/download/"+siaObj+"?destination="+url.QueryEscape(dstFile), s.password); err != nil { return bi, err } return minio.BucketInfo{Name: bucket}, nil @@ -355,7 +361,7 @@ func (s *siaObjects) GetBucketInfo(ctx context.Context, bucket string) (bi minio // ListBuckets will detect and return existing buckets on Sia. func (s *siaObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketInfo, err error) { - sObjs, serr := s.listRenterFiles("") + sObjs, serr := s.listRenterFiles(ctx, "") if serr != nil { return buckets, serr } @@ -388,11 +394,11 @@ func (s *siaObjects) DeleteBucket(ctx context.Context, bucket string) error { sha256sum := sha256.Sum256([]byte(bucket)) var siaObj = path.Join(s.RootDir, bucket, hex.EncodeToString(sha256sum[:])) - return post(s.Address, "/renter/delete/"+siaObj, "", s.password) + return post(ctx, s.Address, "/renter/delete/"+siaObj, "", s.password) } func (s *siaObjects) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) { - siaObjs, siaErr := s.listRenterFiles(bucket) + siaObjs, siaErr := s.listRenterFiles(ctx, bucket) if siaErr != nil { return loi, siaErr } @@ -429,7 +435,7 @@ func (s *siaObjects) GetObject(ctx context.Context, bucket string, object string defer os.Remove(dstFile) var siaObj = path.Join(s.RootDir, bucket, object) - if err := get(s.Address, "/renter/download/"+siaObj+"?destination="+url.QueryEscape(dstFile), s.password); err != nil { + if err := get(ctx, s.Address, "/renter/download/"+siaObj+"?destination="+url.QueryEscape(dstFile), s.password); err != nil { return err } @@ -459,11 +465,16 @@ func (s *siaObjects) GetObject(ctx context.Context, bucket string, object string // Reply back invalid range if the input offset and length fall out of range. if startOffset > size || startOffset+length > size { - return errors.Trace(minio.InvalidRange{ + logger.LogIf(ctx, minio.InvalidRange{ OffsetBegin: startOffset, OffsetEnd: length, ResourceSize: size, }) + return minio.InvalidRange{ + OffsetBegin: startOffset, + OffsetEnd: length, + ResourceSize: size, + } } // Allocate a staging buffer. @@ -476,10 +487,10 @@ func (s *siaObjects) GetObject(ctx context.Context, bucket string, object string // findSiaObject retrieves the siaObjectInfo for the Sia object with the given // Sia path name. -func (s *siaObjects) findSiaObject(bucket, object string) (siaObjectInfo, error) { +func (s *siaObjects) findSiaObject(ctx context.Context, bucket, object string) (siaObjectInfo, error) { siaPath := path.Join(s.RootDir, bucket, object) - sObjs, err := s.listRenterFiles("") + sObjs, err := s.listRenterFiles(ctx, "") if err != nil { return siaObjectInfo{}, err } @@ -489,16 +500,19 @@ func (s *siaObjects) findSiaObject(bucket, object string) (siaObjectInfo, error) return sObj, nil } } - - return siaObjectInfo{}, errors.Trace(minio.ObjectNotFound{ + logger.LogIf(ctx, minio.ObjectNotFound{ Bucket: bucket, Object: object, }) + return siaObjectInfo{}, minio.ObjectNotFound{ + Bucket: bucket, + Object: object, + } } // GetObjectInfo reads object info and replies back ObjectInfo func (s *siaObjects) GetObjectInfo(ctx context.Context, bucket string, object string) (minio.ObjectInfo, error) { - so, err := s.findSiaObject(bucket, object) + so, err := s.findSiaObject(ctx, bucket, object) if err != nil { return minio.ObjectInfo{}, err } @@ -527,11 +541,11 @@ func (s *siaObjects) PutObject(ctx context.Context, bucket string, object string return objInfo, err } - if err = post(s.Address, "/renter/upload/"+path.Join(s.RootDir, bucket, object), "source="+srcFile, s.password); err != nil { + if err = post(ctx, s.Address, "/renter/upload/"+path.Join(s.RootDir, bucket, object), "source="+srcFile, s.password); err != nil { os.Remove(srcFile) return objInfo, err } - defer s.deleteTempFileWhenUploadCompletes(srcFile, bucket, object) + defer s.deleteTempFileWhenUploadCompletes(ctx, srcFile, bucket, object) return minio.ObjectInfo{ Name: object, @@ -546,7 +560,7 @@ func (s *siaObjects) PutObject(ctx context.Context, bucket string, object string func (s *siaObjects) DeleteObject(ctx context.Context, bucket string, object string) error { // Tell Sia daemon to delete the object var siaObj = path.Join(s.RootDir, bucket, object) - return post(s.Address, "/renter/delete/"+siaObj, "", s.password) + return post(ctx, s.Address, "/renter/delete/"+siaObj, "", s.password) } // siaObjectInfo represents object info stored on Sia @@ -565,10 +579,10 @@ type renterFiles struct { } // listRenterFiles will return a list of existing objects in the bucket provided -func (s *siaObjects) listRenterFiles(bucket string) (siaObjs []siaObjectInfo, err error) { +func (s *siaObjects) listRenterFiles(ctx context.Context, bucket string) (siaObjs []siaObjectInfo, err error) { // Get list of all renter files var rf renterFiles - if err = list(s.Address, s.password, &rf); err != nil { + if err = list(ctx, s.Address, s.password, &rf); err != nil { return siaObjs, err } @@ -592,16 +606,15 @@ func (s *siaObjects) listRenterFiles(bucket string) (siaObjs []siaObjectInfo, er // deleteTempFileWhenUploadCompletes checks the status of a Sia file upload // until it reaches 100% upload progress, then deletes the local temp copy from // the filesystem. -func (s *siaObjects) deleteTempFileWhenUploadCompletes(tempFile string, bucket, object string) { +func (s *siaObjects) deleteTempFileWhenUploadCompletes(ctx context.Context, tempFile string, bucket, object string) { var soi siaObjectInfo // Wait until 100% upload instead of 1x redundancy because if we delete // after 1x redundancy, the user has to pay the cost of other hosts // redistributing the file. for soi.UploadProgress < 100.0 { var err error - soi, err = s.findSiaObject(bucket, object) + soi, err = s.findSiaObject(ctx, bucket, object) if err != nil { - minio.ErrorIf(err, "Unable to find file uploaded to Sia path %s/%s", bucket, object) break } diff --git a/cmd/handler-utils.go b/cmd/handler-utils.go index 3db707f2d..67c179e3f 100644 --- a/cmd/handler-utils.go +++ b/cmd/handler-utils.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "io" "mime/multipart" "net" @@ -24,7 +25,7 @@ import ( "net/url" "strings" - "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" httptracer "github.com/minio/minio/pkg/handlers" ) @@ -36,7 +37,7 @@ func parseLocationConstraint(r *http.Request) (location string, s3Error APIError locationConstraint := createBucketLocationConfiguration{} err := xmlDecoder(r.Body, &locationConstraint, r.ContentLength) if err != nil && err != io.EOF { - errorIf(err, "Unable to xml decode location constraint") + logger.LogIf(context.Background(), err) // Treat all other failures as XML parsing errors. return "", ErrMalformedXML } // else for both err as nil or io.EOF @@ -113,9 +114,10 @@ var userMetadataKeyPrefixes = []string{ } // extractMetadataFromHeader extracts metadata from HTTP header. -func extractMetadataFromHeader(header http.Header) (map[string]string, error) { +func extractMetadataFromHeader(ctx context.Context, header http.Header) (map[string]string, error) { if header == nil { - return nil, errors.Trace(errInvalidArgument) + logger.LogIf(ctx, errInvalidArgument) + return nil, errInvalidArgument } metadata := make(map[string]string) @@ -134,7 +136,8 @@ func extractMetadataFromHeader(header http.Header) (map[string]string, error) { // Go through all other headers for any additional headers that needs to be saved. for key := range header { if key != http.CanonicalHeaderKey(key) { - return nil, errors.Trace(errInvalidArgument) + logger.LogIf(ctx, errInvalidArgument) + return nil, errInvalidArgument } for _, prefix := range userMetadataKeyPrefixes { if strings.HasPrefix(key, prefix) { @@ -187,12 +190,13 @@ func trimAwsChunkedContentEncoding(contentEnc string) (trimmedContentEnc string) } // Validate form field size for s3 specification requirement. -func validateFormFieldSize(formValues http.Header) error { +func validateFormFieldSize(ctx context.Context, formValues http.Header) error { // Iterate over form values for k := range formValues { // Check if value's field exceeds S3 limit if int64(len(formValues.Get(k))) > maxFormFieldSize { - return errors.Trace(errSizeUnexpected) + logger.LogIf(ctx, errSizeUnexpected) + return errSizeUnexpected } } @@ -201,7 +205,7 @@ func validateFormFieldSize(formValues http.Header) error { } // Extract form fields and file data from a HTTP POST Policy -func extractPostPolicyFormValues(form *multipart.Form) (filePart io.ReadCloser, fileName string, fileSize int64, formValues http.Header, err error) { +func extractPostPolicyFormValues(ctx context.Context, form *multipart.Form) (filePart io.ReadCloser, fileName string, fileSize int64, formValues http.Header, err error) { /// HTML Form values fileName = "" @@ -212,7 +216,7 @@ func extractPostPolicyFormValues(form *multipart.Form) (filePart io.ReadCloser, } // Validate form values. - if err = validateFormFieldSize(formValues); err != nil { + if err = validateFormFieldSize(ctx, formValues); err != nil { return nil, "", 0, nil, err } @@ -221,7 +225,8 @@ func extractPostPolicyFormValues(form *multipart.Form) (filePart io.ReadCloser, canonicalFormName := http.CanonicalHeaderKey(k) if canonicalFormName == "File" { if len(v) == 0 { - return nil, "", 0, nil, errors.Trace(errInvalidArgument) + logger.LogIf(ctx, errInvalidArgument) + return nil, "", 0, nil, errInvalidArgument } // Fetch fileHeader which has the uploaded file information fileHeader := v[0] @@ -230,17 +235,20 @@ func extractPostPolicyFormValues(form *multipart.Form) (filePart io.ReadCloser, // Open the uploaded part filePart, err = fileHeader.Open() if err != nil { - return nil, "", 0, nil, errors.Trace(err) + logger.LogIf(ctx, err) + return nil, "", 0, nil, err } // Compute file size fileSize, err = filePart.(io.Seeker).Seek(0, 2) if err != nil { - return nil, "", 0, nil, errors.Trace(err) + logger.LogIf(ctx, err) + return nil, "", 0, nil, err } // Reset Seek to the beginning _, err = filePart.(io.Seeker).Seek(0, 0) if err != nil { - return nil, "", 0, nil, errors.Trace(err) + logger.LogIf(ctx, err) + return nil, "", 0, nil, err } // File found and ready for reading break @@ -276,7 +284,10 @@ func getResource(path string, host string, domain string) (string, error) { // In bucket.mydomain.com:9000, strip out :9000 var err error if host, _, err = net.SplitHostPort(host); err != nil { - errorIf(err, "Unable to split %s", host) + reqInfo := (&logger.ReqInfo{}).AppendTags("host", host) + reqInfo.AppendTags("path", path) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) return "", err } } diff --git a/cmd/handler-utils_test.go b/cmd/handler-utils_test.go index 0e9e84151..9b359a858 100644 --- a/cmd/handler-utils_test.go +++ b/cmd/handler-utils_test.go @@ -18,6 +18,7 @@ package cmd import ( "bytes" + "context" "encoding/xml" "io/ioutil" "net/http" @@ -114,7 +115,7 @@ func TestValidateFormFieldSize(t *testing.T) { // Run validate form field size check under all test cases. for i, testCase := range testCases { - err := validateFormFieldSize(testCase.header) + err := validateFormFieldSize(context.Background(), testCase.header) if err != nil { if errors.Cause(err).Error() != testCase.err.Error() { t.Errorf("Test %d: Expected error %s, got %s", i+1, testCase.err, err) @@ -180,7 +181,7 @@ func TestExtractMetadataHeaders(t *testing.T) { // Validate if the extracting headers. for i, testCase := range testCases { - metadata, err := extractMetadataFromHeader(testCase.header) + metadata, err := extractMetadataFromHeader(context.Background(), testCase.header) if err != nil && !testCase.shouldFail { t.Fatalf("Test %d failed to extract metadata: %v", i+1, err) } diff --git a/cmd/http/listener.go b/cmd/http/listener.go index b261f4ba1..08bff7f3b 100644 --- a/cmd/http/listener.go +++ b/cmd/http/listener.go @@ -17,8 +17,8 @@ package http import ( + "context" "crypto/tls" - "errors" "fmt" "io" "net" @@ -28,6 +28,8 @@ import ( "sync" "syscall" "time" + + "github.com/minio/minio/cmd/logger" ) var sslRequiredErrMsg = []byte("HTTP/1.0 403 Forbidden\r\n\r\nSSL required") @@ -85,9 +87,8 @@ type httpListener struct { tcpKeepAliveTimeout time.Duration readTimeout time.Duration writeTimeout time.Duration - updateBytesReadFunc func(int) // function to be called to update bytes read in BufConn. - updateBytesWrittenFunc func(int) // function to be called to update bytes written in BufConn. - errorLogFunc func(error, string, ...interface{}) // function to be called on errors. + updateBytesReadFunc func(int) // function to be called to update bytes read in BufConn. + updateBytesWrittenFunc func(int) // function to be called to update bytes written in BufConn. } // isRoutineNetErr returns true if error is due to a network timeout, @@ -139,17 +140,16 @@ func (listener *httpListener) start() { // Peek bytes of maximum length of all HTTP methods. data, err := bufconn.Peek(methodMaxLen) if err != nil { - if listener.errorLogFunc != nil { - // Peek could fail legitimately when clients abruptly close - // connection. E.g. Chrome browser opens connections speculatively to - // speed up loading of a web page. Peek may also fail due to network - // saturation on a transport with read timeout set. All other kind of - // errors should be logged for further investigation. Thanks @brendanashworth. - if !isRoutineNetErr(err) { - listener.errorLogFunc(err, - "Error in reading from new connection %s at server %s", - bufconn.RemoteAddr(), bufconn.LocalAddr()) - } + // Peek could fail legitimately when clients abruptly close + // connection. E.g. Chrome browser opens connections speculatively to + // speed up loading of a web page. Peek may also fail due to network + // saturation on a transport with read timeout set. All other kind of + // errors should be logged for further investigation. Thanks @brendanashworth. + if !isRoutineNetErr(err) { + reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", bufconn.RemoteAddr().String()) + reqInfo.AppendTags("localAddr", bufconn.LocalAddr().String()) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) } bufconn.Close() return @@ -172,12 +172,11 @@ func (listener *httpListener) start() { if listener.tlsConfig != nil { // As the listener is configured with TLS, try to do TLS handshake, drop the connection if it fails. tlsConn := tls.Server(bufconn, listener.tlsConfig) - if err := tlsConn.Handshake(); err != nil { - if listener.errorLogFunc != nil { - listener.errorLogFunc(err, - "TLS handshake failed with new connection %s at server %s", - bufconn.RemoteAddr(), bufconn.LocalAddr()) - } + if err = tlsConn.Handshake(); err != nil { + reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", bufconn.RemoteAddr().String()) + reqInfo.AppendTags("localAddr", bufconn.LocalAddr().String()) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) bufconn.Close() return } @@ -187,12 +186,13 @@ func (listener *httpListener) start() { listener.updateBytesReadFunc, listener.updateBytesWrittenFunc) // Peek bytes of maximum length of all HTTP methods. - data, err := bufconn.Peek(methodMaxLen) + data, err = bufconn.Peek(methodMaxLen) if err != nil { - if !isRoutineNetErr(err) && listener.errorLogFunc != nil { - listener.errorLogFunc(err, - "Error in reading from new TLS connection %s at server %s", - bufconn.RemoteAddr(), bufconn.LocalAddr()) + if !isRoutineNetErr(err) { + reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", bufconn.RemoteAddr().String()) + reqInfo.AppendTags("localAddr", bufconn.LocalAddr().String()) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) } bufconn.Close() return @@ -205,12 +205,10 @@ func (listener *httpListener) start() { return } } - - if listener.errorLogFunc != nil { - listener.errorLogFunc(errors.New("junk message"), - "Received non-HTTP message from new connection %s at server %s", - bufconn.RemoteAddr(), bufconn.LocalAddr()) - } + reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", bufconn.RemoteAddr().String()) + reqInfo.AppendTags("localAddr", bufconn.LocalAddr().String()) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) bufconn.Close() return @@ -299,8 +297,7 @@ func newHTTPListener(serverAddrs []string, readTimeout time.Duration, writeTimeout time.Duration, updateBytesReadFunc func(int), - updateBytesWrittenFunc func(int), - errorLogFunc func(error, string, ...interface{})) (listener *httpListener, err error) { + updateBytesWrittenFunc func(int)) (listener *httpListener, err error) { var tcpListeners []*net.TCPListener // Close all opened listeners on error @@ -337,7 +334,6 @@ func newHTTPListener(serverAddrs []string, writeTimeout: writeTimeout, updateBytesReadFunc: updateBytesReadFunc, updateBytesWrittenFunc: updateBytesWrittenFunc, - errorLogFunc: errorLogFunc, } listener.start() diff --git a/cmd/http/listener_test.go b/cmd/http/listener_test.go index a6f474cf8..b6452f152 100644 --- a/cmd/http/listener_test.go +++ b/cmd/http/listener_test.go @@ -19,6 +19,7 @@ package http import ( "bufio" "bytes" + "context" "crypto/tls" "errors" "fmt" @@ -205,7 +206,7 @@ func TestNewHTTPListener(t *testing.T) { writeTimeout time.Duration updateBytesReadFunc func(int) updateBytesWrittenFunc func(int) - errorLogFunc func(error, string, ...interface{}) + errorLogFunc func(context.Context, error) expectedErr error }{ {[]string{"93.184.216.34:65432"}, nil, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, nil, errors.New(remoteAddrErrMsg)}, @@ -227,7 +228,6 @@ func TestNewHTTPListener(t *testing.T) { testCase.writeTimeout, testCase.updateBytesReadFunc, testCase.updateBytesWrittenFunc, - testCase.errorLogFunc, ) if testCase.expectedErr == nil { @@ -279,7 +279,6 @@ func TestHTTPListenerStartClose(t *testing.T) { time.Duration(0), nil, nil, - nil, ) if err != nil { t.Fatalf("Test %d: error: expected = , got = %v", i+1, err) @@ -327,7 +326,6 @@ func TestHTTPListenerAddr(t *testing.T) { time.Duration(0), nil, nil, - nil, ) if err != nil { t.Fatalf("Test %d: error: expected = , got = %v", i+1, err) @@ -372,7 +370,6 @@ func TestHTTPListenerAddrs(t *testing.T) { time.Duration(0), nil, nil, - nil, ) if err != nil { t.Fatalf("Test %d: error: expected = , got = %v", i+1, err) @@ -419,7 +416,6 @@ func TestHTTPListenerAccept(t *testing.T) { time.Duration(0), nil, nil, - nil, ) if err != nil { t.Fatalf("Test %d: error: expected = , got = %v", i+1, err) @@ -480,11 +476,6 @@ func TestHTTPListenerAccept(t *testing.T) { func TestHTTPListenerAcceptPeekError(t *testing.T) { tlsConfig := getTLSConfig(t) nonLoopBackIP := getNonLoopBackIP(t) - errorFunc := func(err error, template string, args ...interface{}) { - msg := fmt.Sprintf("error: %v. ", err) - msg += fmt.Sprintf(template, args...) - fmt.Println(msg) - } testCases := []struct { serverAddrs []string @@ -504,7 +495,6 @@ func TestHTTPListenerAcceptPeekError(t *testing.T) { time.Duration(0), nil, nil, - errorFunc, ) if err != nil { t.Fatalf("Test %d: error: expected = , got = %v", i+1, err) @@ -540,11 +530,6 @@ func TestHTTPListenerAcceptPeekError(t *testing.T) { func TestHTTPListenerAcceptTLSError(t *testing.T) { tlsConfig := getTLSConfig(t) nonLoopBackIP := getNonLoopBackIP(t) - errorFunc := func(err error, template string, args ...interface{}) { - msg := fmt.Sprintf("error: %v. ", err) - msg += fmt.Sprintf(template, args...) - fmt.Println(msg) - } testCases := []struct { serverAddrs []string @@ -563,7 +548,6 @@ func TestHTTPListenerAcceptTLSError(t *testing.T) { time.Duration(0), nil, nil, - errorFunc, ) if err != nil { t.Fatalf("Test %d: error: expected = , got = %v", i+1, err) @@ -609,11 +593,6 @@ func TestHTTPListenerAcceptTLSError(t *testing.T) { func TestHTTPListenerAcceptError(t *testing.T) { tlsConfig := getTLSConfig(t) nonLoopBackIP := getNonLoopBackIP(t) - errorFunc := func(err error, template string, args ...interface{}) { - msg := fmt.Sprintf("error: %v. ", err) - msg += fmt.Sprintf(template, args...) - fmt.Println(msg) - } testCases := []struct { serverAddrs []string @@ -635,7 +614,6 @@ func TestHTTPListenerAcceptError(t *testing.T) { time.Duration(0), nil, nil, - errorFunc, ) if err != nil { t.Fatalf("Test %d: error: expected = , got = %v", i+1, err) @@ -761,7 +739,6 @@ func TestHTTPListenerAcceptParallel(t *testing.T) { time.Duration(0), nil, nil, - nil, ) if err != nil { t.Fatalf("Test %d: error: expected = , got = %v", i+1, err) diff --git a/cmd/http/server.go b/cmd/http/server.go index 6db3fbc01..49d1640e8 100644 --- a/cmd/http/server.go +++ b/cmd/http/server.go @@ -50,16 +50,15 @@ const ( // Server - extended http.Server supports multiple addresses to serve and enhanced connection handling. type Server struct { http.Server - Addrs []string // addresses on which the server listens for new connection. - ShutdownTimeout time.Duration // timeout used for graceful server shutdown. - TCPKeepAliveTimeout time.Duration // timeout used for underneath TCP connection. - UpdateBytesReadFunc func(int) // function to be called to update bytes read in bufConn. - UpdateBytesWrittenFunc func(int) // function to be called to update bytes written in bufConn. - ErrorLogFunc func(error, string, ...interface{}) // function to be called on errors. - listenerMutex *sync.Mutex // to guard 'listener' field. - listener *httpListener // HTTP listener for all 'Addrs' field. - inShutdown uint32 // indicates whether the server is in shutdown or not - requestCount int32 // counter holds no. of request in process. + Addrs []string // addresses on which the server listens for new connection. + ShutdownTimeout time.Duration // timeout used for graceful server shutdown. + TCPKeepAliveTimeout time.Duration // timeout used for underneath TCP connection. + UpdateBytesReadFunc func(int) // function to be called to update bytes read in bufConn. + UpdateBytesWrittenFunc func(int) // function to be called to update bytes written in bufConn. + listenerMutex *sync.Mutex // to guard 'listener' field. + listener *httpListener // HTTP listener for all 'Addrs' field. + inShutdown uint32 // indicates whether the server is in shutdown or not + requestCount int32 // counter holds no. of request in process. } // Start - start HTTP server @@ -77,7 +76,6 @@ func (srv *Server) Start() (err error) { tcpKeepAliveTimeout := srv.TCPKeepAliveTimeout updateBytesReadFunc := srv.UpdateBytesReadFunc updateBytesWrittenFunc := srv.UpdateBytesWrittenFunc - errorLogFunc := srv.ErrorLogFunc // if srv.ErrorLogFunc holds non-synced state -> possible data race // Create new HTTP listener. var listener *httpListener @@ -89,7 +87,6 @@ func (srv *Server) Start() (err error) { writeTimeout, updateBytesReadFunc, updateBytesWrittenFunc, - errorLogFunc, ) if err != nil { return err diff --git a/cmd/jwt.go b/cmd/jwt.go index 19eead653..06feb0190 100644 --- a/cmd/jwt.go +++ b/cmd/jwt.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "errors" "fmt" "net/http" @@ -24,6 +25,7 @@ import ( jwtgo "github.com/dgrijalva/jwt-go" jwtreq "github.com/dgrijalva/jwt-go/request" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" ) @@ -97,11 +99,11 @@ func isAuthTokenValid(tokenString string) bool { var claims jwtgo.StandardClaims jwtToken, err := jwtgo.ParseWithClaims(tokenString, &claims, keyFuncCallback) if err != nil { - errorIf(err, "Unable to parse JWT token string") + logger.LogIf(context.Background(), err) return false } if err = claims.Valid(); err != nil { - errorIf(err, "Invalid claims in JWT token string") + logger.LogIf(context.Background(), err) return false } return jwtToken.Valid && claims.Subject == globalServerConfig.GetCredential().AccessKey diff --git a/cmd/lock-instrument.go b/cmd/lock-instrument.go index e21563b86..eb1064baf 100644 --- a/cmd/lock-instrument.go +++ b/cmd/lock-instrument.go @@ -17,10 +17,11 @@ package cmd import ( + "context" "fmt" "time" - "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" ) type statusType string @@ -112,29 +113,34 @@ func (n *nsLockMap) initLockInfoForVolumePath(param nsParam) { // Change the state of the lock from Blocked to Running. func (n *nsLockMap) statusBlockedToRunning(param nsParam, lockSource, opsID string, readLock bool) error { // This function is called outside nsLockMap.mutex.Lock(), so must be held explicitly. + ctx := context.Background() n.lockMapMutex.Lock() defer n.lockMapMutex.Unlock() // Check whether the lock info entry for pair already exists. _, ok := n.debugLockMap[param] if !ok { - return errors.Trace(LockInfoVolPathMissing{param.volume, param.path}) + logger.LogIf(ctx, LockInfoVolPathMissing{param.volume, param.path}) + return LockInfoVolPathMissing{param.volume, param.path} } // Check whether lock info entry for the given `opsID` exists. lockInfo, ok := n.debugLockMap[param].lockInfo[opsID] if !ok { - return errors.Trace(LockInfoOpsIDNotFound{param.volume, param.path, opsID}) + logger.LogIf(ctx, LockInfoOpsIDNotFound{param.volume, param.path, opsID}) + return LockInfoOpsIDNotFound{param.volume, param.path, opsID} } // Check whether lockSource is same. if lockInfo.lockSource != lockSource { - return errors.Trace(LockInfoOriginMismatch{param.volume, param.path, opsID, lockSource}) + logger.LogIf(ctx, LockInfoOriginMismatch{param.volume, param.path, opsID, lockSource}) + return LockInfoOriginMismatch{param.volume, param.path, opsID, lockSource} } // Status of the lock should be set to "Blocked". if lockInfo.status != blockedStatus { - return errors.Trace(LockInfoStateNotBlocked{param.volume, param.path, opsID}) + logger.LogIf(ctx, LockInfoStateNotBlocked{param.volume, param.path, opsID}) + return LockInfoStateNotBlocked{param.volume, param.path, opsID} } // Change lock status to running and update the time. n.debugLockMap[param].lockInfo[opsID] = newDebugLockInfo(lockSource, runningStatus, readLock) @@ -182,24 +188,29 @@ func (n *nsLockMap) statusNoneToBlocked(param nsParam, lockSource, opsID string, // Change the state of the lock from Blocked to none. func (n *nsLockMap) statusBlockedToNone(param nsParam, lockSource, opsID string, readLock bool) error { _, ok := n.debugLockMap[param] + ctx := context.Background() if !ok { - return errors.Trace(LockInfoVolPathMissing{param.volume, param.path}) + logger.LogIf(ctx, LockInfoVolPathMissing{param.volume, param.path}) + return LockInfoVolPathMissing{param.volume, param.path} } // Check whether lock info entry for the given `opsID` exists. lockInfo, ok := n.debugLockMap[param].lockInfo[opsID] if !ok { - return errors.Trace(LockInfoOpsIDNotFound{param.volume, param.path, opsID}) + logger.LogIf(ctx, LockInfoOpsIDNotFound{param.volume, param.path, opsID}) + return LockInfoOpsIDNotFound{param.volume, param.path, opsID} } // Check whether lockSource is same. if lockInfo.lockSource != lockSource { - return errors.Trace(LockInfoOriginMismatch{param.volume, param.path, opsID, lockSource}) + logger.LogIf(ctx, LockInfoOriginMismatch{param.volume, param.path, opsID, lockSource}) + return LockInfoOriginMismatch{param.volume, param.path, opsID, lockSource} } // Status of the lock should be set to "Blocked". if lockInfo.status != blockedStatus { - return errors.Trace(LockInfoStateNotBlocked{param.volume, param.path, opsID}) + logger.LogIf(ctx, LockInfoStateNotBlocked{param.volume, param.path, opsID}) + return LockInfoStateNotBlocked{param.volume, param.path, opsID} } // Update global lock stats. @@ -214,7 +225,8 @@ func (n *nsLockMap) statusBlockedToNone(param nsParam, lockSource, opsID string, func (n *nsLockMap) deleteLockInfoEntryForVolumePath(param nsParam) error { // delete the lock info for the given operation. if _, found := n.debugLockMap[param]; !found { - return errors.Trace(LockInfoVolPathMissing{param.volume, param.path}) + logger.LogIf(context.Background(), LockInfoVolPathMissing{param.volume, param.path}) + return LockInfoVolPathMissing{param.volume, param.path} } // The following stats update is relevant only in case of a @@ -235,17 +247,20 @@ func (n *nsLockMap) deleteLockInfoEntryForVolumePath(param nsParam) error { // Called when the nsLk ref count for the given (volume, path) is // not 0. func (n *nsLockMap) deleteLockInfoEntryForOps(param nsParam, opsID string) error { + ctx := context.Background() // delete the lock info for the given operation. infoMap, found := n.debugLockMap[param] if !found { - return errors.Trace(LockInfoVolPathMissing{param.volume, param.path}) + logger.LogIf(ctx, LockInfoVolPathMissing{param.volume, param.path}) + return LockInfoVolPathMissing{param.volume, param.path} } // The operation finished holding the lock on the resource, remove // the entry for the given operation with the operation ID. opsIDLock, foundInfo := infoMap.lockInfo[opsID] if !foundInfo { // Unlock request with invalid operation ID not accepted. - return errors.Trace(LockInfoOpsIDNotFound{param.volume, param.path, opsID}) + logger.LogIf(ctx, LockInfoOpsIDNotFound{param.volume, param.path, opsID}) + return LockInfoOpsIDNotFound{param.volume, param.path, opsID} } // Update global and (volume, path) lock status. granted := opsIDLock.status == runningStatus diff --git a/cmd/lock-rpc-server-common.go b/cmd/lock-rpc-server-common.go index 124b46496..51c2a17c4 100644 --- a/cmd/lock-rpc-server-common.go +++ b/cmd/lock-rpc-server-common.go @@ -17,8 +17,11 @@ package cmd import ( + "context" "errors" "time" + + "github.com/minio/minio/cmd/logger" ) // Similar to removeEntry but only removes an entry only if the lock entry exists in map. @@ -29,7 +32,10 @@ func (l *localLocker) removeEntryIfExists(nlrip nameLockRequesterInfoPair) { // Remove failed, in case it is a: if nlrip.lri.writer { // Writer: this should never happen as the whole (mapped) entry should have been deleted - errorIf(errors.New(""), "Lock maintenance failed to remove entry for write lock (should never happen) %#v %#v %#v", nlrip.name, nlrip.lri.uid, lri) + reqInfo := (&logger.ReqInfo{}).AppendTags("name", nlrip.name) + reqInfo.AppendTags("uid", nlrip.lri.uid) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, errors.New("Lock maintenance failed to remove entry for write lock (should never happen)")) } // Reader: this can happen if multiple read locks were active and // the one we are looking for has been released concurrently (so it is fine). } // Removal went okay, all is fine. diff --git a/cmd/lock-rpc-server.go b/cmd/lock-rpc-server.go index e3b053d56..58fab9e5a 100644 --- a/cmd/lock-rpc-server.go +++ b/cmd/lock-rpc-server.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "fmt" "math/rand" "sync" @@ -24,7 +25,7 @@ import ( router "github.com/gorilla/mux" "github.com/minio/dsync" - "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" ) const ( @@ -98,7 +99,8 @@ func registerDistNSLockRouter(mux *router.Router, endpoints EndpointList) error func registerStorageLockers(mux *router.Router, lkSrv *lockServer) error { lockRPCServer := newRPCServer() if err := lockRPCServer.RegisterName(lockServiceName, lkSrv); err != nil { - return errors.Trace(err) + logger.LogIf(context.Background(), err) + return err } lockRouter := mux.PathPrefix(minioReservedBucketPath).Subrouter() lockRouter.Path(lockServicePath).Handler(lockRPCServer) diff --git a/cmd/logger/log.go b/cmd/logger/log.go index fd9853025..863309fe8 100644 --- a/cmd/logger/log.go +++ b/cmd/logger/log.go @@ -16,9 +16,13 @@ package logger -import "context" +import ( + "context" + "fmt" + "sync" +) -// Key used for Get/SetContext +// Key used for Get/SetReqInfo type contextKeyType string const contextLogKey = contextKeyType("miniolog") @@ -37,27 +41,59 @@ type ReqInfo struct { API string // API name - GetObject PutObject NewMultipartUpload etc. BucketName string // Bucket name ObjectName string // Object name - Tags []KeyVal // Any additional info not accommodated by above fields + tags []KeyVal // Any additional info not accommodated by above fields + sync.RWMutex } -// AppendTags - appends key/val to ReqInfo.Tags -func (r *ReqInfo) AppendTags(key string, val string) { +// NewReqInfo : +func NewReqInfo(remoteHost, userAgent, requestID, api, bucket, object string) *ReqInfo { + req := ReqInfo{} + req.RemoteHost = remoteHost + req.UserAgent = userAgent + req.API = api + req.RequestID = requestID + req.BucketName = bucket + req.ObjectName = object + return &req +} + +// AppendTags - appends key/val to ReqInfo.tags +func (r *ReqInfo) AppendTags(key string, val string) *ReqInfo { if r == nil { - return + return nil } - r.Tags = append(r.Tags, KeyVal{key, val}) + r.Lock() + defer r.Unlock() + r.tags = append(r.tags, KeyVal{key, val}) + return r } -// SetContext sets ReqInfo in the context. -func SetContext(ctx context.Context, req *ReqInfo) context.Context { +// GetTags - returns the user defined tags +func (r *ReqInfo) GetTags() []KeyVal { + if r == nil { + return nil + } + r.RLock() + defer r.RUnlock() + return append([]KeyVal(nil), r.tags...) +} + +// SetReqInfo sets ReqInfo in the context. +func SetReqInfo(ctx context.Context, req *ReqInfo) context.Context { + if ctx == nil { + LogIf(context.Background(), fmt.Errorf("context is nil")) + return nil + } return context.WithValue(ctx, contextLogKey, req) } -// GetContext returns ReqInfo if set. -func GetContext(ctx context.Context) *ReqInfo { - r, ok := ctx.Value(contextLogKey).(*ReqInfo) - if ok { - return r +// GetReqInfo returns ReqInfo if set. +func GetReqInfo(ctx context.Context) *ReqInfo { + if ctx != nil { + r, ok := ctx.Value(contextLogKey).(*ReqInfo) + if ok { + return r + } } return nil } diff --git a/cmd/logger.go b/cmd/logger/logger.go similarity index 50% rename from cmd/logger.go rename to cmd/logger/logger.go index 390124149..f0e026b60 100644 --- a/cmd/logger.go +++ b/cmd/logger/logger.go @@ -1,5 +1,5 @@ /* - * Minio Cloud Storage, (C) 2015, 2016, 2017 Minio, Inc. + * Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ * limitations under the License. */ -package cmd +package logger import ( + "context" "encoding/json" "fmt" "go/build" @@ -26,11 +27,17 @@ import ( "strings" "time" + "github.com/fatih/color" "github.com/minio/mc/pkg/console" - "github.com/minio/minio/pkg/errors" ) -var log = NewLogger() +// global colors. +var ( + colorBold = color.New(color.Bold).SprintFunc() + colorYellow = color.New(color.FgYellow).SprintfFunc() + colorRed = color.New(color.FgRed).SprintfFunc() +) + var trimStrings []string // Level type @@ -42,6 +49,15 @@ const ( Fatal ) +const loggerTimeFormat string = "15:04:05 MST 01/02/2006" + +var matchingFuncNames = [...]string{ + "http.HandlerFunc.ServeHTTP", + "cmd.serverMain", + "cmd.StartGateway", + // add more here .. +} + func (level Level) String() string { var lvlStr string switch level { @@ -53,61 +69,78 @@ func (level Level) String() string { return lvlStr } +type traceEntry struct { + Message string `json:"message"` + Source []string `json:"source"` + Variables map[string]string `json:"variables,omitempty"` +} +type args struct { + Bucket string `json:"bucket,omitempty"` + Object string `json:"object,omitempty"` +} + +type api struct { + Name string `json:"name,omitempty"` + Args args `json:"args,omitempty"` +} + type logEntry struct { - Level string `json:"level"` - Message string `json:"message"` - Time string `json:"time"` - Cause string `json:"cause"` - Trace []string `json:"trace"` + Level string `json:"level"` + Time string `json:"time"` + API api `json:"api,omitempty"` + RemoteHost string `json:"remotehost,omitempty"` + RequestID string `json:"requestID,omitempty"` + UserAgent string `json:"userAgent,omitempty"` + Cause string `json:"cause,omitempty"` + Trace traceEntry `json:"error"` } -// Logger - for console messages -type Logger struct { - quiet bool - json bool -} - -// NewLogger - to create a new Logger object -func NewLogger() *Logger { - return &Logger{} -} +// quiet: Hide startup messages if enabled +// jsonFlag: Display in JSON format, if enabled +var ( + quiet, jsonFlag bool +) // EnableQuiet - turns quiet option on. -func (log *Logger) EnableQuiet() { - log.quiet = true +func EnableQuiet() { + quiet = true } // EnableJSON - outputs logs in json format. -func (log *Logger) EnableJSON() { - log.json = true - log.quiet = true +func EnableJSON() { + jsonFlag = true + quiet = true } // Println - wrapper to console.Println() with quiet flag. -func (log *Logger) Println(args ...interface{}) { - if !log.quiet { +func Println(args ...interface{}) { + if !quiet { console.Println(args...) } } // Printf - wrapper to console.Printf() with quiet flag. -func (log *Logger) Printf(format string, args ...interface{}) { - if !log.quiet { +func Printf(format string, args ...interface{}) { + if !quiet { console.Printf(format, args...) } } -func init() { +// Init sets the trimStrings to possible GOPATHs +// and GOROOT directories. Also append github.com/minio/minio +// This is done to clean up the filename, when stack trace is +// displayed when an error happens. +func Init(goPath string) { var goPathList []string var defaultgoPathList []string // Add all possible GOPATH paths into trimStrings // Split GOPATH depending on the OS type if runtime.GOOS == "windows" { - goPathList = strings.Split(GOPATH, ";") + goPathList = strings.Split(goPath, ";") defaultgoPathList = strings.Split(build.Default.GOPATH, ";") } else { // All other types of OSs - goPathList = strings.Split(GOPATH, ":") + goPathList = strings.Split(goPath, ":") defaultgoPathList = strings.Split(build.Default.GOPATH, ":") } @@ -155,6 +188,13 @@ func getTrace(traceLevel int) []string { // Form and append a line of stack trace into a // collection, 'trace', to build full stack trace trace = append(trace, fmt.Sprintf("%v:%v:%v()", file, lineNumber, funcName)) + + // Ignore trace logs beyond the following conditions + for _, name := range matchingFuncNames { + if funcName == name { + return trace + } + } } traceLevel++ // Read stack trace information from PC @@ -165,21 +205,7 @@ func getTrace(traceLevel int) []string { func logIf(level Level, err error, msg string, data ...interface{}) { - - isErrIgnored := func(err error) (ok bool) { - err = errors.Cause(err) - switch err.(type) { - case BucketNotFound, BucketNotEmpty, BucketExists: - ok = true - case ObjectNotFound, ObjectExistsAsDirectory: - ok = true - case BucketPolicyNotFound, InvalidUploadID: - ok = true - } - return ok - } - - if err == nil || isErrIgnored(err) { + if err == nil { return } // Get the cause for the Error @@ -187,17 +213,16 @@ func logIf(level Level, err error, msg string, // Get full stack trace trace := getTrace(3) // Get time - timeOfError := UTCNow().Format(time.RFC3339Nano) + timeOfError := time.Now().UTC().Format(time.RFC3339Nano) // Output the formatted log message at console var output string message := fmt.Sprintf(msg, data...) - if log.json { + if jsonFlag { logJSON, err := json.Marshal(&logEntry{ - Level: level.String(), - Message: message, - Time: timeOfError, - Cause: cause, - Trace: trace, + Level: level.String(), + Time: timeOfError, + Cause: cause, + Trace: traceEntry{Source: trace, Message: message}, }) if err != nil { panic("json marshal of logEntry failed: " + err.Error()) @@ -224,10 +249,102 @@ func logIf(level Level, err error, msg string, } } -func errorIf(err error, msg string, data ...interface{}) { - logIf(Error, err, msg, data...) -} - -func fatalIf(err error, msg string, data ...interface{}) { +// FatalIf : +func FatalIf(err error, msg string, data ...interface{}) { logIf(Fatal, err, msg, data...) } + +// LogIf : +func LogIf(ctx context.Context, err error) { + if err == nil { + return + } + + req := GetReqInfo(ctx) + + if req == nil { + req = &ReqInfo{API: "SYSTEM"} + } + + API := "SYSTEM" + if req.API != "" { + API = req.API + } + + tags := make(map[string]string) + for _, entry := range req.GetTags() { + tags[entry.Key] = entry.Val + } + + // Get the cause for the Error + message := err.Error() + // Get full stack trace + trace := getTrace(2) + // Output the formatted log message at console + var output string + if jsonFlag { + logJSON, err := json.Marshal(&logEntry{ + Level: Error.String(), + RemoteHost: req.RemoteHost, + RequestID: req.RequestID, + UserAgent: req.UserAgent, + Time: time.Now().UTC().Format(time.RFC3339Nano), + API: api{Name: API, Args: args{Bucket: req.BucketName, Object: req.ObjectName}}, + Trace: traceEntry{Message: message, Source: trace, Variables: tags}, + }) + if err != nil { + panic(err) + } + output = string(logJSON) + } else { + // Add a sequence number and formatting for each stack trace + // No formatting is required for the first entry + for i, element := range trace { + trace[i] = fmt.Sprintf("%8v: %s", i+1, element) + } + + tagString := "" + for key, value := range tags { + if value != "" { + if tagString != "" { + tagString += ", " + } + tagString += key + "=" + value + } + } + + apiString := "API: " + API + "(" + if req.BucketName != "" { + apiString = apiString + "bucket=" + req.BucketName + } + if req.ObjectName != "" { + apiString = apiString + ", object=" + req.ObjectName + } + apiString += ")" + timeString := "Time: " + time.Now().Format(loggerTimeFormat) + + var requestID string + if req.RequestID != "" { + requestID = "\nRequestID: " + req.RequestID + } + + var remoteHost string + if req.RemoteHost != "" { + remoteHost = "\nRemoteHost: " + req.RemoteHost + } + + var userAgent string + if req.UserAgent != "" { + userAgent = "\nUserAgent: " + req.UserAgent + } + + if len(tags) > 0 { + tagString = "\n " + tagString + } + + output = fmt.Sprintf("\n%s\n%s%s%s%s\nError: %s%s\n%s", + apiString, timeString, requestID, remoteHost, userAgent, + colorRed(colorBold(message)), tagString, strings.Join(trace, "\n")) + } + fmt.Println(output) +} diff --git a/cmd/namespace-lock.go b/cmd/namespace-lock.go index 2976b1fe8..99bc9a5b4 100644 --- a/cmd/namespace-lock.go +++ b/cmd/namespace-lock.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "errors" pathutil "path" "runtime" @@ -29,6 +30,7 @@ import ( "github.com/minio/dsync" "github.com/minio/lsync" "github.com/minio/minio-go/pkg/set" + "github.com/minio/minio/cmd/logger" ) // Global name space lock. @@ -170,9 +172,7 @@ func (n *nsLockMap) lock(volume, path string, lockSource, opsID string, readLock // pair of and till the lock // unblocks. The lock for accessing `globalNSMutex` is held inside // the function itself. - if err := n.statusNoneToBlocked(param, lockSource, opsID, readLock); err != nil { - errorIf(err, fmt.Sprintf("Failed to set lock state to blocked (param = %v; opsID = %s)", param, opsID)) - } + n.statusNoneToBlocked(param, lockSource, opsID, readLock) // Unlock map before Locking NS which might block. n.lockMapMutex.Unlock() @@ -188,26 +188,19 @@ func (n *nsLockMap) lock(volume, path string, lockSource, opsID string, readLock n.lockMapMutex.Lock() defer n.lockMapMutex.Unlock() // Changing the status of the operation from blocked to none - if err := n.statusBlockedToNone(param, lockSource, opsID, readLock); err != nil { - errorIf(err, fmt.Sprintf("Failed to clear the lock state (param = %v; opsID = %s)", param, opsID)) - } + n.statusBlockedToNone(param, lockSource, opsID, readLock) nsLk.ref-- // Decrement ref count since we failed to get the lock // delete the lock state entry for given operation ID. - err := n.deleteLockInfoEntryForOps(param, opsID) - if err != nil { - errorIf(err, fmt.Sprintf("Failed to delete lock info entry (param = %v; opsID = %s)", param, opsID)) - } + n.deleteLockInfoEntryForOps(param, opsID) + if nsLk.ref == 0 { // Remove from the map if there are no more references. delete(n.lockMap, param) // delete the lock state entry for given // pair. - err := n.deleteLockInfoEntryForVolumePath(param) - if err != nil { - errorIf(err, fmt.Sprintf("Failed to delete lock info entry (param = %v)", param)) - } + n.deleteLockInfoEntryForVolumePath(param) } return } @@ -215,9 +208,7 @@ func (n *nsLockMap) lock(volume, path string, lockSource, opsID string, readLock // Changing the status of the operation from blocked to // running. change the state of the lock to be running (from // blocked) for the given pair of and . - if err := n.statusBlockedToRunning(param, lockSource, opsID, readLock); err != nil { - errorIf(err, "Failed to set the lock state to running") - } + n.statusBlockedToRunning(param, lockSource, opsID, readLock) return } @@ -236,17 +227,13 @@ func (n *nsLockMap) unlock(volume, path, opsID string, readLock bool) { nsLk.Unlock() } if nsLk.ref == 0 { - errorIf(errors.New("Namespace reference count cannot be 0"), - "Invalid reference count detected") + logger.LogIf(context.Background(), errors.New("Namespace reference count cannot be 0")) } if nsLk.ref != 0 { nsLk.ref-- // delete the lock state entry for given operation ID. - err := n.deleteLockInfoEntryForOps(param, opsID) - if err != nil { - errorIf(err, "Failed to delete lock info entry") - } + n.deleteLockInfoEntryForOps(param, opsID) } if nsLk.ref == 0 { // Remove from the map if there are no more references. @@ -254,10 +241,7 @@ func (n *nsLockMap) unlock(volume, path, opsID string, readLock bool) { // delete the lock state entry for given // pair. - err := n.deleteLockInfoEntryForVolumePath(param) - if err != nil { - errorIf(err, "Failed to delete lock info entry") - } + n.deleteLockInfoEntryForVolumePath(param) } } } diff --git a/cmd/net.go b/cmd/net.go index fbf7391f3..0bc7b4d8e 100644 --- a/cmd/net.go +++ b/cmd/net.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "errors" "fmt" "net" @@ -30,6 +31,7 @@ import ( humanize "github.com/dustin/go-humanize" "github.com/minio/minio-go/pkg/set" + "github.com/minio/minio/cmd/logger" ) // IPv4 addresses of local host. @@ -38,7 +40,7 @@ var localIP4 = mustGetLocalIP4() // mustSplitHostPort is a wrapper to net.SplitHostPort() where error is assumed to be a fatal. func mustSplitHostPort(hostPort string) (host, port string) { host, port, err := net.SplitHostPort(hostPort) - fatalIf(err, "Unable to split host port %s", hostPort) + logger.FatalIf(err, "Unable to split host port %s", hostPort) return host, port } @@ -46,7 +48,7 @@ func mustSplitHostPort(hostPort string) (host, port string) { func mustGetLocalIP4() (ipList set.StringSet) { ipList = set.NewStringSet() addrs, err := net.InterfaceAddrs() - fatalIf(err, "Unable to get IP addresses of this host.") + logger.FatalIf(err, "Unable to get IP addresses of this host.") for _, addr := range addrs { var ip net.IP @@ -98,8 +100,10 @@ func getHostIP4(host string) (ipList set.StringSet, err error) { if timeElapsed > time.Second { // log the message to console about the host not being // resolveable. - errorIf(err, "Unable to resolve host %s (%s)", host, - humanize.RelTime(startTime, startTime.Add(timeElapsed), "elapsed", "")) + reqInfo := (&logger.ReqInfo{}).AppendTags("host", host) + reqInfo.AppendTags("elapsedTime", humanize.RelTime(startTime, startTime.Add(timeElapsed), "elapsed", "")) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) } } } diff --git a/cmd/notification.go b/cmd/notification.go index d99c7fe3e..0fdf216c0 100644 --- a/cmd/notification.go +++ b/cmd/notification.go @@ -26,6 +26,7 @@ import ( "path" "sync" + "github.com/minio/minio/cmd/logger" xerrors "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/event" "github.com/minio/minio/pkg/hash" @@ -162,7 +163,7 @@ func (sys *NotificationSys) RemoteTargetExist(bucketName string, targetID event. } // initListeners - initializes PeerRPC clients available in listener.json. -func (sys *NotificationSys) initListeners(objAPI ObjectLayer, bucketName string) error { +func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLayer, bucketName string) error { // listener.json is available/applicable only in DistXL mode. if !globalIsDistXL { return nil @@ -181,7 +182,7 @@ func (sys *NotificationSys) initListeners(objAPI ObjectLayer, bucketName string) } defer objLock.Unlock() - reader, err := readConfig(objAPI, configFile) + reader, err := readConfig(ctx, objAPI, configFile) if err != nil && !xerrors.IsErrIgnored(err, errDiskNotFound, errNoSuchNotifications) { return err } @@ -189,8 +190,8 @@ func (sys *NotificationSys) initListeners(objAPI ObjectLayer, bucketName string) listenerList := []ListenBucketNotificationArgs{} if reader != nil { if err = json.NewDecoder(reader).Decode(&listenerList); err != nil { - errorIf(err, "Unable to parse listener.json.") - return xerrors.Trace(err) + logger.LogIf(ctx, err) + return err } } @@ -203,7 +204,8 @@ func (sys *NotificationSys) initListeners(objAPI ObjectLayer, bucketName string) for _, args := range listenerList { var found bool if found, err = isLocalHost(args.Addr.Name); err != nil { - errorIf(err, "unable to check address %v is local host", args.Addr) + logger.GetReqInfo(ctx).AppendTags("host", args.Addr.Name) + logger.LogIf(ctx, err) return err } if found { @@ -218,6 +220,8 @@ func (sys *NotificationSys) initListeners(objAPI ObjectLayer, bucketName string) var exist bool if exist, err = rpcClient.RemoteTargetExist(bucketName, args.TargetID); err != nil { + logger.GetReqInfo(ctx).AppendTags("targetID", args.TargetID.Name) + logger.LogIf(ctx, err) return err } if !exist { @@ -228,6 +232,8 @@ func (sys *NotificationSys) initListeners(objAPI ObjectLayer, bucketName string) target := NewPeerRPCClientTarget(bucketName, args.TargetID, rpcClient) rulesMap := event.NewRulesMap(args.EventNames, args.Pattern, target.ID()) if err = sys.AddRemoteTarget(bucketName, target, rulesMap); err != nil { + logger.GetReqInfo(ctx).AppendTags("targetID", target.id.Name) + logger.LogIf(ctx, err) return err } activeListenerList = append(activeListenerList, args) @@ -235,6 +241,7 @@ func (sys *NotificationSys) initListeners(objAPI ObjectLayer, bucketName string) data, err := json.Marshal(activeListenerList) if err != nil { + logger.LogIf(ctx, err) return err } @@ -253,18 +260,17 @@ func (sys *NotificationSys) Init(objAPI ObjectLayer) error { } for _, bucket := range buckets { - config, err := readNotificationConfig(objAPI, bucket.Name) + ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{BucketName: bucket.Name}) + config, err := readNotificationConfig(ctx, objAPI, bucket.Name) if err != nil { if !xerrors.IsErrIgnored(err, errDiskNotFound, errNoSuchNotifications) { - errorIf(err, "Unable to load notification configuration of bucket %v", bucket.Name) return err } } else { sys.AddRulesMap(bucket.Name, config.ToRulesMap()) } - if err = sys.initListeners(objAPI, bucket.Name); err != nil { - errorIf(err, "Unable to initialize HTTP listener for bucket %v", bucket.Name) + if err = sys.initListeners(ctx, objAPI, bucket.Name); err != nil { return err } } @@ -325,7 +331,9 @@ func (sys *NotificationSys) RemoveAllRemoteTargets() { // RemoveRemoteTarget - closes and removes target by target ID. func (sys *NotificationSys) RemoveRemoteTarget(bucketName string, targetID event.TargetID) { for id, err := range sys.targetList.Remove(targetID) { - errorIf(err, "unable to close target ID %v", id) + reqInfo := (&logger.ReqInfo{}).AppendTags("targetID", id.Name) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) } sys.Lock() @@ -457,8 +465,11 @@ func sendEvent(args eventArgs) { } for targetID, err := range globalNotificationSys.Send(args) { - errorIf(err, "unable to send event %v of bucket: %v, object: %v to target %v", - args.EventName, args.BucketName, args.Object.Name, targetID) + reqInfo := &logger.ReqInfo{BucketName: args.BucketName, ObjectName: args.Object.Name} + reqInfo.AppendTags("EventName", args.EventName.String()) + reqInfo.AppendTags("targetID", targetID.Name) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) } } @@ -472,36 +483,39 @@ func saveConfig(objAPI ObjectLayer, configFile string, data []byte) error { return err } -func readConfig(objAPI ObjectLayer, configFile string) (*bytes.Buffer, error) { +func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) (*bytes.Buffer, error) { var buffer bytes.Buffer // Read entire content by setting size to -1 - err := objAPI.GetObject(context.Background(), minioMetaBucket, configFile, 0, -1, &buffer, "") + err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "") if err != nil { // Ignore if err is ObjectNotFound or IncompleteBody when bucket is not configured with notification if isErrObjectNotFound(err) || isErrIncompleteBody(err) { - return nil, xerrors.Trace(errNoSuchNotifications) + return nil, errNoSuchNotifications } - errorIf(err, "Unable to read file %v", configFile) + logger.GetReqInfo(ctx).AppendTags("configFile", configFile) + logger.LogIf(ctx, err) return nil, err } // Return NoSuchNotifications on empty content. if buffer.Len() == 0 { - return nil, xerrors.Trace(errNoSuchNotifications) + return nil, errNoSuchNotifications } return &buffer, nil } -func readNotificationConfig(objAPI ObjectLayer, bucketName string) (*event.Config, error) { +func readNotificationConfig(ctx context.Context, objAPI ObjectLayer, bucketName string) (*event.Config, error) { // Construct path to notification.xml for the given bucket. configFile := path.Join(bucketConfigPrefix, bucketName, bucketNotificationConfig) - reader, err := readConfig(objAPI, configFile) + reader, err := readConfig(ctx, objAPI, configFile) if err != nil { return nil, err } - return event.ParseConfig(reader, globalServerConfig.GetRegion(), globalNotificationSys.targetList) + config, err := event.ParseConfig(reader, globalServerConfig.GetRegion(), globalNotificationSys.targetList) + logger.LogIf(ctx, err) + return config, err } func saveNotificationConfig(objAPI ObjectLayer, bucketName string, config *event.Config) error { @@ -521,6 +535,8 @@ func SaveListener(objAPI ObjectLayer, bucketName string, eventNames []event.Name return nil } + ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{BucketName: bucketName}) + // Construct path to listener.json for the given bucket. configFile := path.Join(bucketConfigPrefix, bucketName, bucketListenerConfig) transactionConfigFile := configFile + ".transaction" @@ -534,7 +550,7 @@ func SaveListener(objAPI ObjectLayer, bucketName string, eventNames []event.Name } defer objLock.Unlock() - reader, err := readConfig(objAPI, configFile) + reader, err := readConfig(ctx, objAPI, configFile) if err != nil && !xerrors.IsErrIgnored(err, errDiskNotFound, errNoSuchNotifications) { return err } @@ -542,8 +558,8 @@ func SaveListener(objAPI ObjectLayer, bucketName string, eventNames []event.Name listenerList := []ListenBucketNotificationArgs{} if reader != nil { if err = json.NewDecoder(reader).Decode(&listenerList); err != nil { - errorIf(err, "Unable to parse listener.json.") - return xerrors.Trace(err) + logger.LogIf(ctx, err) + return err } } @@ -556,6 +572,7 @@ func SaveListener(objAPI ObjectLayer, bucketName string, eventNames []event.Name data, err := json.Marshal(listenerList) if err != nil { + logger.LogIf(ctx, err) return err } @@ -569,6 +586,8 @@ func RemoveListener(objAPI ObjectLayer, bucketName string, targetID event.Target return nil } + ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{BucketName: bucketName}) + // Construct path to listener.json for the given bucket. configFile := path.Join(bucketConfigPrefix, bucketName, bucketListenerConfig) transactionConfigFile := configFile + ".transaction" @@ -582,7 +601,7 @@ func RemoveListener(objAPI ObjectLayer, bucketName string, targetID event.Target } defer objLock.Unlock() - reader, err := readConfig(objAPI, configFile) + reader, err := readConfig(ctx, objAPI, configFile) if err != nil && !xerrors.IsErrIgnored(err, errDiskNotFound, errNoSuchNotifications) { return err } @@ -590,8 +609,8 @@ func RemoveListener(objAPI ObjectLayer, bucketName string, targetID event.Target listenerList := []ListenBucketNotificationArgs{} if reader != nil { if err = json.NewDecoder(reader).Decode(&listenerList); err != nil { - errorIf(err, "Unable to parse listener.json.") - return xerrors.Trace(err) + logger.LogIf(ctx, err) + return err } } @@ -612,6 +631,7 @@ func RemoveListener(objAPI ObjectLayer, bucketName string, targetID event.Target data, err := json.Marshal(activeListenerList) if err != nil { + logger.LogIf(ctx, err) return err } diff --git a/cmd/object-api-common.go b/cmd/object-api-common.go index 3e0455570..f8e79e90c 100644 --- a/cmd/object-api-common.go +++ b/cmd/object-api-common.go @@ -22,7 +22,7 @@ import ( "sync" humanize "github.com/dustin/go-humanize" - "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" ) const ( @@ -83,15 +83,15 @@ func dirObjectInfo(bucket, object string, size int64, metadata map[string]string } } -func deleteBucketMetadata(bucket string, objAPI ObjectLayer) { +func deleteBucketMetadata(ctx context.Context, bucket string, objAPI ObjectLayer) { // Delete bucket access policy, if present - ignore any errors. - _ = removeBucketPolicy(bucket, objAPI) + removeBucketPolicy(ctx, bucket, objAPI) // Delete notification config, if present - ignore any errors. - _ = removeNotificationConfig(objAPI, bucket) + removeNotificationConfig(ctx, objAPI, bucket) // Delete listener config, if present - ignore any errors. - _ = removeListenerConfig(objAPI, bucket) + removeListenerConfig(ctx, objAPI, bucket) } // Depending on the disk type network or local, initialize storage API. @@ -104,13 +104,15 @@ func newStorageAPI(endpoint Endpoint) (storage StorageAPI, err error) { } // Cleanup a directory recursively. -func cleanupDir(storage StorageAPI, volume, dirPath string) error { +func cleanupDir(ctx context.Context, storage StorageAPI, volume, dirPath string) error { var delFunc func(string) error // Function to delete entries recursively. delFunc = func(entryPath string) error { if !hasSuffix(entryPath, slashSeparator) { // Delete the file entry. - return errors.Trace(storage.DeleteFile(volume, entryPath)) + err := storage.DeleteFile(volume, entryPath) + logger.LogIf(ctx, err) + return err } // If it's a directory, list and call delFunc() for each entry. @@ -119,12 +121,15 @@ func cleanupDir(storage StorageAPI, volume, dirPath string) error { if err == errFileNotFound { return nil } else if err != nil { // For any other errors fail. - return errors.Trace(err) + logger.LogIf(ctx, err) + return err } // else on success.. // Entry path is empty, just delete it. if len(entries) == 0 { - return errors.Trace(storage.DeleteFile(volume, path.Clean(entryPath))) + err = storage.DeleteFile(volume, path.Clean(entryPath)) + logger.LogIf(ctx, err) + return err } // Recurse and delete all other entries. @@ -140,21 +145,19 @@ func cleanupDir(storage StorageAPI, volume, dirPath string) error { } // Removes notification.xml for a given bucket, only used during DeleteBucket. -func removeNotificationConfig(objAPI ObjectLayer, bucket string) error { +func removeNotificationConfig(ctx context.Context, objAPI ObjectLayer, bucket string) error { // Verify bucket is valid. if !IsValidBucketName(bucket) { return BucketNameInvalid{Bucket: bucket} } ncPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig) - - return objAPI.DeleteObject(context.Background(), minioMetaBucket, ncPath) + return objAPI.DeleteObject(ctx, minioMetaBucket, ncPath) } // Remove listener configuration from storage layer. Used when a bucket is deleted. -func removeListenerConfig(objAPI ObjectLayer, bucket string) error { +func removeListenerConfig(ctx context.Context, objAPI ObjectLayer, bucket string) error { // make the path lcPath := path.Join(bucketConfigPrefix, bucket, bucketListenerConfig) - - return objAPI.DeleteObject(context.Background(), minioMetaBucket, lcPath) + return objAPI.DeleteObject(ctx, minioMetaBucket, lcPath) } diff --git a/cmd/object-api-input-checks.go b/cmd/object-api-input-checks.go index a193f9bdf..a78de4737 100644 --- a/cmd/object-api-input-checks.go +++ b/cmd/object-api-input-checks.go @@ -19,165 +19,200 @@ package cmd import ( "context" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/errors" "github.com/skyrings/skyring-common/tools/uuid" ) // Checks on GetObject arguments, bucket and object. -func checkGetObjArgs(bucket, object string) error { - return checkBucketAndObjectNames(bucket, object) +func checkGetObjArgs(ctx context.Context, bucket, object string) error { + return checkBucketAndObjectNames(ctx, bucket, object) } // Checks on DeleteObject arguments, bucket and object. -func checkDelObjArgs(bucket, object string) error { - return checkBucketAndObjectNames(bucket, object) +func checkDelObjArgs(ctx context.Context, bucket, object string) error { + return checkBucketAndObjectNames(ctx, bucket, object) } // Checks bucket and object name validity, returns nil if both are valid. -func checkBucketAndObjectNames(bucket, object string) error { +func checkBucketAndObjectNames(ctx context.Context, bucket, object string) error { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return errors.Trace(BucketNameInvalid{Bucket: bucket}) + logger.LogIf(ctx, BucketNameInvalid{Bucket: bucket}) + return BucketNameInvalid{Bucket: bucket} } // Verify if object is valid. if len(object) == 0 { - return errors.Trace(ObjectNameInvalid{Bucket: bucket, Object: object}) + logger.LogIf(ctx, ObjectNameInvalid{Bucket: bucket, Object: object}) + return ObjectNameInvalid{Bucket: bucket, Object: object} } if !IsValidObjectPrefix(object) { - return errors.Trace(ObjectNameInvalid{Bucket: bucket, Object: object}) + logger.LogIf(ctx, ObjectNameInvalid{Bucket: bucket, Object: object}) + return ObjectNameInvalid{Bucket: bucket, Object: object} } return nil } // Checks for all ListObjects arguments validity. -func checkListObjsArgs(bucket, prefix, marker, delimiter string, obj ObjectLayer) error { +func checkListObjsArgs(ctx context.Context, bucket, prefix, marker, delimiter string, obj ObjectLayer) error { // Verify if bucket exists before validating object name. // This is done on purpose since the order of errors is // important here bucket does not exist error should // happen before we return an error for invalid object name. // FIXME: should be moved to handler layer. - if err := checkBucketExist(bucket, obj); err != nil { - return errors.Trace(err) + if err := checkBucketExist(ctx, bucket, obj); err != nil { + return err } // Validates object prefix validity after bucket exists. if !IsValidObjectPrefix(prefix) { - return errors.Trace(ObjectNameInvalid{ + logger.LogIf(ctx, ObjectNameInvalid{ Bucket: bucket, Object: prefix, }) + return ObjectNameInvalid{ + Bucket: bucket, + Object: prefix, + } } // Verify if delimiter is anything other than '/', which we do not support. if delimiter != "" && delimiter != slashSeparator { - return errors.Trace(UnsupportedDelimiter{ + logger.LogIf(ctx, UnsupportedDelimiter{ Delimiter: delimiter, }) + return UnsupportedDelimiter{ + Delimiter: delimiter, + } } // Verify if marker has prefix. if marker != "" && !hasPrefix(marker, prefix) { - return errors.Trace(InvalidMarkerPrefixCombination{ + logger.LogIf(ctx, InvalidMarkerPrefixCombination{ Marker: marker, Prefix: prefix, }) + return InvalidMarkerPrefixCombination{ + Marker: marker, + Prefix: prefix, + } } return nil } // Checks for all ListMultipartUploads arguments validity. -func checkListMultipartArgs(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, obj ObjectLayer) error { - if err := checkListObjsArgs(bucket, prefix, keyMarker, delimiter, obj); err != nil { +func checkListMultipartArgs(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, obj ObjectLayer) error { + if err := checkListObjsArgs(ctx, bucket, prefix, keyMarker, delimiter, obj); err != nil { return err } if uploadIDMarker != "" { if hasSuffix(keyMarker, slashSeparator) { - return errors.Trace(InvalidUploadIDKeyCombination{ + + logger.LogIf(ctx, InvalidUploadIDKeyCombination{ UploadIDMarker: uploadIDMarker, KeyMarker: keyMarker, }) + return InvalidUploadIDKeyCombination{ + UploadIDMarker: uploadIDMarker, + KeyMarker: keyMarker, + } } id, err := uuid.Parse(uploadIDMarker) if err != nil { - return errors.Trace(err) + logger.LogIf(ctx, err) + return err } if id.IsZero() { - return errors.Trace(MalformedUploadID{ + logger.LogIf(ctx, MalformedUploadID{ UploadID: uploadIDMarker, }) + + return MalformedUploadID{ + UploadID: uploadIDMarker, + } } } return nil } // Checks for NewMultipartUpload arguments validity, also validates if bucket exists. -func checkNewMultipartArgs(bucket, object string, obj ObjectLayer) error { - return checkObjectArgs(bucket, object, obj) +func checkNewMultipartArgs(ctx context.Context, bucket, object string, obj ObjectLayer) error { + return checkObjectArgs(ctx, bucket, object, obj) } // Checks for PutObjectPart arguments validity, also validates if bucket exists. -func checkPutObjectPartArgs(bucket, object string, obj ObjectLayer) error { - return checkObjectArgs(bucket, object, obj) +func checkPutObjectPartArgs(ctx context.Context, bucket, object string, obj ObjectLayer) error { + return checkObjectArgs(ctx, bucket, object, obj) } // Checks for ListParts arguments validity, also validates if bucket exists. -func checkListPartsArgs(bucket, object string, obj ObjectLayer) error { - return checkObjectArgs(bucket, object, obj) +func checkListPartsArgs(ctx context.Context, bucket, object string, obj ObjectLayer) error { + return checkObjectArgs(ctx, bucket, object, obj) } // Checks for CompleteMultipartUpload arguments validity, also validates if bucket exists. -func checkCompleteMultipartArgs(bucket, object string, obj ObjectLayer) error { - return checkObjectArgs(bucket, object, obj) +func checkCompleteMultipartArgs(ctx context.Context, bucket, object string, obj ObjectLayer) error { + return checkObjectArgs(ctx, bucket, object, obj) } // Checks for AbortMultipartUpload arguments validity, also validates if bucket exists. -func checkAbortMultipartArgs(bucket, object string, obj ObjectLayer) error { - return checkObjectArgs(bucket, object, obj) +func checkAbortMultipartArgs(ctx context.Context, bucket, object string, obj ObjectLayer) error { + return checkObjectArgs(ctx, bucket, object, obj) } // Checks Object arguments validity, also validates if bucket exists. -func checkObjectArgs(bucket, object string, obj ObjectLayer) error { +func checkObjectArgs(ctx context.Context, bucket, object string, obj ObjectLayer) error { // Verify if bucket exists before validating object name. // This is done on purpose since the order of errors is // important here bucket does not exist error should // happen before we return an error for invalid object name. // FIXME: should be moved to handler layer. - if err := checkBucketExist(bucket, obj); err != nil { - return errors.Trace(err) + if err := checkBucketExist(ctx, bucket, obj); err != nil { + return err } // Validates object name validity after bucket exists. if !IsValidObjectName(object) { - return errors.Trace(ObjectNameInvalid{ + logger.LogIf(ctx, ObjectNameInvalid{ Bucket: bucket, Object: object, }) + + return ObjectNameInvalid{ + Bucket: bucket, + Object: object, + } } return nil } // Checks for PutObject arguments validity, also validates if bucket exists. -func checkPutObjectArgs(bucket, object string, obj ObjectLayer, size int64) error { +func checkPutObjectArgs(ctx context.Context, bucket, object string, obj ObjectLayer, size int64) error { // Verify if bucket exists before validating object name. // This is done on purpose since the order of errors is // important here bucket does not exist error should // happen before we return an error for invalid object name. // FIXME: should be moved to handler layer. - if err := checkBucketExist(bucket, obj); err != nil { - return errors.Trace(err) + if err := checkBucketExist(ctx, bucket, obj); err != nil { + return err } if len(object) == 0 || hasPrefix(object, slashSeparator) || (hasSuffix(object, slashSeparator) && size != 0) || !IsValidObjectPrefix(object) { - return errors.Trace(ObjectNameInvalid{ + logger.LogIf(ctx, ObjectNameInvalid{ Bucket: bucket, Object: object, }) + + return ObjectNameInvalid{ + Bucket: bucket, + Object: object, + } } return nil } // Checks whether bucket exists and returns appropriate error if not. -func checkBucketExist(bucket string, obj ObjectLayer) error { - _, err := obj.GetBucketInfo(context.Background(), bucket) +func checkBucketExist(ctx context.Context, bucket string, obj ObjectLayer) error { + _, err := obj.GetBucketInfo(ctx, bucket) if err != nil { return errors.Cause(err) } diff --git a/cmd/object-api-multipart_test.go b/cmd/object-api-multipart_test.go index 84b67a1ed..4a1abdbfb 100644 --- a/cmd/object-api-multipart_test.go +++ b/cmd/object-api-multipart_test.go @@ -1847,7 +1847,7 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T }, }, } - s3MD5, err := getCompleteMultipartMD5(inputParts[3].parts) + s3MD5, err := getCompleteMultipartMD5(context.Background(), inputParts[3].parts) if err != nil { t.Fatalf("Obtaining S3MD5 failed") } diff --git a/cmd/object-api-utils.go b/cmd/object-api-utils.go index 6b504f7f9..8cec6b106 100644 --- a/cmd/object-api-utils.go +++ b/cmd/object-api-utils.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "encoding/hex" "fmt" "path" @@ -24,7 +25,7 @@ import ( "strings" "unicode/utf8" - "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" "github.com/skyrings/skyring-common/tools/uuid" ) @@ -174,12 +175,13 @@ func mustGetUUID() string { } // Create an s3 compatible MD5sum for complete multipart transaction. -func getCompleteMultipartMD5(parts []CompletePart) (string, error) { +func getCompleteMultipartMD5(ctx context.Context, parts []CompletePart) (string, error) { var finalMD5Bytes []byte for _, part := range parts { md5Bytes, err := hex.DecodeString(part.ETag) if err != nil { - return "", errors.Trace(err) + logger.LogIf(ctx, err) + return "", err } finalMD5Bytes = append(finalMD5Bytes, md5Bytes...) } diff --git a/cmd/object-api-utils_test.go b/cmd/object-api-utils_test.go index 9629d586f..2918ace65 100644 --- a/cmd/object-api-utils_test.go +++ b/cmd/object-api-utils_test.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "reflect" "testing" ) @@ -148,7 +149,7 @@ func TestGetCompleteMultipartMD5(t *testing.T) { } for i, test := range testCases { - result, err := getCompleteMultipartMD5(test.parts) + result, err := getCompleteMultipartMD5(context.Background(), test.parts) if result != test.expectedResult { t.Fatalf("test %d failed: expected: result=%v, got=%v", i+1, test.expectedResult, result) } diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index dbb998dc0..354bac345 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "crypto/hmac" "encoding/binary" "encoding/hex" @@ -31,6 +32,7 @@ import ( "strconv" mux "github.com/gorilla/mux" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/event" "github.com/minio/minio/pkg/handlers" @@ -63,12 +65,12 @@ func setHeadGetRespHeaders(w http.ResponseWriter, reqParams url.Values) { // this is in keeping with the permissions sections of the docs of both: // HEAD Object: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html // GET Object: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html -func errAllowableObjectNotFound(bucket string, r *http.Request) APIErrorCode { +func errAllowableObjectNotFound(ctx context.Context, bucket string, r *http.Request) APIErrorCode { if getRequestAuthType(r) == authTypeAnonymous { // We care about the bucket as a whole, not a particular resource. resource := "/" + bucket sourceIP := handlers.GetSourceIP(r) - if s3Error := enforceBucketPolicy(bucket, "s3:ListBucket", resource, + if s3Error := enforceBucketPolicy(ctx, bucket, "s3:ListBucket", resource, r.Referer(), sourceIP, r.URL.Query()); s3Error != ErrNone { return ErrAccessDenied } @@ -95,7 +97,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req return } - if s3Error := checkRequestAuthType(r, bucket, "s3:GetObject", globalServerConfig.GetRegion()); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:GetObject", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return } @@ -109,7 +111,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req if err != nil { apiErr := toAPIErrorCode(err) if apiErr == ErrNoSuchKey { - apiErr = errAllowableObjectNotFound(bucket, r) + apiErr = errAllowableObjectNotFound(ctx, bucket, r) } writeErrorResponse(w, apiErr, r.URL) return @@ -135,7 +137,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req } // log the error. - errorIf(err, "Invalid request range") + logger.LogIf(ctx, err) } } @@ -182,7 +184,6 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req // Reads the object at startOffset and writes to mw. if err = getObject(ctx, bucket, object, startOffset, length, httpWriter, objInfo.ETag); err != nil { - errorIf(err, "Unable to write to client.") if !httpWriter.HasWritten() { // write error response only if no data has been written to client yet writeErrorResponse(w, toAPIErrorCode(err), r.URL) } @@ -232,7 +233,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re return } - if s3Error := checkRequestAuthType(r, bucket, "s3:GetObject", globalServerConfig.GetRegion()); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:GetObject", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponseHeadersOnly(w, s3Error) return } @@ -246,7 +247,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re if err != nil { apiErr := toAPIErrorCode(err) if apiErr == ErrNoSuchKey { - apiErr = errAllowableObjectNotFound(bucket, r) + apiErr = errAllowableObjectNotFound(ctx, bucket, r) } writeErrorResponseHeadersOnly(w, apiErr) return @@ -300,7 +301,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re // Extract metadata relevant for an CopyObject operation based on conditional // header values specified in X-Amz-Metadata-Directive. -func getCpObjMetadataFromHeader(header http.Header, userMeta map[string]string) (map[string]string, error) { +func getCpObjMetadataFromHeader(ctx context.Context, header http.Header, userMeta map[string]string) (map[string]string, error) { // Make a copy of the supplied metadata to avoid // to change the original one. defaultMeta := make(map[string]string, len(userMeta)) @@ -311,7 +312,7 @@ func getCpObjMetadataFromHeader(header http.Header, userMeta map[string]string) // if x-amz-metadata-directive says REPLACE then // we extract metadata from the input headers. if isMetadataReplace(header) { - return extractMetadataFromHeader(header) + return extractMetadataFromHeader(ctx, header) } // if x-amz-metadata-directive says COPY then we @@ -340,7 +341,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re return } - if s3Error := checkRequestAuthType(r, dstBucket, "s3:PutObject", globalServerConfig.GetRegion()); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, dstBucket, "s3:PutObject", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return } @@ -501,10 +502,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re } srcInfo.Writer = writer - srcInfo.UserDefined, err = getCpObjMetadataFromHeader(r.Header, srcInfo.UserDefined) + srcInfo.UserDefined, err = getCpObjMetadataFromHeader(ctx, r.Header, srcInfo.UserDefined) if err != nil { pipeWriter.CloseWithError(err) - errorIf(err, "found invalid http request header") writeErrorResponse(w, ErrInternalError, r.URL) return } @@ -628,9 +628,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req } // Extract metadata to be saved from incoming HTTP header. - metadata, err := extractMetadataFromHeader(r.Header) + metadata, err := extractMetadataFromHeader(ctx, r.Header) if err != nil { - errorIf(err, "found invalid http request header") writeErrorResponse(w, ErrInternalError, r.URL) return } @@ -670,7 +669,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html sourceIP := handlers.GetSourceIP(r) - if s3Err = enforceBucketPolicy(bucket, "s3:PutObject", r.URL.Path, r.Referer(), sourceIP, r.URL.Query()); s3Err != ErrNone { + if s3Err = enforceBucketPolicy(ctx, bucket, "s3:PutObject", r.URL.Path, r.Referer(), sourceIP, r.URL.Query()); s3Err != ErrNone { writeErrorResponse(w, s3Err, r.URL) return } @@ -782,7 +781,8 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r writeErrorResponse(w, ErrServerNotInitialized, r.URL) return } - if s3Error := checkRequestAuthType(r, bucket, "s3:PutObject", globalServerConfig.GetRegion()); s3Error != ErrNone { + + if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:PutObject", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return } @@ -825,9 +825,8 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r } // Extract metadata that needs to be saved. - metadata, err := extractMetadataFromHeader(r.Header) + metadata, err := extractMetadataFromHeader(ctx, r.Header) if err != nil { - errorIf(err, "found invalid http request header") writeErrorResponse(w, ErrInternalError, r.URL) return } @@ -869,7 +868,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt return } - if s3Error := checkRequestAuthType(r, dstBucket, "s3:PutObject", globalServerConfig.GetRegion()); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, dstBucket, "s3:PutObject", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return } @@ -931,7 +930,8 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt if hrange, err = parseCopyPartRange(rangeHeader, srcInfo.Size); err != nil { // Handle only errInvalidRange // Ignore other parse error and treat it as regular Get request like Amazon S3. - errorIf(err, "Unable to extract range %s", rangeHeader) + logger.GetReqInfo(ctx).AppendTags("rangeHeader", rangeHeader) + logger.LogIf(ctx, err) writeCopyPartErr(w, err, r.URL) return } @@ -1135,7 +1135,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html - if s3Error := enforceBucketPolicy(bucket, "s3:PutObject", r.URL.Path, + if s3Error := enforceBucketPolicy(ctx, bucket, "s3:PutObject", r.URL.Path, r.Referer(), handlers.GetSourceIP(r), r.URL.Query()); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return @@ -1263,7 +1263,7 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, if api.CacheAPI() != nil { abortMultipartUpload = api.CacheAPI().AbortMultipartUpload } - if s3Error := checkRequestAuthType(r, bucket, "s3:AbortMultipartUpload", globalServerConfig.GetRegion()); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:AbortMultipartUpload", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return } @@ -1278,7 +1278,6 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, uploadID, _, _, _ := getObjectResources(r.URL.Query()) if err := abortMultipartUpload(ctx, bucket, object, uploadID); err != nil { - errorIf(err, "AbortMultipartUpload failed") writeErrorResponse(w, toAPIErrorCode(err), r.URL) return } @@ -1299,7 +1298,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht return } - if s3Error := checkRequestAuthType(r, bucket, "s3:ListMultipartUploadParts", globalServerConfig.GetRegion()); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:ListMultipartUploadParts", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return } @@ -1339,7 +1338,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite return } - if s3Error := checkRequestAuthType(r, bucket, "s3:PutObject", globalServerConfig.GetRegion()); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:PutObject", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return } @@ -1449,7 +1448,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http. return } - if s3Error := checkRequestAuthType(r, bucket, "s3:DeleteObject", globalServerConfig.GetRegion()); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:DeleteObject", globalServerConfig.GetRegion()); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) return } @@ -1466,8 +1465,6 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http. // Ignore delete object errors while replying to client, since we are // suppposed to reply only 204. Additionally log the error for // investigation. - if err := deleteObject(ctx, objectAPI, api.CacheAPI(), bucket, object, r); err != nil { - errorIf(err, "Unable to delete an object %s", pathJoin(bucket, object)) - } + deleteObject(ctx, objectAPI, api.CacheAPI(), bucket, object, r) writeSuccessNoContent(w) } diff --git a/cmd/object-handlers_test.go b/cmd/object-handlers_test.go index 2176e42b0..68780ad51 100644 --- a/cmd/object-handlers_test.go +++ b/cmd/object-handlers_test.go @@ -2206,7 +2206,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s } // on successful complete multipart operation the s3MD5 for the parts uploaded will be returned. - s3MD5, err := getCompleteMultipartMD5(inputParts[3].parts) + s3MD5, err := getCompleteMultipartMD5(context.Background(), inputParts[3].parts) if err != nil { t.Fatalf("Obtaining S3MD5 failed") } diff --git a/cmd/peer-rpc.go b/cmd/peer-rpc.go index ea08b0d9b..865e35f5a 100644 --- a/cmd/peer-rpc.go +++ b/cmd/peer-rpc.go @@ -22,7 +22,7 @@ import ( "path" "github.com/gorilla/mux" - xerrors "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/event" xnet "github.com/minio/minio/pkg/net" ) @@ -103,7 +103,10 @@ func (receiver *PeerRPCReceiver) ListenBucketNotification(args *ListenBucketNoti target := NewPeerRPCClientTarget(args.BucketName, args.TargetID, rpcClient) rulesMap := event.NewRulesMap(args.EventNames, args.Pattern, target.ID()) if err := globalNotificationSys.AddRemoteTarget(args.BucketName, target, rulesMap); err != nil { - errorIf(err, "Unable to add PeerRPCClientTarget %v to globalNotificationSys.targetList.", target) + reqInfo := &logger.ReqInfo{BucketName: target.bucketName} + reqInfo.AppendTags("target", target.id.Name) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) return err } return nil @@ -158,7 +161,10 @@ func (receiver *PeerRPCReceiver) SendEvent(args *SendEventArgs, reply *SendEvent } if err != nil { - errorIf(err, "unable to send event %v to target %v", args.Event, args.TargetID) + reqInfo := (&logger.ReqInfo{}).AppendTags("Event", args.Event.EventName.String()) + reqInfo.AppendTags("target", args.TargetID.Name) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) } reply.Error = err @@ -169,7 +175,8 @@ func (receiver *PeerRPCReceiver) SendEvent(args *SendEventArgs, reply *SendEvent func registerS3PeerRPCRouter(router *mux.Router) error { peerRPCServer := newRPCServer() if err := peerRPCServer.RegisterName("Peer", &PeerRPCReceiver{}); err != nil { - return xerrors.Trace(err) + logger.LogIf(context.Background(), err) + return err } subrouter := router.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter() @@ -250,7 +257,11 @@ func (rpcClient *PeerRPCClient) SendEvent(bucketName string, targetID, remoteTar } if reply.Error != nil { - errorIf(reply.Error, "unable to send event %v to rpc target %v of bucket %v", args, targetID, bucketName) + reqInfo := &logger.ReqInfo{BucketName: bucketName} + reqInfo.AppendTags("targetID", targetID.Name) + reqInfo.AppendTags("event", eventData.EventName.String()) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, reply.Error) globalNotificationSys.RemoveRemoteTarget(bucketName, targetID) } diff --git a/cmd/posix-list-dir-others.go b/cmd/posix-list-dir-others.go index 5156695b1..4cadd79ff 100644 --- a/cmd/posix-list-dir-others.go +++ b/cmd/posix-list-dir-others.go @@ -19,10 +19,13 @@ package cmd import ( + "context" "io" "os" "path" "strings" + + "github.com/minio/minio/cmd/logger" ) // Return all the entries at the directory dirPath. @@ -57,7 +60,9 @@ func readDir(dirPath string) (entries []string, err error) { var st os.FileInfo st, err = os.Stat((path.Join(dirPath, fi.Name()))) if err != nil { - errorIf(err, "Unable to stat path %s", path.Join(dirPath, fi.Name())) + reqInfo := (&logger.ReqInfo{}).AppendTags("path", path.Join(dirPath, fi.Name())) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) continue } // Append to entries if symbolic link exists and is valid. diff --git a/cmd/posix.go b/cmd/posix.go index ac9f59163..9d08838d7 100644 --- a/cmd/posix.go +++ b/cmd/posix.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "encoding/hex" "io" "io/ioutil" @@ -30,6 +31,7 @@ import ( "syscall" humanize "github.com/dustin/go-humanize" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/disk" ) @@ -75,7 +77,7 @@ func isDirEmpty(dirname string) bool { f, err := os.Open((dirname)) if err != nil { if !os.IsNotExist(err) { - errorIf(err, "Unable to access directory") + logger.LogIf(context.Background(), err) } return false @@ -85,7 +87,7 @@ func isDirEmpty(dirname string) bool { _, err = f.Readdirnames(1) if err != io.EOF { if !os.IsNotExist(err) { - errorIf(err, "Unable to list directory") + logger.LogIf(context.Background(), err) } return false diff --git a/cmd/prepare-storage.go b/cmd/prepare-storage.go index a91521d34..29bfa8463 100644 --- a/cmd/prepare-storage.go +++ b/cmd/prepare-storage.go @@ -17,11 +17,13 @@ package cmd import ( + "context" "fmt" "os" "time" "github.com/minio/mc/pkg/console" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/errors" ) @@ -29,19 +31,21 @@ var printEndpointError = func() func(Endpoint, error) { printOnce := make(map[Endpoint]map[string]bool) return func(endpoint Endpoint, err error) { + reqInfo := (&logger.ReqInfo{}).AppendTags("endpoint", endpoint.Host) + ctx := logger.SetReqInfo(context.Background(), reqInfo) m, ok := printOnce[endpoint] if !ok { m = make(map[string]bool) m[err.Error()] = true printOnce[endpoint] = m - errorIf(err, "%s: %s", endpoint, err) + logger.LogIf(ctx, err) return } if m[err.Error()] { return } m[err.Error()] = true - errorIf(err, "%s: %s", endpoint, err) + logger.LogIf(ctx, err) } }() @@ -147,7 +151,7 @@ func connectLoadInitFormats(firstDisk bool, endpoints EndpointList, setCount, dr if !firstDisk { return nil, errNotFirstDisk } - return initFormatXL(storageDisks, setCount, drivesPerSet) + return initFormatXL(context.Background(), storageDisks, setCount, drivesPerSet) } // Following function is added to fix a regressions which was introduced @@ -178,7 +182,7 @@ func connectLoadInitFormats(firstDisk bool, endpoints EndpointList, setCount, dr } // Format disks before initialization of object layer. -func waitForFormatXL(firstDisk bool, endpoints EndpointList, setCount, disksPerSet int) (format *formatXLV3, err error) { +func waitForFormatXL(ctx context.Context, firstDisk bool, endpoints EndpointList, setCount, disksPerSet int) (format *formatXLV3, err error) { if len(endpoints) == 0 || setCount == 0 || disksPerSet == 0 { return nil, errInvalidArgument } diff --git a/cmd/rpc-server.go b/cmd/rpc-server.go index c0b295883..24c1f4fd8 100644 --- a/cmd/rpc-server.go +++ b/cmd/rpc-server.go @@ -17,11 +17,13 @@ package cmd import ( + "context" "io" "net/http" "net/rpc" miniohttp "github.com/minio/minio/cmd/http" + "github.com/minio/minio/cmd/logger" ) // ServeHTTP implements an http.Handler that answers RPC requests, @@ -34,7 +36,9 @@ func (server *rpcServer) ServeHTTP(w http.ResponseWriter, req *http.Request) { conn, _, err := w.(http.Hijacker).Hijack() if err != nil { - errorIf(err, "rpc hijacking failed for: %s", req.RemoteAddr) + reqInfo := (&logger.ReqInfo{}).AppendTags("remoteaddr", req.RemoteAddr) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) return } diff --git a/cmd/server-main.go b/cmd/server-main.go index 6946f270f..74320873b 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "net/http" "os" "os/signal" @@ -26,7 +27,7 @@ import ( "github.com/minio/cli" "github.com/minio/dsync" miniohttp "github.com/minio/minio/cmd/http" - "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" ) var serverFlags = []cli.Flag{ @@ -128,17 +129,17 @@ func serverHandleCmdArgs(ctx *cli.Context) { // Server address. serverAddr := ctx.String("address") - fatalIf(CheckLocalServerAddr(serverAddr), "Invalid address ‘%s’ in command line argument.", serverAddr) + logger.FatalIf(CheckLocalServerAddr(serverAddr), "Invalid address ‘%s’ in command line argument.", serverAddr) var setupType SetupType var err error if len(ctx.Args()) > serverCommandLineArgsMax { - fatalIf(errInvalidArgument, "Invalid total number of arguments (%d) passed, supported upto 32 unique arguments", len(ctx.Args())) + logger.FatalIf(errInvalidArgument, "Invalid total number of arguments (%d) passed, supported upto 32 unique arguments", len(ctx.Args())) } globalMinioAddr, globalEndpoints, setupType, globalXLSetCount, globalXLSetDriveCount, err = createServerEndpoints(serverAddr, ctx.Args()...) - fatalIf(err, "Invalid command line arguments server=‘%s’, args=%s", serverAddr, ctx.Args()) + logger.FatalIf(err, "Invalid command line arguments server=‘%s’, args=%s", serverAddr, ctx.Args()) globalMinioHost, globalMinioPort = mustSplitHostPort(globalMinioAddr) if runtime.GOOS == "darwin" { @@ -146,7 +147,7 @@ func serverHandleCmdArgs(ctx *cli.Context) { // to IPv6 address ie minio will start listening on IPv6 address whereas another // (non-)minio process is listening on IPv4 of given port. // To avoid this error sutiation we check for port availability only for macOS. - fatalIf(checkPortAvailability(globalMinioPort), "Port %d already in use", globalMinioPort) + logger.FatalIf(checkPortAvailability(globalMinioPort), "Port %d already in use", globalMinioPort) } globalIsXL = (setupType == XLSetupType) @@ -168,6 +169,10 @@ func serverHandleEnvVars() { } +func init() { + logger.Init(GOPATH) +} + // serverMain handler called for 'minio server' command. func serverMain(ctx *cli.Context) { if (!ctx.IsSet("sets") && !ctx.Args().Present()) || ctx.Args().First() == "help" { @@ -178,13 +183,13 @@ func serverMain(ctx *cli.Context) { // enable json and quite modes if jason flag is turned on. jsonFlag := ctx.IsSet("json") || ctx.GlobalIsSet("json") if jsonFlag { - log.EnableJSON() + logger.EnableJSON() } // Get quiet flag from command line argument. quietFlag := ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet") if quietFlag { - log.EnableQuiet() + logger.EnableQuiet() } // Handle all server command args. @@ -194,22 +199,19 @@ func serverMain(ctx *cli.Context) { serverHandleEnvVars() // Create certs path. - fatalIf(createConfigDir(), "Unable to create configuration directories.") + logger.FatalIf(createConfigDir(), "Unable to create configuration directories.") // Initialize server config. initConfig() - // Init the error tracing module. - errors.Init(GOPATH, "github.com/minio/minio") - // Check and load SSL certificates. var err error globalPublicCerts, globalRootCAs, globalTLSCertificate, globalIsSSL, err = getSSLConfig() - fatalIf(err, "Invalid SSL certificate file") + logger.FatalIf(err, "Invalid SSL certificate file") // Is distributed setup, error out if no certificates are found for HTTPS endpoints. if globalIsDistXL && globalEndpoints.IsHTTPS() && !globalIsSSL { - fatalIf(errInvalidArgument, "No certificates found for HTTPS endpoints (%s)", globalEndpoints) + logger.FatalIf(errInvalidArgument, "No certificates found for HTTPS endpoints (%s)", globalEndpoints) } if !quietFlag { @@ -224,12 +226,12 @@ func serverMain(ctx *cli.Context) { } // Set system resources to maximum. - errorIf(setMaxResources(), "Unable to change resource limit") + logger.LogIf(context.Background(), setMaxResources()) // Set nodes for dsync for distributed setup. if globalIsDistXL { globalDsync, err = dsync.New(newDsyncNodes(globalEndpoints)) - fatalIf(err, "Unable to initialize distributed locking on %s", globalEndpoints) + logger.FatalIf(err, "Unable to initialize distributed locking on %s", globalEndpoints) } // Initialize name space lock. @@ -241,11 +243,11 @@ func serverMain(ctx *cli.Context) { // Configure server. var handler http.Handler handler, err = configureServerHandler(globalEndpoints) - fatalIf(err, "Unable to configure one of server's RPC services.") + logger.FatalIf(err, "Unable to configure one of server's RPC services.") // Initialize notification system. globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints) - fatalIf(err, "Unable to initialize notification system.") + logger.FatalIf(err, "Unable to initialize notification system.") // Initialize Admin Peers inter-node communication only in distributed setup. initGlobalAdminPeers(globalEndpoints) @@ -255,7 +257,6 @@ func serverMain(ctx *cli.Context) { globalHTTPServer.WriteTimeout = globalConnWriteTimeout globalHTTPServer.UpdateBytesReadFunc = globalConnStats.incInputBytes globalHTTPServer.UpdateBytesWrittenFunc = globalConnStats.incOutputBytes - globalHTTPServer.ErrorLogFunc = errorIf go func() { globalHTTPServerErrorCh <- globalHTTPServer.Start() }() @@ -264,9 +265,9 @@ func serverMain(ctx *cli.Context) { newObject, err := newObjectLayer(globalEndpoints) if err != nil { - errorIf(err, "Initializing object layer failed") + logger.LogIf(context.Background(), err) err = globalHTTPServer.Shutdown() - errorIf(err, "Unable to shutdown http server") + logger.LogIf(context.Background(), err) os.Exit(1) } @@ -294,7 +295,7 @@ func newObjectLayer(endpoints EndpointList) (newObject ObjectLayer, err error) { return NewFSObjectLayer(endpoints[0].Path) } - format, err := waitForFormatXL(endpoints[0].IsLocal, endpoints, globalXLSetCount, globalXLSetDriveCount) + format, err := waitForFormatXL(context.Background(), endpoints[0].IsLocal, endpoints, globalXLSetCount, globalXLSetDriveCount) if err != nil { return nil, err } diff --git a/cmd/server-startup-msg.go b/cmd/server-startup-msg.go index 501a102f1..42831ed63 100644 --- a/cmd/server-startup-msg.go +++ b/cmd/server-startup-msg.go @@ -25,6 +25,7 @@ import ( "strings" humanize "github.com/dustin/go-humanize" + "github.com/minio/minio/cmd/logger" ) // Documentation links, these are part of message printing code. @@ -114,17 +115,17 @@ func printServerCommonMsg(apiEndpoints []string) { apiEndpointStr := strings.Join(apiEndpoints, " ") // Colorize the message and print. - log.Println(colorBlue("Endpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr))) - log.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey))) - log.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey))) + logger.Println(colorBlue("Endpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr))) + logger.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey))) + logger.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey))) if region != "" { - log.Println(colorBlue("Region: ") + colorBold(fmt.Sprintf(getFormatStr(len(region), 3), region))) + logger.Println(colorBlue("Region: ") + colorBold(fmt.Sprintf(getFormatStr(len(region), 3), region))) } printEventNotifiers() if globalIsBrowserEnabled { - log.Println(colorBlue("\nBrowser Access:")) - log.Println(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr)) + logger.Println(colorBlue("\nBrowser Access:")) + logger.Println(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr)) } } @@ -140,7 +141,7 @@ func printEventNotifiers() { arnMsg += colorBold(fmt.Sprintf(getFormatStr(len(arn), 1), arn)) } - log.Println(arnMsg) + logger.Println(arnMsg) } // Prints startup message for command line access. Prints link to our documentation @@ -150,24 +151,24 @@ func printCLIAccessMsg(endPoint string, alias string) { cred := globalServerConfig.GetCredential() // Configure 'mc', following block prints platform specific information for minio client. - log.Println(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide) + logger.Println(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide) if runtime.GOOS == globalWindowsOSName { mcMessage := fmt.Sprintf("$ mc.exe config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey) - log.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage)) + logger.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage)) } else { mcMessage := fmt.Sprintf("$ mc config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey) - log.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage)) + logger.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage)) } } // Prints startup message for Object API acces, prints link to our SDK documentation. func printObjectAPIMsg() { - log.Println(colorBlue("\nObject API (Amazon S3 compatible):")) - log.Println(colorBlue(" Go: ") + fmt.Sprintf(getFormatStr(len(goQuickStartGuide), 8), goQuickStartGuide)) - log.Println(colorBlue(" Java: ") + fmt.Sprintf(getFormatStr(len(javaQuickStartGuide), 6), javaQuickStartGuide)) - log.Println(colorBlue(" Python: ") + fmt.Sprintf(getFormatStr(len(pyQuickStartGuide), 4), pyQuickStartGuide)) - log.Println(colorBlue(" JavaScript: ") + jsQuickStartGuide) - log.Println(colorBlue(" .NET: ") + fmt.Sprintf(getFormatStr(len(dotnetQuickStartGuide), 6), dotnetQuickStartGuide)) + logger.Println(colorBlue("\nObject API (Amazon S3 compatible):")) + logger.Println(colorBlue(" Go: ") + fmt.Sprintf(getFormatStr(len(goQuickStartGuide), 8), goQuickStartGuide)) + logger.Println(colorBlue(" Java: ") + fmt.Sprintf(getFormatStr(len(javaQuickStartGuide), 6), javaQuickStartGuide)) + logger.Println(colorBlue(" Python: ") + fmt.Sprintf(getFormatStr(len(pyQuickStartGuide), 4), pyQuickStartGuide)) + logger.Println(colorBlue(" JavaScript: ") + jsQuickStartGuide) + logger.Println(colorBlue(" .NET: ") + fmt.Sprintf(getFormatStr(len(dotnetQuickStartGuide), 6), dotnetQuickStartGuide)) } // Get formatted disk/storage info message. @@ -184,15 +185,15 @@ func getStorageInfoMsg(storageInfo StorageInfo) string { // Prints startup message of storage capacity and erasure information. func printStorageInfo(storageInfo StorageInfo) { - log.Println(getStorageInfoMsg(storageInfo)) - log.Println() + logger.Println(getStorageInfoMsg(storageInfo)) + logger.Println() } func printCacheStorageInfo(storageInfo StorageInfo) { msg := fmt.Sprintf("%s %s Free, %s Total", colorBlue("Cache Capacity:"), humanize.IBytes(uint64(storageInfo.Free)), humanize.IBytes(uint64(storageInfo.Total))) - log.Println(msg) + logger.Println(msg) } // Prints certificate expiry date warning @@ -215,5 +216,5 @@ func getCertificateChainMsg(certs []*x509.Certificate) string { // Prints the certificate expiry message. func printCertificateMsg(certs []*x509.Certificate) { - log.Println(getCertificateChainMsg(certs)) + logger.Println(getCertificateChainMsg(certs)) } diff --git a/cmd/server_test.go b/cmd/server_test.go index a3d8b6a9e..b6f930fa1 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -18,6 +18,7 @@ package cmd import ( "bytes" + "context" "crypto/tls" "crypto/x509" "encoding/xml" @@ -2732,7 +2733,7 @@ func (s *TestSuiteCommon) TestObjectMultipart(c *check) { part.ETag = canonicalizeETag(part.ETag) parts = append(parts, part) } - etag, err := getCompleteMultipartMD5(parts) + etag, err := getCompleteMultipartMD5(context.Background(), parts) c.Assert(err, nil) c.Assert(canonicalizeETag(response.Header.Get("Etag")), etag) } diff --git a/cmd/signals.go b/cmd/signals.go index 913e3c4e2..ffb48c7a3 100644 --- a/cmd/signals.go +++ b/cmd/signals.go @@ -19,6 +19,8 @@ package cmd import ( "context" "os" + + "github.com/minio/minio/cmd/logger" ) func handleSignals() { @@ -44,11 +46,11 @@ func handleSignals() { } err = globalHTTPServer.Shutdown() - errorIf(err, "Unable to shutdown http server") + logger.LogIf(context.Background(), err) if objAPI := newObjectLayerFn(); objAPI != nil { oerr = objAPI.Shutdown(context.Background()) - errorIf(oerr, "Unable to shutdown object layer") + logger.LogIf(context.Background(), oerr) } return (err == nil && oerr == nil) @@ -57,33 +59,32 @@ func handleSignals() { for { select { case err := <-globalHTTPServerErrorCh: - errorIf(err, "http server exited abnormally") + logger.LogIf(context.Background(), err) var oerr error if objAPI := newObjectLayerFn(); objAPI != nil { oerr = objAPI.Shutdown(context.Background()) - errorIf(oerr, "Unable to shutdown object layer") } exit(err == nil && oerr == nil) case osSignal := <-globalOSSignalCh: stopHTTPTrace() - log.Printf("Exiting on signal %v\n", osSignal) + logger.Printf("Exiting on signal %v\n", osSignal) exit(stopProcess()) case signal := <-globalServiceSignalCh: switch signal { case serviceStatus: // Ignore this at the moment. case serviceRestart: - log.Println("Restarting on service signal") + logger.Println("Restarting on service signal") err := globalHTTPServer.Shutdown() - errorIf(err, "Unable to shutdown http server") + logger.LogIf(context.Background(), err) stopHTTPTrace() rerr := restartProcess() - errorIf(rerr, "Unable to restart the server") + logger.LogIf(context.Background(), rerr) exit(err == nil && rerr == nil) case serviceStop: - log.Println("Stopping on service signal") + logger.Println("Stopping on service signal") stopHTTPTrace() exit(stopProcess()) } diff --git a/cmd/storage-class_test.go b/cmd/storage-class_test.go index f230ff57c..cba2c6e44 100644 --- a/cmd/storage-class_test.go +++ b/cmd/storage-class_test.go @@ -193,7 +193,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin t.Fatalf("Failed to putObject %v", err) } - parts1, errs1 := readAllXLMetadata(xlDisks, bucket, object1) + parts1, errs1 := readAllXLMetadata(context.Background(), xlDisks, bucket, object1) // Object for test case 2 - No StorageClass defined, MetaData in PutObject requesting RRS Class object2 := "object2" @@ -204,7 +204,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin t.Fatalf("Failed to putObject %v", err) } - parts2, errs2 := readAllXLMetadata(xlDisks, bucket, object2) + parts2, errs2 := readAllXLMetadata(context.Background(), xlDisks, bucket, object2) // Object for test case 3 - No StorageClass defined, MetaData in PutObject requesting Standard Storage Class object3 := "object3" @@ -215,7 +215,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin t.Fatalf("Failed to putObject %v", err) } - parts3, errs3 := readAllXLMetadata(xlDisks, bucket, object3) + parts3, errs3 := readAllXLMetadata(context.Background(), xlDisks, bucket, object3) // Object for test case 4 - Standard StorageClass defined as Parity 6, MetaData in PutObject requesting Standard Storage Class object4 := "object4" @@ -231,7 +231,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin t.Fatalf("Failed to putObject %v", err) } - parts4, errs4 := readAllXLMetadata(xlDisks, bucket, object4) + parts4, errs4 := readAllXLMetadata(context.Background(), xlDisks, bucket, object4) // Object for test case 5 - RRS StorageClass defined as Parity 2, MetaData in PutObject requesting RRS Class // Reset global storage class flags @@ -249,7 +249,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin t.Fatalf("Failed to putObject %v", err) } - parts5, errs5 := readAllXLMetadata(xlDisks, bucket, object5) + parts5, errs5 := readAllXLMetadata(context.Background(), xlDisks, bucket, object5) // Object for test case 6 - RRS StorageClass defined as Parity 2, MetaData in PutObject requesting Standard Storage Class // Reset global storage class flags @@ -267,7 +267,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin t.Fatalf("Failed to putObject %v", err) } - parts6, errs6 := readAllXLMetadata(xlDisks, bucket, object6) + parts6, errs6 := readAllXLMetadata(context.Background(), xlDisks, bucket, object6) // Object for test case 7 - Standard StorageClass defined as Parity 5, MetaData in PutObject requesting RRS Class // Reset global storage class flags @@ -285,7 +285,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin t.Fatalf("Failed to putObject %v", err) } - parts7, errs7 := readAllXLMetadata(xlDisks, bucket, object7) + parts7, errs7 := readAllXLMetadata(context.Background(), xlDisks, bucket, object7) tests := []struct { parts []xlMetaV1 diff --git a/cmd/storage-rpc-server.go b/cmd/storage-rpc-server.go index 363fed97e..3a6a3cc8a 100644 --- a/cmd/storage-rpc-server.go +++ b/cmd/storage-rpc-server.go @@ -17,13 +17,14 @@ package cmd import ( + "context" "io" "path" "time" router "github.com/gorilla/mux" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/disk" - "github.com/minio/minio/pkg/errors" ) // Storage server implements rpc primitives to facilitate exporting a @@ -224,7 +225,8 @@ func registerStorageRPCRouters(mux *router.Router, endpoints EndpointList) error // Initialize storage rpc servers for every disk that is hosted on this node. storageRPCs, err := newStorageRPCServer(endpoints) if err != nil { - return errors.Trace(err) + logger.LogIf(context.Background(), err) + return err } // Create a unique route for each disk exported from this node. @@ -232,7 +234,8 @@ func registerStorageRPCRouters(mux *router.Router, endpoints EndpointList) error storageRPCServer := newRPCServer() err = storageRPCServer.RegisterName("Storage", stServer) if err != nil { - return errors.Trace(err) + logger.LogIf(context.Background(), err) + return err } // Add minio storage routes. storageRouter := mux.PathPrefix(minioReservedBucketPath).Subrouter() diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index 732038644..78ca4fb92 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -55,6 +55,7 @@ import ( router "github.com/gorilla/mux" "github.com/minio/minio-go/pkg/policy" "github.com/minio/minio-go/pkg/s3signer" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" "github.com/minio/minio/pkg/bpool" "github.com/minio/minio/pkg/hash" @@ -74,8 +75,7 @@ func init() { // Set system resources to maximum. setMaxResources() - log = NewLogger() - log.EnableQuiet() + logger.EnableQuiet() } // concurreny level for certain parallel tests. @@ -187,7 +187,7 @@ func prepareXL32() (ObjectLayer, []string, error) { endpoints := append(endpoints1, endpoints2...) fsDirs := append(fsDirs1, fsDirs2...) - format, err := waitForFormatXL(true, endpoints, 2, 16) + format, err := waitForFormatXL(context.Background(), true, endpoints, 2, 16) if err != nil { removeRoots(fsDirs) return nil, nil, err @@ -1685,7 +1685,7 @@ func newTestObjectLayer(endpoints EndpointList) (newObject ObjectLayer, err erro return NewFSObjectLayer(endpoints[0].Path) } - _, err = waitForFormatXL(endpoints[0].IsLocal, endpoints, 1, 16) + _, err = waitForFormatXL(context.Background(), endpoints[0].IsLocal, endpoints, 1, 16) if err != nil { return nil, err } @@ -2423,12 +2423,12 @@ func generateTLSCertKey(host string) ([]byte, []byte, error) { func mustGetNewEndpointList(args ...string) (endpoints EndpointList) { if len(args) == 1 { endpoint, err := NewEndpoint(args[0]) - fatalIf(err, "unable to create new endpoint") + logger.FatalIf(err, "unable to create new endpoint") endpoints = append(endpoints, endpoint) } else { var err error endpoints, err = NewEndpointList(args...) - fatalIf(err, "unable to create new endpoint list") + logger.FatalIf(err, "unable to create new endpoint list") } return endpoints } diff --git a/cmd/tree-walk.go b/cmd/tree-walk.go index d902f6169..ebdffdde9 100644 --- a/cmd/tree-walk.go +++ b/cmd/tree-walk.go @@ -17,10 +17,11 @@ package cmd import ( + "context" "sort" "strings" - "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" ) // Tree walk result carries results of tree walking. @@ -124,7 +125,7 @@ func filterListEntries(bucket, prefixDir string, entries []string, prefixEntry s } // treeWalk walks directory tree recursively pushing treeWalkResult into the channel as and when it encounters files. -func doTreeWalk(bucket, prefixDir, entryPrefixMatch, marker string, recursive bool, listDir listDirFunc, isLeaf isLeafFunc, resultCh chan treeWalkResult, endWalkCh chan struct{}, isEnd bool) error { +func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker string, recursive bool, listDir listDirFunc, isLeaf isLeafFunc, resultCh chan treeWalkResult, endWalkCh chan struct{}, isEnd bool) error { // Example: // if prefixDir="one/two/three/" and marker="four/five.txt" treeWalk is recursively // called with prefixDir="one/two/three/four/" and marker="five.txt" @@ -143,7 +144,8 @@ func doTreeWalk(bucket, prefixDir, entryPrefixMatch, marker string, recursive bo if err != nil { select { case <-endWalkCh: - return errors.Trace(errWalkAbort) + logger.LogIf(ctx, errWalkAbort) + return errWalkAbort case resultCh <- treeWalkResult{err: err}: return err } @@ -196,7 +198,7 @@ func doTreeWalk(bucket, prefixDir, entryPrefixMatch, marker string, recursive bo // markIsEnd is passed to this entry's treeWalk() so that treeWalker.end can be marked // true at the end of the treeWalk stream. markIsEnd := i == len(entries)-1 && isEnd - if tErr := doTreeWalk(bucket, pathJoin(prefixDir, entry), prefixMatch, markerArg, recursive, listDir, isLeaf, resultCh, endWalkCh, markIsEnd); tErr != nil { + if tErr := doTreeWalk(ctx, bucket, pathJoin(prefixDir, entry), prefixMatch, markerArg, recursive, listDir, isLeaf, resultCh, endWalkCh, markIsEnd); tErr != nil { return tErr } continue @@ -205,7 +207,8 @@ func doTreeWalk(bucket, prefixDir, entryPrefixMatch, marker string, recursive bo isEOF := ((i == len(entries)-1) && isEnd) select { case <-endWalkCh: - return errors.Trace(errWalkAbort) + logger.LogIf(ctx, errWalkAbort) + return errWalkAbort case resultCh <- treeWalkResult{entry: pathJoin(prefixDir, entry), end: isEOF}: } } @@ -215,7 +218,7 @@ func doTreeWalk(bucket, prefixDir, entryPrefixMatch, marker string, recursive bo } // Initiate a new treeWalk in a goroutine. -func startTreeWalk(bucket, prefix, marker string, recursive bool, listDir listDirFunc, isLeaf isLeafFunc, endWalkCh chan struct{}) chan treeWalkResult { +func startTreeWalk(ctx context.Context, bucket, prefix, marker string, recursive bool, listDir listDirFunc, isLeaf isLeafFunc, endWalkCh chan struct{}) chan treeWalkResult { // Example 1 // If prefix is "one/two/three/" and marker is "one/two/three/four/five.txt" // treeWalk is called with prefixDir="one/two/three/" and marker="four/five.txt" @@ -237,7 +240,7 @@ func startTreeWalk(bucket, prefix, marker string, recursive bool, listDir listDi marker = strings.TrimPrefix(marker, prefixDir) go func() { isEnd := true // Indication to start walking the tree with end as true. - doTreeWalk(bucket, prefixDir, entryPrefixMatch, marker, recursive, listDir, isLeaf, resultCh, endWalkCh, isEnd) + doTreeWalk(ctx, bucket, prefixDir, entryPrefixMatch, marker, recursive, listDir, isLeaf, resultCh, endWalkCh, isEnd) close(resultCh) }() return resultCh diff --git a/cmd/tree-walk_test.go b/cmd/tree-walk_test.go index 081e6f653..e1626ca2b 100644 --- a/cmd/tree-walk_test.go +++ b/cmd/tree-walk_test.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "fmt" "io/ioutil" "os" @@ -131,7 +132,7 @@ func testTreeWalkPrefix(t *testing.T, listDir listDirFunc, isLeaf isLeafFunc) { // Start the tree walk go-routine. prefix := "d/" endWalkCh := make(chan struct{}) - twResultCh := startTreeWalk(volume, prefix, "", true, listDir, isLeaf, endWalkCh) + twResultCh := startTreeWalk(context.Background(), volume, prefix, "", true, listDir, isLeaf, endWalkCh) // Check if all entries received on the channel match the prefix. for res := range twResultCh { @@ -146,7 +147,7 @@ func testTreeWalkMarker(t *testing.T, listDir listDirFunc, isLeaf isLeafFunc) { // Start the tree walk go-routine. prefix := "" endWalkCh := make(chan struct{}) - twResultCh := startTreeWalk(volume, prefix, "d/g", true, listDir, isLeaf, endWalkCh) + twResultCh := startTreeWalk(context.Background(), volume, prefix, "d/g", true, listDir, isLeaf, endWalkCh) // Check if only 3 entries, namely d/g/h, i/j/k, lmn are received on the channel. expectedCount := 3 @@ -186,7 +187,7 @@ func TestTreeWalk(t *testing.T) { isLeaf := func(volume, prefix string) bool { return !hasSuffix(prefix, slashSeparator) } - listDir := listDirFactory(isLeaf, xlTreeWalkIgnoredErrs, disk) + listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk) // Simple test for prefix based walk. testTreeWalkPrefix(t, listDir, isLeaf) // Simple test when marker is set. @@ -221,7 +222,7 @@ func TestTreeWalkTimeout(t *testing.T) { isLeaf := func(volume, prefix string) bool { return !hasSuffix(prefix, slashSeparator) } - listDir := listDirFactory(isLeaf, xlTreeWalkIgnoredErrs, disk) + listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk) // TreeWalk pool with 2 seconds timeout for tree-walk go routines. pool := newTreeWalkPool(2 * time.Second) @@ -230,7 +231,7 @@ func TestTreeWalkTimeout(t *testing.T) { prefix := "" marker := "" recursive := true - resultCh := startTreeWalk(volume, prefix, marker, recursive, listDir, isLeaf, endWalkCh) + resultCh := startTreeWalk(context.Background(), volume, prefix, marker, recursive, listDir, isLeaf, endWalkCh) params := listParams{ bucket: volume, @@ -294,7 +295,7 @@ func TestListDir(t *testing.T) { } // create listDir function. - listDir := listDirFactory(func(volume, prefix string) bool { + listDir := listDirFactory(context.Background(), func(volume, prefix string) bool { return !hasSuffix(prefix, slashSeparator) }, xlTreeWalkIgnoredErrs, disk1, disk2) @@ -373,7 +374,7 @@ func TestRecursiveTreeWalk(t *testing.T) { } // Create listDir function. - listDir := listDirFactory(isLeaf, xlTreeWalkIgnoredErrs, disk1) + listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk1) // Create the namespace. var files = []string{ @@ -447,7 +448,7 @@ func TestRecursiveTreeWalk(t *testing.T) { }}, } for i, testCase := range testCases { - for entry := range startTreeWalk(volume, + for entry := range startTreeWalk(context.Background(), volume, testCase.prefix, testCase.marker, testCase.recursive, listDir, isLeaf, endWalkCh) { if _, found := testCase.expected[entry.entry]; !found { @@ -479,7 +480,7 @@ func TestSortedness(t *testing.T) { return !hasSuffix(prefix, slashSeparator) } // Create listDir function. - listDir := listDirFactory(isLeaf, xlTreeWalkIgnoredErrs, disk1) + listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk1) // Create the namespace. var files = []string{ @@ -519,7 +520,7 @@ func TestSortedness(t *testing.T) { } for i, test := range testCases { var actualEntries []string - for entry := range startTreeWalk(volume, + for entry := range startTreeWalk(context.Background(), volume, test.prefix, test.marker, test.recursive, listDir, isLeaf, endWalkCh) { actualEntries = append(actualEntries, entry.entry) @@ -553,7 +554,7 @@ func TestTreeWalkIsEnd(t *testing.T) { return !hasSuffix(prefix, slashSeparator) } // Create listDir function. - listDir := listDirFactory(isLeaf, xlTreeWalkIgnoredErrs, disk1) + listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk1) // Create the namespace. var files = []string{ @@ -594,7 +595,7 @@ func TestTreeWalkIsEnd(t *testing.T) { } for i, test := range testCases { var entry treeWalkResult - for entry = range startTreeWalk(volume, test.prefix, test.marker, test.recursive, listDir, isLeaf, endWalkCh) { + for entry = range startTreeWalk(context.Background(), volume, test.prefix, test.marker, test.recursive, listDir, isLeaf, endWalkCh) { } if entry.entry != test.expectedEntry { t.Errorf("Test %d: Expected entry %s, but received %s with the EOF marker", i, test.expectedEntry, entry.entry) diff --git a/cmd/update-main.go b/cmd/update-main.go index 50551ab0c..bc32a10ff 100644 --- a/cmd/update-main.go +++ b/cmd/update-main.go @@ -18,6 +18,7 @@ package cmd import ( "bufio" + "context" "crypto" "encoding/hex" "fmt" @@ -32,6 +33,7 @@ import ( "github.com/fatih/color" "github.com/inconshreveable/go-update" "github.com/minio/cli" + "github.com/minio/minio/cmd/logger" _ "github.com/minio/sha256-simd" // Needed for sha256 hash verifier. "github.com/segmentio/go-prompt" ) @@ -154,7 +156,7 @@ func IsDocker() bool { } // Log error, as we will not propagate it to caller - errorIf(err, "Error in docker check.") + logger.LogIf(context.Background(), err) return err == nil } @@ -184,7 +186,7 @@ func IsBOSH() bool { } // Log error, as we will not propagate it to caller - errorIf(err, "Error in BOSH check.") + logger.LogIf(context.Background(), err) return err == nil } @@ -199,7 +201,9 @@ func getHelmVersion(helmInfoFilePath string) string { // Log errors and return "" as Minio can be deployed // without Helm charts as well. if !os.IsNotExist(err) { - errorIf(err, "Unable to read %s", helmInfoFilePath) + reqInfo := (&logger.ReqInfo{}).AppendTags("helmInfoFilePath", helmInfoFilePath) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) } return "" } @@ -491,33 +495,33 @@ func mainUpdate(ctx *cli.Context) { quiet := ctx.Bool("quiet") || ctx.GlobalBool("quiet") if quiet { - log.EnableQuiet() + logger.EnableQuiet() } minioMode := "" updateMsg, sha256Hex, _, latestReleaseTime, err := getUpdateInfo(10*time.Second, minioMode) if err != nil { - log.Println(err) + logger.Println(err) os.Exit(-1) } // Nothing to update running the latest release. if updateMsg == "" { - log.Println(greenColorSprintf("You are already running the most recent version of ‘minio’.")) + logger.Println(greenColorSprintf("You are already running the most recent version of ‘minio’.")) os.Exit(0) } - log.Println(updateMsg) + logger.Println(updateMsg) // if the in-place update is disabled then we shouldn't ask the // user to update the binaries. if strings.Contains(updateMsg, minioReleaseURL) && !globalInplaceUpdateDisabled { var successMsg string successMsg, err = doUpdate(sha256Hex, latestReleaseTime, shouldUpdate(quiet, sha256Hex, latestReleaseTime)) if err != nil { - log.Println(err) + logger.Println(err) os.Exit(-1) } - log.Println(successMsg) + logger.Println(successMsg) os.Exit(1) } } diff --git a/cmd/utils.go b/cmd/utils.go index 27ec6eedf..7e892304e 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -44,7 +44,9 @@ import ( // Close Http tracing file. func stopHTTPTrace() { if globalHTTPTraceFile != nil { - errorIf(globalHTTPTraceFile.Close(), "Unable to close httpTraceFile %s", globalHTTPTraceFile.Name()) + reqInfo := (&logger.ReqInfo{}).AppendTags("traceFile", globalHTTPTraceFile.Name()) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, globalHTTPTraceFile.Close()) globalHTTPTraceFile = nil } } @@ -331,8 +333,8 @@ func newContext(r *http.Request, api string) context.Context { if prefix != "" { object = prefix } - - return logger.SetContext(context.Background(), &logger.ReqInfo{r.RemoteAddr, r.Header.Get("user-agent"), "", api, bucket, object, nil}) + reqInfo := &logger.ReqInfo{RemoteHost: r.RemoteAddr, UserAgent: r.Header.Get("user-agent"), API: api, BucketName: bucket, ObjectName: object} + return logger.SetReqInfo(context.Background(), reqInfo) } // isNetworkOrHostDown - if there was a network error or if the host is down. diff --git a/cmd/web-handlers.go b/cmd/web-handlers.go index 88ed6c2a5..630c3552f 100644 --- a/cmd/web-handlers.go +++ b/cmd/web-handlers.go @@ -35,6 +35,7 @@ import ( "github.com/gorilla/rpc/v2/json2" "github.com/minio/minio-go/pkg/policy" "github.com/minio/minio/browser" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/event" @@ -384,7 +385,9 @@ func (web *webAPIHandlers) Login(r *http.Request, args *LoginArgs, reply *LoginR if err != nil { // Make sure to log errors related to browser login, // for security and auditing reasons. - errorIf(err, "Unable to login request from %s", r.RemoteAddr) + reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", r.RemoteAddr) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) return toJSONError(err) } @@ -463,7 +466,7 @@ func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *Se reply.PeerErrMsgs = make(map[string]string) for svr, errVal := range errsMap { tErr := fmt.Errorf("Unable to change credentials on %s: %v", svr, errVal) - errorIf(tErr, "Credentials change could not be propagated successfully!") + logger.LogIf(context.Background(), tErr) reply.PeerErrMsgs[svr] = errVal.Error() } @@ -571,7 +574,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) { } // Extract incoming metadata if any. - metadata, err := extractMetadataFromHeader(r.Header) + metadata, err := extractMetadataFromHeader(context.Background(), r.Header) if err != nil { writeErrorResponse(w, ErrInternalError, r.URL) return @@ -1095,7 +1098,7 @@ func toWebAPIError(err error) APIError { } // Log unexpected and unhandled errors. - errorIf(err, errUnexpected.Error()) + logger.LogIf(context.Background(), err) return APIError{ Code: "InternalError", HTTPStatusCode: http.StatusInternalServerError, diff --git a/cmd/web-handlers_test.go b/cmd/web-handlers_test.go index 21f0ec0f0..7bd1c93c9 100644 --- a/cmd/web-handlers_test.go +++ b/cmd/web-handlers_test.go @@ -1286,7 +1286,7 @@ func testWebGetBucketPolicyHandler(obj ObjectLayer, instanceType string, t TestE }, }, } - if err := writeBucketPolicy(bucketName, obj, policyVal); err != nil { + if err := writeBucketPolicy(context.Background(), bucketName, obj, policyVal); err != nil { t.Fatal("Unexpected error: ", err) } @@ -1380,7 +1380,7 @@ func testWebListAllBucketPoliciesHandler(obj ObjectLayer, instanceType string, t }, }, } - if err := writeBucketPolicy(bucketName, obj, policyVal); err != nil { + if err := writeBucketPolicy(context.Background(), bucketName, obj, policyVal); err != nil { t.Fatal("Unexpected error: ", err) } diff --git a/cmd/xl-sets.go b/cmd/xl-sets.go index 34bc2644b..e86e52b04 100644 --- a/cmd/xl-sets.go +++ b/cmd/xl-sets.go @@ -28,6 +28,7 @@ import ( "time" "github.com/minio/minio-go/pkg/policy" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/bpool" "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/hash" @@ -214,7 +215,7 @@ func newXLSets(endpoints EndpointList, format *formatXLV3, setCount int, drivesP nsMutex: mutex, bp: bpool.NewBytePoolCap(setCount*drivesPerSet, blockSizeV1, blockSizeV1*2), } - go s.sets[i].cleanupStaleMultipartUploads(globalMultipartCleanupInterval, globalMultipartExpiry, globalServiceDoneCh) + go s.sets[i].cleanupStaleMultipartUploads(context.Background(), globalMultipartCleanupInterval, globalMultipartExpiry, globalServiceDoneCh) } // Connect disks right away. @@ -431,7 +432,7 @@ func (s *xlSets) ListObjectsV2(ctx context.Context, bucket, prefix, continuation // SetBucketPolicy persist the new policy on the bucket. func (s *xlSets) SetBucketPolicy(ctx context.Context, bucket string, policy policy.BucketAccessPolicy) error { - return persistAndNotifyBucketPolicyChange(bucket, false, policy, s) + return persistAndNotifyBucketPolicyChange(ctx, bucket, false, policy, s) } // GetBucketPolicy will return a policy on a bucket @@ -446,7 +447,7 @@ func (s *xlSets) GetBucketPolicy(ctx context.Context, bucket string) (policy.Buc // DeleteBucketPolicy deletes all policies on bucket func (s *xlSets) DeleteBucketPolicy(ctx context.Context, bucket string) error { - return persistAndNotifyBucketPolicyChange(bucket, true, emptyBucketPolicy, s) + return persistAndNotifyBucketPolicyChange(ctx, bucket, true, emptyBucketPolicy, s) } // RefreshBucketPolicy refreshes policy cache from disk @@ -498,7 +499,7 @@ func (s *xlSets) DeleteBucket(ctx context.Context, bucket string) error { } // Delete all bucket metadata. - deleteBucketMetadata(bucket, s) + deleteBucketMetadata(ctx, bucket, s) // Success. return nil @@ -585,26 +586,26 @@ func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, destBucke } go func() { - if gerr := srcSet.getObject(srcBucket, srcObject, 0, srcInfo.Size, srcInfo.Writer, srcInfo.ETag); gerr != nil { + if gerr := srcSet.getObject(ctx, srcBucket, srcObject, 0, srcInfo.Size, srcInfo.Writer, srcInfo.ETag); gerr != nil { if gerr = srcInfo.Writer.Close(); gerr != nil { - errorIf(gerr, "Unable to read the object %s/%s.", srcBucket, srcObject) + logger.LogIf(ctx, gerr) } return } // Close writer explicitly signalling we wrote all data. if gerr := srcInfo.Writer.Close(); gerr != nil { - errorIf(gerr, "Unable to read the object %s/%s.", srcBucket, srcObject) + logger.LogIf(ctx, gerr) return } }() - return destSet.putObject(destBucket, destObject, srcInfo.Reader, srcInfo.UserDefined) + return destSet.putObject(ctx, destBucket, destObject, srcInfo.Reader, srcInfo.UserDefined) } // Returns function "listDir" of the type listDirFunc. // isLeaf - is used by listDir function to check if an entry is a leaf or non-leaf entry. // disks - used for doing disk.ListDir(). Sets passes set of disks. -func listDirSetsFactory(isLeaf isLeafFunc, treeWalkIgnoredErrs []error, sets ...[]StorageAPI) listDirFunc { +func listDirSetsFactory(ctx context.Context, isLeaf isLeafFunc, treeWalkIgnoredErrs []error, sets ...[]StorageAPI) listDirFunc { listDirInternal := func(bucket, prefixDir, prefixEntry string, disks []StorageAPI) (mergedEntries []string, err error) { for _, disk := range disks { if disk == nil { @@ -620,7 +621,8 @@ func listDirSetsFactory(isLeaf isLeafFunc, treeWalkIgnoredErrs []error, sets ... if errors.IsErrIgnored(err, treeWalkIgnoredErrs...) { continue } - return nil, errors.Trace(err) + logger.LogIf(ctx, err) + return nil, err } // Find elements in entries which are not in mergedEntries @@ -679,7 +681,7 @@ func listDirSetsFactory(isLeaf isLeafFunc, treeWalkIgnoredErrs []error, sets ... // value through the walk channel receives the data properly lexically sorted. func (s *xlSets) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) { // validate all the inputs for listObjects - if err = checkListObjsArgs(bucket, prefix, marker, delimiter, s); err != nil { + if err = checkListObjsArgs(ctx, bucket, prefix, marker, delimiter, s); err != nil { return result, err } @@ -707,8 +709,8 @@ func (s *xlSets) ListObjects(ctx context.Context, bucket, prefix, marker, delimi setDisks = append(setDisks, set.getLoadBalancedDisks()) } - listDir := listDirSetsFactory(isLeaf, xlTreeWalkIgnoredErrs, setDisks...) - walkResultCh = startTreeWalk(bucket, prefix, marker, recursive, listDir, isLeaf, endWalkCh) + listDir := listDirSetsFactory(ctx, isLeaf, xlTreeWalkIgnoredErrs, setDisks...) + walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, endWalkCh) } for i := 0; i < maxKeys; { @@ -726,9 +728,9 @@ func (s *xlSets) ListObjects(ctx context.Context, bucket, prefix, marker, delimi var objInfo ObjectInfo var err error if hasSuffix(walkResult.entry, slashSeparator) { - objInfo, err = s.getHashedSet(walkResult.entry).getObjectInfoDir(bucket, walkResult.entry) + objInfo, err = s.getHashedSet(walkResult.entry).getObjectInfoDir(ctx, bucket, walkResult.entry) } else { - objInfo, err = s.getHashedSet(walkResult.entry).getObjectInfo(bucket, walkResult.entry) + objInfo, err = s.getHashedSet(walkResult.entry).getObjectInfo(ctx, bucket, walkResult.entry) } if err != nil { // Ignore errFileNotFound as the object might have got @@ -787,12 +789,12 @@ func (s *xlSets) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destB go func() { if gerr := srcSet.GetObject(ctx, srcBucket, srcObject, startOffset, length, srcInfo.Writer, srcInfo.ETag); gerr != nil { if gerr = srcInfo.Writer.Close(); gerr != nil { - errorIf(gerr, "Unable to read %s of the object `%s/%s`.", srcBucket, srcObject) + logger.LogIf(ctx, gerr) return } } if gerr := srcInfo.Writer.Close(); gerr != nil { - errorIf(gerr, "Unable to read %s of the object `%s/%s`.", srcBucket, srcObject) + logger.LogIf(ctx, gerr) return } }() @@ -1034,7 +1036,7 @@ func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResult } // Save formats `format.json` across all disks. - if err = saveFormatXLAll(storageDisks, tmpNewFormats); err != nil { + if err = saveFormatXLAll(ctx, storageDisks, tmpNewFormats); err != nil { return madmin.HealResultItem{}, err } @@ -1228,7 +1230,7 @@ func listDirSetsHealFactory(isLeaf isLeafFunc, sets ...[]StorageAPI) listDirFunc } // listObjectsHeal - wrapper function implemented over file tree walk. -func (s *xlSets) listObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) { +func (s *xlSets) listObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) { // Default is recursive, if delimiter is set then list non recursive. recursive := true if delimiter == slashSeparator { @@ -1252,7 +1254,7 @@ func (s *xlSets) listObjectsHeal(bucket, prefix, marker, delimiter string, maxKe } listDir := listDirSetsHealFactory(isLeaf, setDisks...) - walkResultCh = startTreeWalk(bucket, prefix, marker, recursive, listDir, nil, endWalkCh) + walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, nil, endWalkCh) } var objInfos []ObjectInfo @@ -1272,9 +1274,9 @@ func (s *xlSets) listObjectsHeal(bucket, prefix, marker, delimiter string, maxKe var objInfo ObjectInfo var err error if hasSuffix(walkResult.entry, slashSeparator) { - objInfo, err = s.getHashedSet(walkResult.entry).getObjectInfoDir(bucket, walkResult.entry) + objInfo, err = s.getHashedSet(walkResult.entry).getObjectInfoDir(ctx, bucket, walkResult.entry) } else { - objInfo, err = s.getHashedSet(walkResult.entry).getObjectInfo(bucket, walkResult.entry) + objInfo, err = s.getHashedSet(walkResult.entry).getObjectInfo(ctx, bucket, walkResult.entry) } if err != nil { // Ignore errFileNotFound @@ -1320,7 +1322,7 @@ func (s *xlSets) listObjectsHeal(bucket, prefix, marker, delimiter string, maxKe // This is not implemented yet, will be implemented later to comply with Admin API refactor. func (s *xlSets) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) { - if err = checkListObjsArgs(bucket, prefix, marker, delimiter, s); err != nil { + if err = checkListObjsArgs(ctx, bucket, prefix, marker, delimiter, s); err != nil { return loi, err } @@ -1343,7 +1345,7 @@ func (s *xlSets) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, de } // Initiate a list operation, if successful filter and return quickly. - listObjInfo, err := s.listObjectsHeal(bucket, prefix, marker, delimiter, maxKeys) + listObjInfo, err := s.listObjectsHeal(ctx, bucket, prefix, marker, delimiter, maxKeys) if err == nil { // We got the entries successfully return. return listObjInfo, nil diff --git a/cmd/xl-sets_test.go b/cmd/xl-sets_test.go index 988d5841e..eea961ff0 100644 --- a/cmd/xl-sets_test.go +++ b/cmd/xl-sets_test.go @@ -77,18 +77,18 @@ func TestNewXLSets(t *testing.T) { } endpoints := mustGetNewEndpointList(erasureDisks...) - _, err := waitForFormatXL(true, endpoints, 0, 16) + _, err := waitForFormatXL(context.Background(), true, endpoints, 0, 16) if err != errInvalidArgument { t.Fatalf("Expecting error, got %s", err) } - _, err = waitForFormatXL(true, nil, 1, 16) + _, err = waitForFormatXL(context.Background(), true, nil, 1, 16) if err != errInvalidArgument { t.Fatalf("Expecting error, got %s", err) } // Initializes all erasure disks - format, err := waitForFormatXL(true, endpoints, 1, 16) + format, err := waitForFormatXL(context.Background(), true, endpoints, 1, 16) if err != nil { t.Fatalf("Unable to format disks for erasure, %s", err) } @@ -113,7 +113,7 @@ func TestStorageInfoSets(t *testing.T) { endpoints := mustGetNewEndpointList(erasureDisks...) // Initializes all erasure disks - format, err := waitForFormatXL(true, endpoints, 1, 16) + format, err := waitForFormatXL(context.Background(), true, endpoints, 1, 16) if err != nil { t.Fatalf("Unable to format disks for erasure, %s", err) } diff --git a/cmd/xl-v1-bucket.go b/cmd/xl-v1-bucket.go index cde8709a2..f9f7621f2 100644 --- a/cmd/xl-v1-bucket.go +++ b/cmd/xl-v1-bucket.go @@ -23,6 +23,7 @@ import ( "sync" "github.com/minio/minio-go/pkg/policy" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/errors" ) @@ -38,7 +39,8 @@ var bucketMetadataOpIgnoredErrs = append(bucketOpIgnoredErrs, errVolumeNotFound) func (xl xlObjects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return errors.Trace(BucketNameInvalid{Bucket: bucket}) + logger.LogIf(ctx, BucketNameInvalid{Bucket: bucket}) + return BucketNameInvalid{Bucket: bucket} } // Initialize sync waitgroup. @@ -50,7 +52,8 @@ func (xl xlObjects) MakeBucketWithLocation(ctx context.Context, bucket, location // Make a volume entry on all underlying storage disks. for index, disk := range xl.getDisks() { if disk == nil { - dErrs[index] = errors.Trace(errDiskNotFound) + logger.LogIf(ctx, errDiskNotFound) + dErrs[index] = errDiskNotFound continue } wg.Add(1) @@ -59,7 +62,10 @@ func (xl xlObjects) MakeBucketWithLocation(ctx context.Context, bucket, location defer wg.Done() err := disk.MakeVol(bucket) if err != nil { - dErrs[index] = errors.Trace(err) + if err != errVolumeExists { + logger.LogIf(ctx, err) + } + dErrs[index] = err } }(index, disk) } @@ -68,7 +74,7 @@ func (xl xlObjects) MakeBucketWithLocation(ctx context.Context, bucket, location wg.Wait() writeQuorum := len(xl.getDisks())/2 + 1 - err := reduceWriteQuorumErrs(dErrs, bucketOpIgnoredErrs, writeQuorum) + err := reduceWriteQuorumErrs(ctx, dErrs, bucketOpIgnoredErrs, writeQuorum) if errors.Cause(err) == errXLWriteQuorum { // Purge successfully created buckets if we don't have writeQuorum. undoMakeBucket(xl.getDisks(), bucket) @@ -118,7 +124,7 @@ func undoMakeBucket(storageDisks []StorageAPI, bucket string) { } // getBucketInfo - returns the BucketInfo from one of the load balanced disks. -func (xl xlObjects) getBucketInfo(bucketName string) (bucketInfo BucketInfo, err error) { +func (xl xlObjects) getBucketInfo(ctx context.Context, bucketName string) (bucketInfo BucketInfo, err error) { var bucketErrs []error for _, disk := range xl.getLoadBalancedDisks() { if disk == nil { @@ -133,7 +139,8 @@ func (xl xlObjects) getBucketInfo(bucketName string) (bucketInfo BucketInfo, err } return bucketInfo, nil } - err = errors.Trace(serr) + logger.LogIf(ctx, serr) + err = serr // For any reason disk went offline continue and pick the next one. if errors.IsErrIgnored(err, bucketMetadataOpIgnoredErrs...) { bucketErrs = append(bucketErrs, err) @@ -147,7 +154,7 @@ func (xl xlObjects) getBucketInfo(bucketName string) (bucketInfo BucketInfo, err // `nil` is deliberately passed for ignoredErrs // because these errors were already ignored. readQuorum := len(xl.getDisks()) / 2 - return BucketInfo{}, reduceReadQuorumErrs(bucketErrs, nil, readQuorum) + return BucketInfo{}, reduceReadQuorumErrs(ctx, bucketErrs, nil, readQuorum) } // GetBucketInfo - returns BucketInfo for a bucket. @@ -162,7 +169,7 @@ func (xl xlObjects) GetBucketInfo(ctx context.Context, bucket string) (bi Bucket return bi, BucketNameInvalid{Bucket: bucket} } - bucketInfo, err := xl.getBucketInfo(bucket) + bucketInfo, err := xl.getBucketInfo(ctx, bucket) if err != nil { return bi, toObjectErr(err, bucket) } @@ -170,7 +177,7 @@ func (xl xlObjects) GetBucketInfo(ctx context.Context, bucket string) (bi Bucket } // listBuckets - returns list of all buckets from a disk picked at random. -func (xl xlObjects) listBuckets() (bucketsInfo []BucketInfo, err error) { +func (xl xlObjects) listBuckets(ctx context.Context) (bucketsInfo []BucketInfo, err error) { for _, disk := range xl.getLoadBalancedDisks() { if disk == nil { continue @@ -199,7 +206,7 @@ func (xl xlObjects) listBuckets() (bucketsInfo []BucketInfo, err error) { } return bucketsInfo, nil } - err = errors.Trace(err) + logger.LogIf(ctx, err) // Ignore any disks not found. if errors.IsErrIgnored(err, bucketMetadataOpIgnoredErrs...) { continue @@ -211,7 +218,7 @@ func (xl xlObjects) listBuckets() (bucketsInfo []BucketInfo, err error) { // ListBuckets - lists all the buckets, sorted by its name. func (xl xlObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) { - bucketInfos, err := xl.listBuckets() + bucketInfos, err := xl.listBuckets(ctx) if err != nil { return nil, toObjectErr(err) } @@ -241,7 +248,8 @@ func (xl xlObjects) DeleteBucket(ctx context.Context, bucket string) error { // Remove a volume entry on all underlying storage disks. for index, disk := range xl.getDisks() { if disk == nil { - dErrs[index] = errors.Trace(errDiskNotFound) + logger.LogIf(ctx, errDiskNotFound) + dErrs[index] = errDiskNotFound continue } wg.Add(1) @@ -252,11 +260,12 @@ func (xl xlObjects) DeleteBucket(ctx context.Context, bucket string) error { err := disk.DeleteVol(bucket) if err != nil { - dErrs[index] = errors.Trace(err) + logger.LogIf(ctx, err) + dErrs[index] = err return } // Cleanup all the previously incomplete multiparts. - err = cleanupDir(disk, minioMetaMultipartBucket, bucket) + err = cleanupDir(ctx, disk, minioMetaMultipartBucket, bucket) if err != nil { if errors.Cause(err) == errVolumeNotFound { @@ -271,7 +280,7 @@ func (xl xlObjects) DeleteBucket(ctx context.Context, bucket string) error { wg.Wait() writeQuorum := len(xl.getDisks())/2 + 1 - err := reduceWriteQuorumErrs(dErrs, bucketOpIgnoredErrs, writeQuorum) + err := reduceWriteQuorumErrs(ctx, dErrs, bucketOpIgnoredErrs, writeQuorum) if errors.Cause(err) == errXLWriteQuorum { xl.undoDeleteBucket(bucket) } @@ -284,7 +293,7 @@ func (xl xlObjects) DeleteBucket(ctx context.Context, bucket string) error { // SetBucketPolicy sets policy on bucket func (xl xlObjects) SetBucketPolicy(ctx context.Context, bucket string, policy policy.BucketAccessPolicy) error { - return persistAndNotifyBucketPolicyChange(bucket, false, policy, xl) + return persistAndNotifyBucketPolicyChange(ctx, bucket, false, policy, xl) } // GetBucketPolicy will get policy on bucket @@ -299,7 +308,7 @@ func (xl xlObjects) GetBucketPolicy(ctx context.Context, bucket string) (policy. // DeleteBucketPolicy deletes all policies on bucket func (xl xlObjects) DeleteBucketPolicy(ctx context.Context, bucket string) error { - return persistAndNotifyBucketPolicyChange(bucket, true, emptyBucketPolicy, xl) + return persistAndNotifyBucketPolicyChange(ctx, bucket, true, emptyBucketPolicy, xl) } // RefreshBucketPolicy refreshes policy cache from disk diff --git a/cmd/xl-v1-common.go b/cmd/xl-v1-common.go index b044a70bb..3d28c0ac8 100644 --- a/cmd/xl-v1-common.go +++ b/cmd/xl-v1-common.go @@ -17,8 +17,10 @@ package cmd import ( + "context" "path" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/errors" ) @@ -34,7 +36,7 @@ func (xl xlObjects) getLoadBalancedDisks() (disks []StorageAPI) { // This function does the following check, suppose // object is "a/b/c/d", stat makes sure that objects ""a/b/c"" // "a/b" and "a" do not exist. -func (xl xlObjects) parentDirIsObject(bucket, parent string) bool { +func (xl xlObjects) parentDirIsObject(ctx context.Context, bucket, parent string) bool { var isParentDirObject func(string) bool isParentDirObject = func(p string) bool { if p == "." || p == "/" { @@ -66,7 +68,11 @@ func (xl xlObjects) isObject(bucket, prefix string) (ok bool) { if errors.IsErrIgnored(err, xlTreeWalkIgnoredErrs...) { continue } - errorIf(err, "Unable to stat a file %s/%s/%s", bucket, prefix, xlMetaJSONFile) + reqInfo := &logger.ReqInfo{BucketName: bucket} + reqInfo.AppendTags("prefix", prefix) + reqInfo.AppendTags("xlMetaJSONFile", xlMetaJSONFile) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) } // Exhausted all disks - return false. return false } diff --git a/cmd/xl-v1-common_test.go b/cmd/xl-v1-common_test.go index b72d669db..97e3dae98 100644 --- a/cmd/xl-v1-common_test.go +++ b/cmd/xl-v1-common_test.go @@ -92,7 +92,7 @@ func TestXLParentDirIsObject(t *testing.T) { } for i, testCase := range testCases { - gotValue := fs.parentDirIsObject(bucketName, testCase.objectName) + gotValue := fs.parentDirIsObject(context.Background(), bucketName, testCase.objectName) if testCase.parentIsObject != gotValue { t.Errorf("Test %d: Unexpected value returned got %t, expected %t", i+1, gotValue, testCase.parentIsObject) } diff --git a/cmd/xl-v1-healing-common.go b/cmd/xl-v1-healing-common.go index 414be858e..f0c8077df 100644 --- a/cmd/xl-v1-healing-common.go +++ b/cmd/xl-v1-healing-common.go @@ -17,10 +17,11 @@ package cmd import ( + "context" "path/filepath" "time" - "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/cmd/logger" ) // commonTime returns a maximally occurring time from a list of time. @@ -150,7 +151,7 @@ func getLatestXLMeta(partsMetadata []xlMetaV1, errs []error) (xlMetaV1, int) { // // - non-nil error if any of the disks failed unexpectedly (i.e. error // other than file not found and not a checksum error). -func disksWithAllParts(onlineDisks []StorageAPI, partsMetadata []xlMetaV1, errs []error, bucket, +func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []xlMetaV1, errs []error, bucket, object string) ([]StorageAPI, []error, error) { availableDisks := make([]StorageAPI, len(onlineDisks)) @@ -182,8 +183,9 @@ func disksWithAllParts(onlineDisks []StorageAPI, partsMetadata []xlMetaV1, errs dataErrs[i] = hErr break case hErr != nil: + logger.LogIf(ctx, hErr) // abort on unhandled errors - return nil, nil, errors.Trace(hErr) + return nil, nil, hErr } } diff --git a/cmd/xl-v1-healing-common_test.go b/cmd/xl-v1-healing-common_test.go index 4ae2a7894..9a581b235 100644 --- a/cmd/xl-v1-healing-common_test.go +++ b/cmd/xl-v1-healing-common_test.go @@ -213,7 +213,7 @@ func TestListOnlineDisks(t *testing.T) { } // Fetch xl.json from first disk to construct partsMetadata for the tests. - xlMeta, err := readXLMeta(xlDisks[0], bucket, object) + xlMeta, err := readXLMeta(context.Background(), xlDisks[0], bucket, object) if err != nil { t.Fatalf("Test %d: Failed to read xl.json %v", i+1, err) } @@ -265,7 +265,7 @@ func TestListOnlineDisks(t *testing.T) { i+1, test.expectedTime, modTime) } - availableDisks, newErrs, _ := disksWithAllParts(onlineDisks, partsMetadata, test.errs, bucket, object) + availableDisks, newErrs, _ := disksWithAllParts(context.Background(), onlineDisks, partsMetadata, test.errs, bucket, object) test.errs = newErrs if test._tamperBackend != noTamper { @@ -279,6 +279,7 @@ func TestListOnlineDisks(t *testing.T) { } func TestDisksWithAllParts(t *testing.T) { + ctx := context.Background() rootPath, err := newTestConfig(globalMinioDefaultRegion) if err != nil { t.Fatalf("Failed to initialize config - %v", err) @@ -299,19 +300,19 @@ func TestDisksWithAllParts(t *testing.T) { xl := obj.(*xlObjects) xlDisks := xl.storageDisks - err = obj.MakeBucketWithLocation(context.Background(), "bucket", "") + err = obj.MakeBucketWithLocation(ctx, "bucket", "") if err != nil { t.Fatalf("Failed to make a bucket %v", err) } - _, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil) + _, err = obj.PutObject(ctx, bucket, object, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil) if err != nil { t.Fatalf("Failed to putObject %v", err) } - partsMetadata, errs := readAllXLMetadata(xlDisks, bucket, object) + partsMetadata, errs := readAllXLMetadata(ctx, xlDisks, bucket, object) readQuorum := len(xl.storageDisks) / 2 - if reducedErr := reduceReadQuorumErrs(errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil { + if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil { t.Fatalf("Failed to read xl meta data %v", reducedErr) } @@ -330,7 +331,7 @@ func TestDisksWithAllParts(t *testing.T) { } errs = make([]error, len(xlDisks)) - filteredDisks, errs, err := disksWithAllParts(xlDisks, partsMetadata, errs, bucket, object) + filteredDisks, errs, err := disksWithAllParts(ctx, xlDisks, partsMetadata, errs, bucket, object) if err != nil { t.Errorf("Unexpected error: %s", err) } @@ -360,12 +361,12 @@ func TestDisksWithAllParts(t *testing.T) { // Test that all disks are returned without any failures with // unmodified meta data - partsMetadata, errs = readAllXLMetadata(xlDisks, bucket, object) + partsMetadata, errs = readAllXLMetadata(ctx, xlDisks, bucket, object) if err != nil { t.Fatalf("Failed to read xl meta data %v", err) } - filteredDisks, errs, err = disksWithAllParts(xlDisks, partsMetadata, errs, bucket, object) + filteredDisks, errs, err = disksWithAllParts(ctx, xlDisks, partsMetadata, errs, bucket, object) if err != nil { t.Errorf("Unexpected error: %s", err) } diff --git a/cmd/xl-v1-healing.go b/cmd/xl-v1-healing.go index 6f1c679a5..7614cc5e3 100644 --- a/cmd/xl-v1-healing.go +++ b/cmd/xl-v1-healing.go @@ -22,17 +22,19 @@ import ( "path" "sync" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/madmin" ) func (xl xlObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) { - return madmin.HealResultItem{}, errors.Trace(NotImplemented{}) + logger.LogIf(ctx, NotImplemented{}) + return madmin.HealResultItem{}, NotImplemented{} } // checks for bucket if it exists in writeQuorum number of disks, this call // is only used by healBucket(). -func checkBucketExistsInQuorum(storageDisks []StorageAPI, bucketName string) (err error) { +func checkBucketExistsInQuorum(ctx context.Context, storageDisks []StorageAPI, bucketName string) (err error) { var wg = &sync.WaitGroup{} errs := make([]error, len(storageDisks)) @@ -50,7 +52,7 @@ func checkBucketExistsInQuorum(storageDisks []StorageAPI, bucketName string) (er wg.Wait() readQuorum := len(storageDisks) / 2 - return reduceWriteQuorumErrs(errs, nil, readQuorum) + return reduceWriteQuorumErrs(ctx, errs, nil, readQuorum) } // Heals a bucket if it doesn't exist on one of the disks, additionally @@ -64,7 +66,7 @@ func (xl xlObjects) HealBucket(ctx context.Context, bucket string, dryRun bool) // Check if bucket doesn't exist in writeQuorum number of disks, if quorum // number of disks returned that bucket does not exist we quickly return // and do not proceed to heal. - if err = checkBucketExistsInQuorum(storageDisks, bucket); err != nil { + if err = checkBucketExistsInQuorum(ctx, storageDisks, bucket); err != nil { return results, err } @@ -73,7 +75,7 @@ func (xl xlObjects) HealBucket(ctx context.Context, bucket string, dryRun bool) // Heal bucket. var result madmin.HealResultItem - result, err = healBucket(storageDisks, bucket, writeQuorum, dryRun) + result, err = healBucket(ctx, storageDisks, bucket, writeQuorum, dryRun) if err != nil { return nil, err } @@ -86,7 +88,7 @@ func (xl xlObjects) HealBucket(ctx context.Context, bucket string, dryRun bool) } // Heal bucket - create buckets on disks where it does not exist. -func healBucket(storageDisks []StorageAPI, bucket string, writeQuorum int, +func healBucket(ctx context.Context, storageDisks []StorageAPI, bucket string, writeQuorum int, dryRun bool) (res madmin.HealResultItem, err error) { // Initialize sync waitgroup. @@ -102,7 +104,8 @@ func healBucket(storageDisks []StorageAPI, bucket string, writeQuorum int, // Make a volume entry on all underlying storage disks. for index, disk := range storageDisks { if disk == nil { - dErrs[index] = errors.Trace(errDiskNotFound) + logger.LogIf(ctx, errDiskNotFound) + dErrs[index] = errDiskNotFound beforeState[index] = madmin.DriveStateOffline afterState[index] = madmin.DriveStateOffline continue @@ -182,7 +185,7 @@ func healBucket(storageDisks []StorageAPI, bucket string, writeQuorum int, }) } - reducedErr := reduceWriteQuorumErrs(dErrs, bucketOpIgnoredErrs, writeQuorum) + reducedErr := reduceWriteQuorumErrs(ctx, dErrs, bucketOpIgnoredErrs, writeQuorum) if errors.Cause(reducedErr) == errXLWriteQuorum { // Purge successfully created buckets if we don't have writeQuorum. undoMakeBucket(storageDisks, bucket) @@ -196,7 +199,9 @@ func healBucketMetadata(xl xlObjects, bucket string, dryRun bool) ( results []madmin.HealResultItem, err error) { healBucketMetaFn := func(metaPath string) error { - result, healErr := xl.HealObject(context.Background(), minioMetaBucket, metaPath, dryRun) + reqInfo := &logger.ReqInfo{BucketName: bucket} + ctx := logger.SetReqInfo(context.Background(), reqInfo) + result, healErr := xl.HealObject(ctx, minioMetaBucket, metaPath, dryRun) // If object is not found, no result to add. if isErrObjectNotFound(healErr) { return nil @@ -270,15 +275,15 @@ func listAllBuckets(storageDisks []StorageAPI) (buckets map[string]VolInfo, } // Heals an object by re-writing corrupt/missing erasure blocks. -func healObject(storageDisks []StorageAPI, bucket string, object string, +func healObject(ctx context.Context, storageDisks []StorageAPI, bucket string, object string, quorum int, dryRun bool) (result madmin.HealResultItem, err error) { - partsMetadata, errs := readAllXLMetadata(storageDisks, bucket, object) + partsMetadata, errs := readAllXLMetadata(ctx, storageDisks, bucket, object) // readQuorum suffices for xl.json since we use monotonic // system time to break the tie when a split-brain situation // arises. - if reducedErr := reduceReadQuorumErrs(errs, nil, quorum); reducedErr != nil { + if reducedErr := reduceReadQuorumErrs(ctx, errs, nil, quorum); reducedErr != nil { return result, toObjectErr(reducedErr, bucket, object) } @@ -287,7 +292,7 @@ func healObject(storageDisks []StorageAPI, bucket string, object string, latestDisks, modTime := listOnlineDisks(storageDisks, partsMetadata, errs) // List of disks having all parts as per latest xl.json. - availableDisks, dataErrs, aErr := disksWithAllParts(latestDisks, partsMetadata, errs, bucket, object) + availableDisks, dataErrs, aErr := disksWithAllParts(ctx, latestDisks, partsMetadata, errs, bucket, object) if aErr != nil { return result, toObjectErr(aErr, bucket, object) } @@ -387,7 +392,7 @@ func healObject(storageDisks []StorageAPI, bucket string, object string, // Latest xlMetaV1 for reference. If a valid metadata is not // present, it is as good as object not found. - latestMeta, pErr := pickValidXLMeta(partsMetadata, modTime) + latestMeta, pErr := pickValidXLMeta(ctx, partsMetadata, modTime) if pErr != nil { return result, toObjectErr(pErr, bucket, object) } @@ -430,7 +435,7 @@ func healObject(storageDisks []StorageAPI, bucket string, object string, // Heal each part. erasureHealFile() will write the healed // part to .minio/tmp/uuid/ which needs to be renamed later to // the final location. - storage, err := NewErasureStorage(latestDisks, latestMeta.Erasure.DataBlocks, + storage, err := NewErasureStorage(ctx, latestDisks, latestMeta.Erasure.DataBlocks, latestMeta.Erasure.ParityBlocks, latestMeta.Erasure.BlockSize) if err != nil { return result, toObjectErr(err, bucket, object) @@ -449,7 +454,7 @@ func healObject(storageDisks []StorageAPI, bucket string, object string, } } // Heal the part file. - file, hErr := storage.HealFile(outDatedDisks, bucket, pathJoin(object, partName), + file, hErr := storage.HealFile(ctx, outDatedDisks, bucket, pathJoin(object, partName), erasure.BlockSize, minioMetaTmpBucket, pathJoin(tmpID, partName), partSize, algorithm, checksums) if hErr != nil { @@ -489,7 +494,7 @@ func healObject(storageDisks []StorageAPI, bucket string, object string, } // Generate and write `xl.json` generated from other disks. - outDatedDisks, aErr = writeUniqueXLMetadata(outDatedDisks, minioMetaTmpBucket, tmpID, + outDatedDisks, aErr = writeUniqueXLMetadata(ctx, outDatedDisks, minioMetaTmpBucket, tmpID, partsMetadata, diskCount(outDatedDisks)) if aErr != nil { return result, toObjectErr(aErr, bucket, object) @@ -505,7 +510,8 @@ func healObject(storageDisks []StorageAPI, bucket string, object string, aErr = disk.RenameFile(minioMetaTmpBucket, retainSlash(tmpID), bucket, retainSlash(object)) if aErr != nil { - return result, toObjectErr(errors.Trace(aErr), bucket, object) + logger.LogIf(ctx, aErr) + return result, toObjectErr(aErr, bucket, object) } for i, v := range result.Before.Drives { @@ -530,7 +536,7 @@ func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRu // FIXME: Metadata is read again in the healObject() call below. // Read metadata files from all the disks - partsMetadata, errs := readAllXLMetadata(xl.getDisks(), bucket, object) + partsMetadata, errs := readAllXLMetadata(ctx, xl.getDisks(), bucket, object) // get read quorum for this object var readQuorum int @@ -547,5 +553,5 @@ func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRu defer objectLock.RUnlock() // Heal the object. - return healObject(xl.getDisks(), bucket, object, readQuorum, dryRun) + return healObject(ctx, xl.getDisks(), bucket, object, readQuorum, dryRun) } diff --git a/cmd/xl-v1-list-objects.go b/cmd/xl-v1-list-objects.go index ce961fbac..d430bccd7 100644 --- a/cmd/xl-v1-list-objects.go +++ b/cmd/xl-v1-list-objects.go @@ -26,7 +26,7 @@ import ( // Returns function "listDir" of the type listDirFunc. // isLeaf - is used by listDir function to check if an entry is a leaf or non-leaf entry. // disks - used for doing disk.ListDir() -func listDirFactory(isLeaf isLeafFunc, treeWalkIgnoredErrs []error, disks ...StorageAPI) listDirFunc { +func listDirFactory(ctx context.Context, isLeaf isLeafFunc, treeWalkIgnoredErrs []error, disks ...StorageAPI) listDirFunc { // Returns sorted merged entries from all the disks. listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool, err error) { for _, disk := range disks { @@ -42,7 +42,7 @@ func listDirFactory(isLeaf isLeafFunc, treeWalkIgnoredErrs []error, disks ...Sto if errors.IsErrIgnored(err, treeWalkIgnoredErrs...) { continue } - return nil, false, errors.Trace(err) + return nil, false, err } // Find elements in entries which are not in mergedEntries @@ -68,7 +68,7 @@ func listDirFactory(isLeaf isLeafFunc, treeWalkIgnoredErrs []error, disks ...Sto } // listObjects - wrapper function implemented over file tree walk. -func (xl xlObjects) listObjects(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) { +func (xl xlObjects) listObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) { // Default is recursive, if delimiter is set then list non recursive. recursive := true if delimiter == slashSeparator { @@ -80,8 +80,8 @@ func (xl xlObjects) listObjects(bucket, prefix, marker, delimiter string, maxKey if walkResultCh == nil { endWalkCh = make(chan struct{}) isLeaf := xl.isObject - listDir := listDirFactory(isLeaf, xlTreeWalkIgnoredErrs, xl.getLoadBalancedDisks()...) - walkResultCh = startTreeWalk(bucket, prefix, marker, recursive, listDir, isLeaf, endWalkCh) + listDir := listDirFactory(ctx, isLeaf, xlTreeWalkIgnoredErrs, xl.getLoadBalancedDisks()...) + walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, endWalkCh) } var objInfos []ObjectInfo @@ -109,7 +109,7 @@ func (xl xlObjects) listObjects(bucket, prefix, marker, delimiter string, maxKey } else { // Set the Mode to a "regular" file. var err error - objInfo, err = xl.getObjectInfo(bucket, entry) + objInfo, err = xl.getObjectInfo(ctx, bucket, entry) if err != nil { // Ignore errFileNotFound as the object might have got // deleted in the interim period of listing and getObjectInfo(), @@ -149,7 +149,7 @@ func (xl xlObjects) listObjects(bucket, prefix, marker, delimiter string, maxKey // ListObjects - list all objects at prefix, delimited by '/'. func (xl xlObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) { - if err := checkListObjsArgs(bucket, prefix, marker, delimiter, xl); err != nil { + if err := checkListObjsArgs(ctx, bucket, prefix, marker, delimiter, xl); err != nil { return loi, err } @@ -180,7 +180,7 @@ func (xl xlObjects) ListObjects(ctx context.Context, bucket, prefix, marker, del } // Initiate a list operation, if successful filter and return quickly. - listObjInfo, err := xl.listObjects(bucket, prefix, marker, delimiter, maxKeys) + listObjInfo, err := xl.listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys) if err == nil { // We got the entries successfully return. return listObjInfo, nil diff --git a/cmd/xl-v1-metadata.go b/cmd/xl-v1-metadata.go index 362b2166f..087a9e66d 100644 --- a/cmd/xl-v1-metadata.go +++ b/cmd/xl-v1-metadata.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "crypto" "encoding/hex" "encoding/json" @@ -28,6 +29,7 @@ import ( "time" "github.com/minio/highwayhash" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/errors" sha256 "github.com/minio/sha256-simd" "golang.org/x/crypto/blake2b" @@ -361,7 +363,7 @@ func (m *xlMetaV1) AddObjectPart(partNumber int, partName string, partETag strin } // ObjectToPartOffset - translate offset of an object to offset of its individual part. -func (m xlMetaV1) ObjectToPartOffset(offset int64) (partIndex int, partOffset int64, err error) { +func (m xlMetaV1) ObjectToPartOffset(ctx context.Context, offset int64) (partIndex int, partOffset int64, err error) { if offset == 0 { // Special case - if offset is 0, then partIndex and partOffset are always 0. return 0, 0, nil @@ -377,35 +379,38 @@ func (m xlMetaV1) ObjectToPartOffset(offset int64) (partIndex int, partOffset in // Continue to towards the next part. partOffset -= part.Size } + logger.LogIf(ctx, InvalidRange{}) // Offset beyond the size of the object return InvalidRange. - return 0, 0, errors.Trace(InvalidRange{}) + return 0, 0, InvalidRange{} } // pickValidXLMeta - picks one valid xlMeta content and returns from a // slice of xlmeta content. If no value is found this function panics // and dies. -func pickValidXLMeta(metaArr []xlMetaV1, modTime time.Time) (xmv xlMetaV1, e error) { +func pickValidXLMeta(ctx context.Context, metaArr []xlMetaV1, modTime time.Time) (xmv xlMetaV1, e error) { // Pick latest valid metadata. for _, meta := range metaArr { if meta.IsValid() && meta.Stat.ModTime.Equal(modTime) { return meta, nil } } - return xmv, errors.Trace(fmt.Errorf("No valid xl.json present")) + err := fmt.Errorf("No valid xl.json present") + logger.LogIf(ctx, err) + return xmv, err } // list of all errors that can be ignored in a metadata operation. var objMetadataOpIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied, errVolumeNotFound, errFileNotFound, errFileAccessDenied, errCorruptedFormat) // readXLMetaParts - returns the XL Metadata Parts from xl.json of one of the disks picked at random. -func (xl xlObjects) readXLMetaParts(bucket, object string) (xlMetaParts []objectPartInfo, xlMeta map[string]string, err error) { +func (xl xlObjects) readXLMetaParts(ctx context.Context, bucket, object string) (xlMetaParts []objectPartInfo, xlMeta map[string]string, err error) { var ignoredErrs []error for _, disk := range xl.getLoadBalancedDisks() { if disk == nil { ignoredErrs = append(ignoredErrs, errDiskNotFound) continue } - xlMetaParts, xlMeta, err = readXLMetaParts(disk, bucket, object) + xlMetaParts, xlMeta, err = readXLMetaParts(ctx, disk, bucket, object) if err == nil { return xlMetaParts, xlMeta, nil } @@ -421,11 +426,11 @@ func (xl xlObjects) readXLMetaParts(bucket, object string) (xlMetaParts []object // If all errors were ignored, reduce to maximal occurrence // based on the read quorum. readQuorum := len(xl.getDisks()) / 2 - return nil, nil, reduceReadQuorumErrs(ignoredErrs, nil, readQuorum) + return nil, nil, reduceReadQuorumErrs(ctx, ignoredErrs, nil, readQuorum) } // readXLMetaStat - return xlMetaV1.Stat and xlMetaV1.Meta from one of the disks picked at random. -func (xl xlObjects) readXLMetaStat(bucket, object string) (xlStat statInfo, xlMeta map[string]string, err error) { +func (xl xlObjects) readXLMetaStat(ctx context.Context, bucket, object string) (xlStat statInfo, xlMeta map[string]string, err error) { var ignoredErrs []error for _, disk := range xl.getLoadBalancedDisks() { if disk == nil { @@ -433,7 +438,7 @@ func (xl xlObjects) readXLMetaStat(bucket, object string) (xlStat statInfo, xlMe continue } // parses only xlMetaV1.Meta and xlMeta.Stat - xlStat, xlMeta, err = readXLMetaStat(disk, bucket, object) + xlStat, xlMeta, err = readXLMetaStat(ctx, disk, bucket, object) if err == nil { return xlStat, xlMeta, nil } @@ -449,30 +454,35 @@ func (xl xlObjects) readXLMetaStat(bucket, object string) (xlStat statInfo, xlMe // If all errors were ignored, reduce to maximal occurrence // based on the read quorum. readQuorum := len(xl.getDisks()) / 2 - return statInfo{}, nil, reduceReadQuorumErrs(ignoredErrs, nil, readQuorum) + return statInfo{}, nil, reduceReadQuorumErrs(ctx, ignoredErrs, nil, readQuorum) } // deleteXLMetadata - deletes `xl.json` on a single disk. -func deleteXLMetdata(disk StorageAPI, bucket, prefix string) error { +func deleteXLMetdata(ctx context.Context, disk StorageAPI, bucket, prefix string) error { jsonFile := path.Join(prefix, xlMetaJSONFile) - return errors.Trace(disk.DeleteFile(bucket, jsonFile)) + err := disk.DeleteFile(bucket, jsonFile) + logger.LogIf(ctx, err) + return err } // writeXLMetadata - writes `xl.json` to a single disk. -func writeXLMetadata(disk StorageAPI, bucket, prefix string, xlMeta xlMetaV1) error { +func writeXLMetadata(ctx context.Context, disk StorageAPI, bucket, prefix string, xlMeta xlMetaV1) error { jsonFile := path.Join(prefix, xlMetaJSONFile) // Marshal json. metadataBytes, err := json.Marshal(&xlMeta) if err != nil { - return errors.Trace(err) + logger.LogIf(ctx, err) + return err } // Persist marshalled data. - return errors.Trace(disk.AppendFile(bucket, jsonFile, metadataBytes)) + err = disk.AppendFile(bucket, jsonFile, metadataBytes) + logger.LogIf(ctx, err) + return err } // deleteAllXLMetadata - deletes all partially written `xl.json` depending on errs. -func deleteAllXLMetadata(disks []StorageAPI, bucket, prefix string, errs []error) { +func deleteAllXLMetadata(ctx context.Context, disks []StorageAPI, bucket, prefix string, errs []error) { var wg = &sync.WaitGroup{} // Delete all the `xl.json` left over. for index, disk := range disks { @@ -486,29 +496,30 @@ func deleteAllXLMetadata(disks []StorageAPI, bucket, prefix string, errs []error if errs[index] != nil { return } - _ = deleteXLMetdata(disk, bucket, prefix) + _ = deleteXLMetdata(ctx, disk, bucket, prefix) }(index, disk) } wg.Wait() } // Rename `xl.json` content to destination location for each disk in order. -func renameXLMetadata(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, quorum int) ([]StorageAPI, error) { +func renameXLMetadata(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, quorum int) ([]StorageAPI, error) { isDir := false srcXLJSON := path.Join(srcEntry, xlMetaJSONFile) dstXLJSON := path.Join(dstEntry, xlMetaJSONFile) - return rename(disks, srcBucket, srcXLJSON, dstBucket, dstXLJSON, isDir, quorum) + return rename(ctx, disks, srcBucket, srcXLJSON, dstBucket, dstXLJSON, isDir, quorum) } // writeUniqueXLMetadata - writes unique `xl.json` content for each disk in order. -func writeUniqueXLMetadata(disks []StorageAPI, bucket, prefix string, xlMetas []xlMetaV1, quorum int) ([]StorageAPI, error) { +func writeUniqueXLMetadata(ctx context.Context, disks []StorageAPI, bucket, prefix string, xlMetas []xlMetaV1, quorum int) ([]StorageAPI, error) { var wg = &sync.WaitGroup{} var mErrs = make([]error, len(disks)) // Start writing `xl.json` to all disks in parallel. for index, disk := range disks { if disk == nil { - mErrs[index] = errors.Trace(errDiskNotFound) + logger.LogIf(ctx, errDiskNotFound) + mErrs[index] = errDiskNotFound continue } wg.Add(1) @@ -520,7 +531,7 @@ func writeUniqueXLMetadata(disks []StorageAPI, bucket, prefix string, xlMetas [] xlMetas[index].Erasure.Index = index + 1 // Write unique `xl.json` for a disk at index. - err := writeXLMetadata(disk, bucket, prefix, xlMetas[index]) + err := writeXLMetadata(ctx, disk, bucket, prefix, xlMetas[index]) if err != nil { mErrs[index] = err } @@ -530,23 +541,24 @@ func writeUniqueXLMetadata(disks []StorageAPI, bucket, prefix string, xlMetas [] // Wait for all the routines. wg.Wait() - err := reduceWriteQuorumErrs(mErrs, objectOpIgnoredErrs, quorum) + err := reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, quorum) if errors.Cause(err) == errXLWriteQuorum { // Delete all `xl.json` successfully renamed. - deleteAllXLMetadata(disks, bucket, prefix, mErrs) + deleteAllXLMetadata(ctx, disks, bucket, prefix, mErrs) } return evalDisks(disks, mErrs), err } // writeSameXLMetadata - write `xl.json` on all disks in order. -func writeSameXLMetadata(disks []StorageAPI, bucket, prefix string, xlMeta xlMetaV1, writeQuorum int) ([]StorageAPI, error) { +func writeSameXLMetadata(ctx context.Context, disks []StorageAPI, bucket, prefix string, xlMeta xlMetaV1, writeQuorum int) ([]StorageAPI, error) { var wg = &sync.WaitGroup{} var mErrs = make([]error, len(disks)) // Start writing `xl.json` to all disks in parallel. for index, disk := range disks { if disk == nil { - mErrs[index] = errors.Trace(errDiskNotFound) + logger.LogIf(ctx, errDiskNotFound) + mErrs[index] = errDiskNotFound continue } wg.Add(1) @@ -558,7 +570,7 @@ func writeSameXLMetadata(disks []StorageAPI, bucket, prefix string, xlMeta xlMet metadata.Erasure.Index = index + 1 // Write xl metadata. - err := writeXLMetadata(disk, bucket, prefix, metadata) + err := writeXLMetadata(ctx, disk, bucket, prefix, metadata) if err != nil { mErrs[index] = err } @@ -568,10 +580,10 @@ func writeSameXLMetadata(disks []StorageAPI, bucket, prefix string, xlMeta xlMet // Wait for all the routines. wg.Wait() - err := reduceWriteQuorumErrs(mErrs, objectOpIgnoredErrs, writeQuorum) + err := reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, writeQuorum) if errors.Cause(err) == errXLWriteQuorum { // Delete all `xl.json` successfully renamed. - deleteAllXLMetadata(disks, bucket, prefix, mErrs) + deleteAllXLMetadata(ctx, disks, bucket, prefix, mErrs) } return evalDisks(disks, mErrs), err } diff --git a/cmd/xl-v1-metadata_test.go b/cmd/xl-v1-metadata_test.go index 8dfcdb80d..ce4d08ddb 100644 --- a/cmd/xl-v1-metadata_test.go +++ b/cmd/xl-v1-metadata_test.go @@ -76,7 +76,7 @@ func testXLReadStat(obj ObjectLayer, instanceType string, disks []string, t *tes } } - _, _, err = obj.(*xlObjects).readXLMetaStat(bucketName, objectName) + _, _, err = obj.(*xlObjects).readXLMetaStat(context.Background(), bucketName, objectName) if err != nil { t.Fatal(err) } @@ -85,7 +85,7 @@ func testXLReadStat(obj ObjectLayer, instanceType string, disks []string, t *tes removeDiskN(disks, 7) // Removing disk shouldn't affect reading object info. - _, _, err = obj.(*xlObjects).readXLMetaStat(bucketName, objectName) + _, _, err = obj.(*xlObjects).readXLMetaStat(context.Background(), bucketName, objectName) if err != nil { t.Fatal(err) } @@ -94,7 +94,7 @@ func testXLReadStat(obj ObjectLayer, instanceType string, disks []string, t *tes os.RemoveAll(path.Join(disk, bucketName)) } - _, _, err = obj.(*xlObjects).readXLMetaStat(bucketName, objectName) + _, _, err = obj.(*xlObjects).readXLMetaStat(context.Background(), bucketName, objectName) if errors2.Cause(err) != errVolumeNotFound { t.Fatal(err) } @@ -160,7 +160,7 @@ func testXLReadMetaParts(obj ObjectLayer, instanceType string, disks []string, t uploadIDPath := obj.(*xlObjects).getUploadIDDir(bucketNames[0], objectNames[0], uploadIDs[0]) - _, _, err = obj.(*xlObjects).readXLMetaParts(minioMetaMultipartBucket, uploadIDPath) + _, _, err = obj.(*xlObjects).readXLMetaParts(context.Background(), minioMetaMultipartBucket, uploadIDPath) if err != nil { t.Fatal(err) } @@ -169,7 +169,7 @@ func testXLReadMetaParts(obj ObjectLayer, instanceType string, disks []string, t removeDiskN(disks, 7) // Removing disk shouldn't affect reading object parts info. - _, _, err = obj.(*xlObjects).readXLMetaParts(minioMetaMultipartBucket, uploadIDPath) + _, _, err = obj.(*xlObjects).readXLMetaParts(context.Background(), minioMetaMultipartBucket, uploadIDPath) if err != nil { t.Fatal(err) } @@ -179,7 +179,7 @@ func testXLReadMetaParts(obj ObjectLayer, instanceType string, disks []string, t os.RemoveAll(path.Join(disk, minioMetaMultipartBucket, obj.(*xlObjects).getMultipartSHADir(bucketNames[0], objectNames[0]))) } - _, _, err = obj.(*xlObjects).readXLMetaParts(minioMetaMultipartBucket, uploadIDPath) + _, _, err = obj.(*xlObjects).readXLMetaParts(context.Background(), minioMetaMultipartBucket, uploadIDPath) if errors2.Cause(err) != errFileNotFound { t.Fatal(err) } @@ -298,7 +298,7 @@ func TestObjectToPartOffset(t *testing.T) { // Test them. for _, testCase := range testCases { - index, offset, err := xlMeta.ObjectToPartOffset(testCase.offset) + index, offset, err := xlMeta.ObjectToPartOffset(context.Background(), testCase.offset) err = errors2.Cause(err) if err != testCase.expectedErr { t.Fatalf("%+v: expected = %s, got: %s", testCase, testCase.expectedErr, err) @@ -355,7 +355,7 @@ func TestPickValidXLMeta(t *testing.T) { }, } for i, test := range testCases { - xlMeta, err := pickValidXLMeta(test.metaArr, test.modTime) + xlMeta, err := pickValidXLMeta(context.Background(), test.metaArr, test.modTime) if test.expectedErr != nil { if errors2.Cause(err).Error() != test.expectedErr.Error() { t.Errorf("Test %d: Expected to fail with %v but received %v", diff --git a/cmd/xl-v1-multipart.go b/cmd/xl-v1-multipart.go index d181045ac..fbb51148c 100644 --- a/cmd/xl-v1-multipart.go +++ b/cmd/xl-v1-multipart.go @@ -26,6 +26,7 @@ import ( "sync" "time" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/hash" "github.com/minio/minio/pkg/mimedb" @@ -40,7 +41,7 @@ func (xl xlObjects) getMultipartSHADir(bucket, object string) string { } // isUploadIDExists - verify if a given uploadID exists and is valid. -func (xl xlObjects) isUploadIDExists(bucket, object, uploadID string) bool { +func (xl xlObjects) isUploadIDExists(ctx context.Context, bucket, object, uploadID string) bool { return xl.isObject(minioMetaMultipartBucket, xl.getUploadIDDir(bucket, object, uploadID)) } @@ -65,7 +66,7 @@ func (xl xlObjects) removeObjectPart(bucket, object, uploadID, partName string) } // statPart - returns fileInfo structure for a successful stat on part file. -func (xl xlObjects) statPart(bucket, object, uploadID, partName string) (fileInfo FileInfo, err error) { +func (xl xlObjects) statPart(ctx context.Context, bucket, object, uploadID, partName string) (fileInfo FileInfo, err error) { var ignoredErrs []error partNamePath := path.Join(xl.getUploadIDDir(bucket, object, uploadID), partName) for _, disk := range xl.getLoadBalancedDisks() { @@ -83,16 +84,17 @@ func (xl xlObjects) statPart(bucket, object, uploadID, partName string) (fileInf continue } // Error is not ignored, return right here. - return FileInfo{}, errors.Trace(err) + logger.LogIf(ctx, err) + return FileInfo{}, err } // If all errors were ignored, reduce to maximal occurrence // based on the read quorum. readQuorum := len(xl.getDisks()) / 2 - return FileInfo{}, reduceReadQuorumErrs(ignoredErrs, nil, readQuorum) + return FileInfo{}, reduceReadQuorumErrs(ctx, ignoredErrs, nil, readQuorum) } // commitXLMetadata - commit `xl.json` from source prefix to destination prefix in the given slice of disks. -func commitXLMetadata(disks []StorageAPI, srcBucket, srcPrefix, dstBucket, dstPrefix string, quorum int) ([]StorageAPI, error) { +func commitXLMetadata(ctx context.Context, disks []StorageAPI, srcBucket, srcPrefix, dstBucket, dstPrefix string, quorum int) ([]StorageAPI, error) { var wg = &sync.WaitGroup{} var mErrs = make([]error, len(disks)) @@ -102,7 +104,8 @@ func commitXLMetadata(disks []StorageAPI, srcBucket, srcPrefix, dstBucket, dstPr // Rename `xl.json` to all disks in parallel. for index, disk := range disks { if disk == nil { - mErrs[index] = errors.Trace(errDiskNotFound) + logger.LogIf(ctx, errDiskNotFound) + mErrs[index] = errDiskNotFound continue } wg.Add(1) @@ -115,7 +118,8 @@ func commitXLMetadata(disks []StorageAPI, srcBucket, srcPrefix, dstBucket, dstPr // Renames `xl.json` from source prefix to destination prefix. rErr := disk.RenameFile(srcBucket, srcJSONFile, dstBucket, dstJSONFile) if rErr != nil { - mErrs[index] = errors.Trace(rErr) + logger.LogIf(ctx, rErr) + mErrs[index] = rErr return } mErrs[index] = nil @@ -124,10 +128,10 @@ func commitXLMetadata(disks []StorageAPI, srcBucket, srcPrefix, dstBucket, dstPr // Wait for all the routines. wg.Wait() - err := reduceWriteQuorumErrs(mErrs, objectOpIgnoredErrs, quorum) + err := reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, quorum) if errors.Cause(err) == errXLWriteQuorum { // Delete all `xl.json` successfully renamed. - deleteAllXLMetadata(disks, dstBucket, dstPrefix, mErrs) + deleteAllXLMetadata(ctx, disks, dstBucket, dstPrefix, mErrs) } return evalDisks(disks, mErrs), err } @@ -140,7 +144,7 @@ func commitXLMetadata(disks []StorageAPI, srcBucket, srcPrefix, dstBucket, dstPr // towards simplification of multipart APIs. // The resulting ListMultipartsInfo structure is unmarshalled directly as XML. func (xl xlObjects) ListMultipartUploads(ctx context.Context, bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, e error) { - if err := checkListMultipartArgs(bucket, object, keyMarker, uploadIDMarker, delimiter, xl); err != nil { + if err := checkListMultipartArgs(ctx, bucket, object, keyMarker, uploadIDMarker, delimiter, xl); err != nil { return result, err } @@ -158,7 +162,8 @@ func (xl xlObjects) ListMultipartUploads(ctx context.Context, bucket, object, ke if err == errFileNotFound { return result, nil } - return result, errors.Trace(err) + logger.LogIf(ctx, err) + return result, err } for i := range uploadIDs { uploadIDs[i] = strings.TrimSuffix(uploadIDs[i], slashSeparator) @@ -184,7 +189,7 @@ func (xl xlObjects) ListMultipartUploads(ctx context.Context, bucket, object, ke // '.minio.sys/multipart/bucket/object/uploads.json' on all the // disks. `uploads.json` carries metadata regarding on-going multipart // operation(s) on the object. -func (xl xlObjects) newMultipartUpload(bucket string, object string, meta map[string]string) (string, error) { +func (xl xlObjects) newMultipartUpload(ctx context.Context, bucket string, object string, meta map[string]string) (string, error) { dataBlocks, parityBlocks := getRedundancyCount(meta[amzStorageClass], len(xl.getDisks())) @@ -213,17 +218,17 @@ func (xl xlObjects) newMultipartUpload(bucket string, object string, meta map[st tempUploadIDPath := uploadID // Write updated `xl.json` to all disks. - disks, err := writeSameXLMetadata(xl.getDisks(), minioMetaTmpBucket, tempUploadIDPath, xlMeta, writeQuorum) + disks, err := writeSameXLMetadata(ctx, xl.getDisks(), minioMetaTmpBucket, tempUploadIDPath, xlMeta, writeQuorum) if err != nil { return "", toObjectErr(err, minioMetaTmpBucket, tempUploadIDPath) } // delete the tmp path later in case we fail to rename (ignore // returned errors) - this will be a no-op in case of a rename // success. - defer xl.deleteObject(minioMetaTmpBucket, tempUploadIDPath) + defer xl.deleteObject(ctx, minioMetaTmpBucket, tempUploadIDPath) // Attempt to rename temp upload object to actual upload path object - _, rErr := renameObject(disks, minioMetaTmpBucket, tempUploadIDPath, minioMetaMultipartBucket, uploadIDPath, writeQuorum) + _, rErr := renameObject(ctx, disks, minioMetaTmpBucket, tempUploadIDPath, minioMetaMultipartBucket, uploadIDPath, writeQuorum) if rErr != nil { return "", toObjectErr(rErr, minioMetaMultipartBucket, uploadIDPath) } @@ -238,14 +243,14 @@ func (xl xlObjects) newMultipartUpload(bucket string, object string, meta map[st // // Implements S3 compatible initiate multipart API. func (xl xlObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string) (string, error) { - if err := checkNewMultipartArgs(bucket, object, xl); err != nil { + if err := checkNewMultipartArgs(ctx, bucket, object, xl); err != nil { return "", err } // No metadata is set, allocate a new one. if meta == nil { meta = make(map[string]string) } - return xl.newMultipartUpload(bucket, object, meta) + return xl.newMultipartUpload(ctx, bucket, object, meta) } // CopyObjectPart - reads incoming stream and internally erasure codes @@ -262,20 +267,20 @@ func (xl xlObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, ds } defer objectSRLock.RUnlock() - if err := checkNewMultipartArgs(srcBucket, srcObject, xl); err != nil { + if err := checkNewMultipartArgs(ctx, srcBucket, srcObject, xl); err != nil { return pi, err } go func() { - if gerr := xl.getObject(srcBucket, srcObject, startOffset, length, srcInfo.Writer, srcInfo.ETag); gerr != nil { + if gerr := xl.getObject(ctx, srcBucket, srcObject, startOffset, length, srcInfo.Writer, srcInfo.ETag); gerr != nil { if gerr = srcInfo.Writer.Close(); gerr != nil { - errorIf(gerr, "Unable to read %s of the object `%s/%s`.", srcBucket, srcObject) + logger.LogIf(ctx, gerr) } return } // Close writer explicitly signalling we wrote all data. if gerr := srcInfo.Writer.Close(); gerr != nil { - errorIf(gerr, "Unable to read %s of the object `%s/%s`.", srcBucket, srcObject) + logger.LogIf(ctx, gerr) return } }() @@ -295,13 +300,14 @@ func (xl xlObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, ds // // Implements S3 compatible Upload Part API. func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (pi PartInfo, e error) { - if err := checkPutObjectPartArgs(bucket, object, xl); err != nil { + if err := checkPutObjectPartArgs(ctx, bucket, object, xl); err != nil { return pi, err } // Validate input data size and it can never be less than zero. if data.Size() < 0 { - return pi, toObjectErr(errors.Trace(errInvalidArgument)) + logger.LogIf(ctx, errInvalidArgument) + return pi, toObjectErr(errInvalidArgument) } var partsMetadata []xlMetaV1 @@ -315,13 +321,13 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID } // Validates if upload ID exists. - if !xl.isUploadIDExists(bucket, object, uploadID) { + if !xl.isUploadIDExists(ctx, bucket, object, uploadID) { preUploadIDLock.RUnlock() - return pi, errors.Trace(InvalidUploadID{UploadID: uploadID}) + return pi, InvalidUploadID{UploadID: uploadID} } // Read metadata associated with the object from all disks. - partsMetadata, errs = readAllXLMetadata(xl.getDisks(), minioMetaMultipartBucket, + partsMetadata, errs = readAllXLMetadata(ctx, xl.getDisks(), minioMetaMultipartBucket, uploadIDPath) // get Quorum for this object @@ -330,7 +336,7 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID return pi, toObjectErr(err, bucket, object) } - reducedErr := reduceWriteQuorumErrs(errs, objectOpIgnoredErrs, writeQuorum) + reducedErr := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum) if errors.Cause(reducedErr) == errXLWriteQuorum { preUploadIDLock.RUnlock() return pi, toObjectErr(reducedErr, bucket, object) @@ -341,7 +347,7 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID onlineDisks, modTime := listOnlineDisks(xl.getDisks(), partsMetadata, errs) // Pick one from the first valid metadata. - xlMeta, err := pickValidXLMeta(partsMetadata, modTime) + xlMeta, err := pickValidXLMeta(ctx, partsMetadata, modTime) if err != nil { return pi, err } @@ -356,15 +362,15 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID tmpPartPath := path.Join(tmpPart, partSuffix) // Delete the temporary object part. If PutObjectPart succeeds there would be nothing to delete. - defer xl.deleteObject(minioMetaTmpBucket, tmpPart) + defer xl.deleteObject(ctx, minioMetaTmpBucket, tmpPart) if data.Size() > 0 { - if pErr := xl.prepareFile(minioMetaTmpBucket, tmpPartPath, data.Size(), onlineDisks, xlMeta.Erasure.BlockSize, xlMeta.Erasure.DataBlocks, writeQuorum); err != nil { + if pErr := xl.prepareFile(ctx, minioMetaTmpBucket, tmpPartPath, data.Size(), onlineDisks, xlMeta.Erasure.BlockSize, xlMeta.Erasure.DataBlocks, writeQuorum); err != nil { return pi, toObjectErr(pErr, bucket, object) } } - storage, err := NewErasureStorage(onlineDisks, xlMeta.Erasure.DataBlocks, xlMeta.Erasure.ParityBlocks, xlMeta.Erasure.BlockSize) + storage, err := NewErasureStorage(ctx, onlineDisks, xlMeta.Erasure.DataBlocks, xlMeta.Erasure.ParityBlocks, xlMeta.Erasure.BlockSize) if err != nil { return pi, toObjectErr(err, bucket, object) } @@ -373,7 +379,7 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID buffer := xl.bp.Get() defer xl.bp.Put(buffer) - file, err := storage.CreateFile(data, minioMetaTmpBucket, tmpPartPath, buffer, DefaultBitrotAlgorithm, writeQuorum) + file, err := storage.CreateFile(ctx, data, minioMetaTmpBucket, tmpPartPath, buffer, DefaultBitrotAlgorithm, writeQuorum) if err != nil { return pi, toObjectErr(err, bucket, object) } @@ -381,7 +387,8 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID // Should return IncompleteBody{} error when reader has fewer bytes // than specified in request header. if file.Size < data.Size() { - return pi, errors.Trace(IncompleteBody{}) + logger.LogIf(ctx, IncompleteBody{}) + return pi, IncompleteBody{} } // post-upload check (write) lock @@ -392,20 +399,20 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID defer postUploadIDLock.Unlock() // Validate again if upload ID still exists. - if !xl.isUploadIDExists(bucket, object, uploadID) { - return pi, errors.Trace(InvalidUploadID{UploadID: uploadID}) + if !xl.isUploadIDExists(ctx, bucket, object, uploadID) { + return pi, InvalidUploadID{UploadID: uploadID} } // Rename temporary part file to its final location. partPath := path.Join(uploadIDPath, partSuffix) - onlineDisks, err = renamePart(onlineDisks, minioMetaTmpBucket, tmpPartPath, minioMetaMultipartBucket, partPath, writeQuorum) + onlineDisks, err = renamePart(ctx, onlineDisks, minioMetaTmpBucket, tmpPartPath, minioMetaMultipartBucket, partPath, writeQuorum) if err != nil { return pi, toObjectErr(err, minioMetaMultipartBucket, partPath) } // Read metadata again because it might be updated with parallel upload of another part. - partsMetadata, errs = readAllXLMetadata(onlineDisks, minioMetaMultipartBucket, uploadIDPath) - reducedErr = reduceWriteQuorumErrs(errs, objectOpIgnoredErrs, writeQuorum) + partsMetadata, errs = readAllXLMetadata(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath) + reducedErr = reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum) if errors.Cause(reducedErr) == errXLWriteQuorum { return pi, toObjectErr(reducedErr, bucket, object) } @@ -414,7 +421,7 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID onlineDisks, modTime = listOnlineDisks(onlineDisks, partsMetadata, errs) // Pick one from the first valid metadata. - xlMeta, err = pickValidXLMeta(partsMetadata, modTime) + xlMeta, err = pickValidXLMeta(ctx, partsMetadata, modTime) if err != nil { return pi, err } @@ -440,15 +447,15 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID tempXLMetaPath := newUUID // Writes a unique `xl.json` each disk carrying new checksum related information. - if onlineDisks, err = writeUniqueXLMetadata(onlineDisks, minioMetaTmpBucket, tempXLMetaPath, partsMetadata, writeQuorum); err != nil { + if onlineDisks, err = writeUniqueXLMetadata(ctx, onlineDisks, minioMetaTmpBucket, tempXLMetaPath, partsMetadata, writeQuorum); err != nil { return pi, toObjectErr(err, minioMetaTmpBucket, tempXLMetaPath) } - if _, err = commitXLMetadata(onlineDisks, minioMetaTmpBucket, tempXLMetaPath, minioMetaMultipartBucket, uploadIDPath, writeQuorum); err != nil { + if _, err = commitXLMetadata(ctx, onlineDisks, minioMetaTmpBucket, tempXLMetaPath, minioMetaMultipartBucket, uploadIDPath, writeQuorum); err != nil { return pi, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath) } - fi, err := xl.statPart(bucket, object, uploadID, partSuffix) + fi, err := xl.statPart(ctx, bucket, object, uploadID, partSuffix) if err != nil { return pi, toObjectErr(err, minioMetaMultipartBucket, partSuffix) } @@ -470,7 +477,7 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID // ListPartsInfo structure is marshalled directly into XML and // replied back to the client. func (xl xlObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int) (result ListPartsInfo, e error) { - if err := checkListPartsArgs(bucket, object, xl); err != nil { + if err := checkListPartsArgs(ctx, bucket, object, xl); err != nil { return result, err } // Hold lock so that there is no competing @@ -482,13 +489,13 @@ func (xl xlObjects) ListObjectParts(ctx context.Context, bucket, object, uploadI } defer uploadIDLock.Unlock() - if !xl.isUploadIDExists(bucket, object, uploadID) { - return result, errors.Trace(InvalidUploadID{UploadID: uploadID}) + if !xl.isUploadIDExists(ctx, bucket, object, uploadID) { + return result, InvalidUploadID{UploadID: uploadID} } uploadIDPath := xl.getUploadIDDir(bucket, object, uploadID) - xlParts, xlMeta, err := xl.readXLMetaParts(minioMetaMultipartBucket, uploadIDPath) + xlParts, xlMeta, err := xl.readXLMetaParts(ctx, minioMetaMultipartBucket, uploadIDPath) if err != nil { return result, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath) } @@ -520,7 +527,7 @@ func (xl xlObjects) ListObjectParts(ctx context.Context, bucket, object, uploadI count := maxParts for _, part := range parts { var fi FileInfo - fi, err = xl.statPart(bucket, object, uploadID, part.Name) + fi, err = xl.statPart(ctx, bucket, object, uploadID, part.Name) if err != nil { return result, toObjectErr(err, minioMetaBucket, path.Join(uploadID, part.Name)) } @@ -553,7 +560,7 @@ func (xl xlObjects) ListObjectParts(ctx context.Context, bucket, object, uploadI // // Implements S3 compatible Complete multipart API. func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart) (oi ObjectInfo, e error) { - if err := checkCompleteMultipartArgs(bucket, object, xl); err != nil { + if err := checkCompleteMultipartArgs(ctx, bucket, object, xl); err != nil { return oi, err } // Hold write lock on the object. @@ -577,24 +584,24 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string, } defer uploadIDLock.Unlock() - if !xl.isUploadIDExists(bucket, object, uploadID) { - return oi, errors.Trace(InvalidUploadID{UploadID: uploadID}) + if !xl.isUploadIDExists(ctx, bucket, object, uploadID) { + return oi, InvalidUploadID{UploadID: uploadID} } // Check if an object is present as one of the parent dir. // -- FIXME. (needs a new kind of lock). - if xl.parentDirIsObject(bucket, path.Dir(object)) { - return oi, toObjectErr(errors.Trace(errFileAccessDenied), bucket, object) + if xl.parentDirIsObject(ctx, bucket, path.Dir(object)) { + return oi, toObjectErr(errFileAccessDenied, bucket, object) } // Calculate s3 compatible md5sum for complete multipart. - s3MD5, err := getCompleteMultipartMD5(parts) + s3MD5, err := getCompleteMultipartMD5(ctx, parts) if err != nil { return oi, err } // Read metadata associated with the object from all disks. - partsMetadata, errs := readAllXLMetadata(xl.getDisks(), minioMetaMultipartBucket, uploadIDPath) + partsMetadata, errs := readAllXLMetadata(ctx, xl.getDisks(), minioMetaMultipartBucket, uploadIDPath) // get Quorum for this object _, writeQuorum, err := objectQuorumFromMeta(xl, partsMetadata, errs) @@ -602,7 +609,7 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string, return oi, toObjectErr(err, bucket, object) } - reducedErr := reduceWriteQuorumErrs(errs, objectOpIgnoredErrs, writeQuorum) + reducedErr := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum) if errors.Cause(reducedErr) == errXLWriteQuorum { return oi, toObjectErr(reducedErr, bucket, object) } @@ -613,7 +620,7 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string, var objectSize int64 // Pick one from the first valid metadata. - xlMeta, err := pickValidXLMeta(partsMetadata, modTime) + xlMeta, err := pickValidXLMeta(ctx, partsMetadata, modTime) if err != nil { return oi, err } @@ -635,21 +642,28 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string, partIdx := objectPartIndex(currentXLMeta.Parts, part.PartNumber) // All parts should have same part number. if partIdx == -1 { - return oi, errors.Trace(InvalidPart{}) + logger.LogIf(ctx, InvalidPart{}) + return oi, InvalidPart{} } // All parts should have same ETag as previously generated. if currentXLMeta.Parts[partIdx].ETag != part.ETag { - return oi, errors.Trace(InvalidPart{}) + logger.LogIf(ctx, InvalidPart{}) + return oi, InvalidPart{} } // All parts except the last part has to be atleast 5MB. if (i < len(parts)-1) && !isMinAllowedPartSize(currentXLMeta.Parts[partIdx].Size) { - return oi, errors.Trace(PartTooSmall{ + logger.LogIf(ctx, PartTooSmall{ PartNumber: part.PartNumber, PartSize: currentXLMeta.Parts[partIdx].Size, PartETag: part.ETag, }) + return oi, PartTooSmall{ + PartNumber: part.PartNumber, + PartSize: currentXLMeta.Parts[partIdx].Size, + PartETag: part.ETag, + } } // Last part could have been uploaded as 0bytes, do not need @@ -689,12 +703,12 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string, } // Write unique `xl.json` for each disk. - if onlineDisks, err = writeUniqueXLMetadata(onlineDisks, minioMetaTmpBucket, tempUploadIDPath, partsMetadata, writeQuorum); err != nil { + if onlineDisks, err = writeUniqueXLMetadata(ctx, onlineDisks, minioMetaTmpBucket, tempUploadIDPath, partsMetadata, writeQuorum); err != nil { return oi, toObjectErr(err, minioMetaTmpBucket, tempUploadIDPath) } var rErr error - onlineDisks, rErr = commitXLMetadata(onlineDisks, minioMetaTmpBucket, tempUploadIDPath, minioMetaMultipartBucket, uploadIDPath, writeQuorum) + onlineDisks, rErr = commitXLMetadata(ctx, onlineDisks, minioMetaTmpBucket, tempUploadIDPath, minioMetaMultipartBucket, uploadIDPath, writeQuorum) if rErr != nil { return oi, toObjectErr(rErr, minioMetaMultipartBucket, uploadIDPath) } @@ -704,12 +718,12 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string, newUniqueID := mustGetUUID() // Delete success renamed object. - defer xl.deleteObject(minioMetaTmpBucket, newUniqueID) + defer xl.deleteObject(ctx, minioMetaTmpBucket, newUniqueID) // NOTE: Do not use online disks slice here. // The reason is that existing object should be purged // regardless of `xl.json` status and rolled back in case of errors. - _, err = renameObject(xl.getDisks(), bucket, object, minioMetaTmpBucket, newUniqueID, writeQuorum) + _, err = renameObject(ctx, xl.getDisks(), bucket, object, minioMetaTmpBucket, newUniqueID, writeQuorum) if err != nil { return oi, toObjectErr(err, bucket, object) } @@ -731,12 +745,13 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string, // Deny if WORM is enabled if globalWORMEnabled { if xl.isObject(bucket, object) { - return ObjectInfo{}, errors.Trace(ObjectAlreadyExists{Bucket: bucket, Object: object}) + logger.LogIf(ctx, ObjectAlreadyExists{Bucket: bucket, Object: object}) + return ObjectInfo{}, ObjectAlreadyExists{Bucket: bucket, Object: object} } } // Rename the multipart object to final location. - if _, err = renameObject(onlineDisks, minioMetaMultipartBucket, uploadIDPath, bucket, object, writeQuorum); err != nil { + if _, err = renameObject(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, bucket, object, writeQuorum); err != nil { return oi, toObjectErr(err, bucket, object) } @@ -745,21 +760,22 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string, } // Wrapper which removes all the uploaded parts. -func (xl xlObjects) cleanupUploadedParts(uploadIDPath string, writeQuorum int) error { +func (xl xlObjects) cleanupUploadedParts(ctx context.Context, uploadIDPath string, writeQuorum int) error { var errs = make([]error, len(xl.getDisks())) var wg = &sync.WaitGroup{} // Cleanup uploadID for all disks. for index, disk := range xl.getDisks() { if disk == nil { - errs[index] = errors.Trace(errDiskNotFound) + logger.LogIf(ctx, errDiskNotFound) + errs[index] = errDiskNotFound continue } wg.Add(1) // Cleanup each uploadID in a routine. go func(index int, disk StorageAPI) { defer wg.Done() - err := cleanupDir(disk, minioMetaMultipartBucket, uploadIDPath) + err := cleanupDir(ctx, disk, minioMetaMultipartBucket, uploadIDPath) if err != nil { errs[index] = err } @@ -769,7 +785,7 @@ func (xl xlObjects) cleanupUploadedParts(uploadIDPath string, writeQuorum int) e // Wait for all the cleanups to finish. wg.Wait() - return reduceWriteQuorumErrs(errs, objectOpIgnoredErrs, writeQuorum) + return reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum) } // AbortMultipartUpload - aborts an ongoing multipart operation @@ -784,7 +800,7 @@ func (xl xlObjects) cleanupUploadedParts(uploadIDPath string, writeQuorum int) e // that this is an atomic idempotent operation. Subsequent calls have // no affect and further requests to the same uploadID would not be honored. func (xl xlObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error { - if err := checkAbortMultipartArgs(bucket, object, xl); err != nil { + if err := checkAbortMultipartArgs(ctx, bucket, object, xl); err != nil { return err } // Construct uploadIDPath. @@ -797,12 +813,12 @@ func (xl xlObjects) AbortMultipartUpload(ctx context.Context, bucket, object, up } defer uploadIDLock.Unlock() - if !xl.isUploadIDExists(bucket, object, uploadID) { - return errors.Trace(InvalidUploadID{UploadID: uploadID}) + if !xl.isUploadIDExists(ctx, bucket, object, uploadID) { + return InvalidUploadID{UploadID: uploadID} } // Read metadata associated with the object from all disks. - partsMetadata, errs := readAllXLMetadata(xl.getDisks(), minioMetaMultipartBucket, uploadIDPath) + partsMetadata, errs := readAllXLMetadata(ctx, xl.getDisks(), minioMetaMultipartBucket, uploadIDPath) // get Quorum for this object _, writeQuorum, err := objectQuorumFromMeta(xl, partsMetadata, errs) @@ -811,7 +827,7 @@ func (xl xlObjects) AbortMultipartUpload(ctx context.Context, bucket, object, up } // Cleanup all uploaded parts. - if err = xl.cleanupUploadedParts(uploadIDPath, writeQuorum); err != nil { + if err = xl.cleanupUploadedParts(ctx, uploadIDPath, writeQuorum); err != nil { return toObjectErr(err, bucket, object) } @@ -820,7 +836,7 @@ func (xl xlObjects) AbortMultipartUpload(ctx context.Context, bucket, object, up } // Clean-up the old multipart uploads. Should be run in a Go routine. -func (xl xlObjects) cleanupStaleMultipartUploads(cleanupInterval, expiry time.Duration, doneCh chan struct{}) { +func (xl xlObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh chan struct{}) { ticker := time.NewTicker(cleanupInterval) for { @@ -839,13 +855,13 @@ func (xl xlObjects) cleanupStaleMultipartUploads(cleanupInterval, expiry time.Du if disk == nil { continue } - xl.cleanupStaleMultipartUploadsOnDisk(disk, expiry) + xl.cleanupStaleMultipartUploadsOnDisk(ctx, disk, expiry) } } } // Remove the old multipart uploads on the given disk. -func (xl xlObjects) cleanupStaleMultipartUploadsOnDisk(disk StorageAPI, expiry time.Duration) { +func (xl xlObjects) cleanupStaleMultipartUploadsOnDisk(ctx context.Context, disk StorageAPI, expiry time.Duration) { now := time.Now() shaDirs, err := disk.ListDir(minioMetaMultipartBucket, "") if err != nil { @@ -868,7 +884,7 @@ func (xl xlObjects) cleanupStaleMultipartUploadsOnDisk(disk StorageAPI, expiry t // when it removes files. We igore the error message from xl.cleanupUploadedParts() as we can't // return it to any client. Hence we set quorum to 0. quorum := 0 - xl.cleanupUploadedParts(uploadIDPath, quorum) + xl.cleanupUploadedParts(ctx, uploadIDPath, quorum) } } } diff --git a/cmd/xl-v1-multipart_test.go b/cmd/xl-v1-multipart_test.go index 9e36c45fe..f2befbd9d 100644 --- a/cmd/xl-v1-multipart_test.go +++ b/cmd/xl-v1-multipart_test.go @@ -57,7 +57,7 @@ func TestXLCleanupStaleMultipartUploads(t *testing.T) { t.Fatal("Unexpected err: ", err) } - go xl.cleanupStaleMultipartUploads(20*time.Millisecond, 0, globalServiceDoneCh) + go xl.cleanupStaleMultipartUploads(context.Background(), 20*time.Millisecond, 0, globalServiceDoneCh) // Wait for 40ms such that - we have given enough time for // cleanup routine to kick in. diff --git a/cmd/xl-v1-object.go b/cmd/xl-v1-object.go index 782291273..f67c9bd56 100644 --- a/cmd/xl-v1-object.go +++ b/cmd/xl-v1-object.go @@ -25,6 +25,7 @@ import ( "strings" "sync" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/hash" "github.com/minio/minio/pkg/mimedb" @@ -34,7 +35,7 @@ import ( var objectOpIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied) // putObjectDir hints the bottom layer to create a new directory. -func (xl xlObjects) putObjectDir(bucket, object string, writeQuorum int) error { +func (xl xlObjects) putObjectDir(ctx context.Context, bucket, object string, writeQuorum int) error { var wg = &sync.WaitGroup{} errs := make([]error, len(xl.getDisks())) @@ -53,11 +54,11 @@ func (xl xlObjects) putObjectDir(bucket, object string, writeQuorum int) error { } wg.Wait() - return reduceWriteQuorumErrs(errs, objectOpIgnoredErrs, writeQuorum) + return reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum) } // prepareFile hints the bottom layer to optimize the creation of a new object -func (xl xlObjects) prepareFile(bucket, object string, size int64, onlineDisks []StorageAPI, blockSize int64, dataBlocks, writeQuorum int) error { +func (xl xlObjects) prepareFile(ctx context.Context, bucket, object string, size int64, onlineDisks []StorageAPI, blockSize int64, dataBlocks, writeQuorum int) error { pErrs := make([]error, len(onlineDisks)) // Calculate the real size of the part in one disk. actualSize := xl.sizeOnDisk(size, blockSize, dataBlocks) @@ -72,7 +73,7 @@ func (xl xlObjects) prepareFile(bucket, object string, size int64, onlineDisks [ } } } - return reduceWriteQuorumErrs(pErrs, objectOpIgnoredErrs, writeQuorum) + return reduceWriteQuorumErrs(ctx, pErrs, objectOpIgnoredErrs, writeQuorum) } /// Object Operations @@ -86,7 +87,7 @@ func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBuc // Read metadata associated with the object from all disks. storageDisks := xl.getDisks() - metaArr, errs := readAllXLMetadata(storageDisks, srcBucket, srcObject) + metaArr, errs := readAllXLMetadata(ctx, storageDisks, srcBucket, srcObject) // get Quorum for this object readQuorum, writeQuorum, err := objectQuorumFromMeta(xl, metaArr, errs) @@ -94,7 +95,7 @@ func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBuc return oi, toObjectErr(err, srcBucket, srcObject) } - if reducedErr := reduceReadQuorumErrs(errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil { + if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil { return oi, toObjectErr(reducedErr, srcBucket, srcObject) } @@ -102,7 +103,7 @@ func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBuc _, modTime := listOnlineDisks(storageDisks, metaArr, errs) // Pick latest valid metadata. - xlMeta, err := pickValidXLMeta(metaArr, modTime) + xlMeta, err := pickValidXLMeta(ctx, metaArr, modTime) if err != nil { return oi, toObjectErr(err, srcBucket, srcObject) } @@ -123,11 +124,11 @@ func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBuc tempObj := mustGetUUID() // Write unique `xl.json` for each disk. - if onlineDisks, err = writeUniqueXLMetadata(storageDisks, minioMetaTmpBucket, tempObj, metaArr, writeQuorum); err != nil { + if onlineDisks, err = writeUniqueXLMetadata(ctx, storageDisks, minioMetaTmpBucket, tempObj, metaArr, writeQuorum); err != nil { return oi, toObjectErr(err, srcBucket, srcObject) } // Rename atomically `xl.json` from tmp location to destination for each disk. - if _, err = renameXLMetadata(onlineDisks, minioMetaTmpBucket, tempObj, srcBucket, srcObject, writeQuorum); err != nil { + if _, err = renameXLMetadata(ctx, onlineDisks, minioMetaTmpBucket, tempObj, srcBucket, srcObject, writeQuorum); err != nil { return oi, toObjectErr(err, srcBucket, srcObject) } return xlMeta.ToObjectInfo(srcBucket, srcObject), nil @@ -138,8 +139,7 @@ func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBuc go func() { var startOffset int64 // Read the whole file. - if gerr := xl.getObject(srcBucket, srcObject, startOffset, length, pipeWriter, srcInfo.ETag); gerr != nil { - errorIf(gerr, "Unable to read %s of the object `%s/%s`.", srcBucket, srcObject) + if gerr := xl.getObject(ctx, srcBucket, srcObject, startOffset, length, pipeWriter, srcInfo.ETag); gerr != nil { pipeWriter.CloseWithError(toObjectErr(gerr, srcBucket, srcObject)) return } @@ -148,10 +148,11 @@ func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBuc hashReader, err := hash.NewReader(pipeReader, length, "", "") if err != nil { - return oi, toObjectErr(errors.Trace(err), dstBucket, dstObject) + logger.LogIf(ctx, err) + return oi, toObjectErr(err, dstBucket, dstObject) } - objInfo, err := xl.putObject(dstBucket, dstObject, hashReader, srcInfo.UserDefined) + objInfo, err := xl.putObject(ctx, dstBucket, dstObject, hashReader, srcInfo.UserDefined) if err != nil { return oi, toObjectErr(err, dstBucket, dstObject) } @@ -175,34 +176,37 @@ func (xl xlObjects) GetObject(ctx context.Context, bucket, object string, startO return err } defer objectLock.RUnlock() - return xl.getObject(bucket, object, startOffset, length, writer, etag) + return xl.getObject(ctx, bucket, object, startOffset, length, writer, etag) } // getObject wrapper for xl GetObject -func (xl xlObjects) getObject(bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error { +func (xl xlObjects) getObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error { - if err := checkGetObjArgs(bucket, object); err != nil { + if err := checkGetObjArgs(ctx, bucket, object); err != nil { return err } // Start offset cannot be negative. if startOffset < 0 { - return errors.Trace(errUnexpected) + logger.LogIf(ctx, errUnexpected) + return errUnexpected } // Writer cannot be nil. if writer == nil { - return errors.Trace(errUnexpected) + logger.LogIf(ctx, errUnexpected) + return errUnexpected } // If its a directory request, we return an empty body. if hasSuffix(object, slashSeparator) { _, err := writer.Write([]byte("")) - return toObjectErr(errors.Trace(err), bucket, object) + logger.LogIf(ctx, err) + return toObjectErr(err, bucket, object) } // Read metadata associated with the object from all disks. - metaArr, errs := readAllXLMetadata(xl.getDisks(), bucket, object) + metaArr, errs := readAllXLMetadata(ctx, xl.getDisks(), bucket, object) // get Quorum for this object readQuorum, _, err := objectQuorumFromMeta(xl, metaArr, errs) @@ -210,7 +214,7 @@ func (xl xlObjects) getObject(bucket, object string, startOffset int64, length i return toObjectErr(err, bucket, object) } - if reducedErr := reduceReadQuorumErrs(errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil { + if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil { return toObjectErr(reducedErr, bucket, object) } @@ -218,7 +222,7 @@ func (xl xlObjects) getObject(bucket, object string, startOffset int64, length i onlineDisks, modTime := listOnlineDisks(xl.getDisks(), metaArr, errs) // Pick latest valid metadata. - xlMeta, err := pickValidXLMeta(metaArr, modTime) + xlMeta, err := pickValidXLMeta(ctx, metaArr, modTime) if err != nil { return err } @@ -236,13 +240,14 @@ func (xl xlObjects) getObject(bucket, object string, startOffset int64, length i // Reply back invalid range if the input offset and length fall out of range. if startOffset > xlMeta.Stat.Size || startOffset+length > xlMeta.Stat.Size { - return errors.Trace(InvalidRange{startOffset, length, xlMeta.Stat.Size}) + logger.LogIf(ctx, InvalidRange{startOffset, length, xlMeta.Stat.Size}) + return InvalidRange{startOffset, length, xlMeta.Stat.Size} } // Get start part index and offset. - partIndex, partOffset, err := xlMeta.ObjectToPartOffset(startOffset) + partIndex, partOffset, err := xlMeta.ObjectToPartOffset(ctx, startOffset) if err != nil { - return errors.Trace(InvalidRange{startOffset, length, xlMeta.Stat.Size}) + return InvalidRange{startOffset, length, xlMeta.Stat.Size} } // Calculate endOffset according to length @@ -252,13 +257,13 @@ func (xl xlObjects) getObject(bucket, object string, startOffset int64, length i } // Get last part index to read given length. - lastPartIndex, _, err := xlMeta.ObjectToPartOffset(endOffset) + lastPartIndex, _, err := xlMeta.ObjectToPartOffset(ctx, endOffset) if err != nil { - return errors.Trace(InvalidRange{startOffset, length, xlMeta.Stat.Size}) + return InvalidRange{startOffset, length, xlMeta.Stat.Size} } var totalBytesRead int64 - storage, err := NewErasureStorage(onlineDisks, xlMeta.Erasure.DataBlocks, xlMeta.Erasure.ParityBlocks, xlMeta.Erasure.BlockSize) + storage, err := NewErasureStorage(ctx, onlineDisks, xlMeta.Erasure.DataBlocks, xlMeta.Erasure.ParityBlocks, xlMeta.Erasure.BlockSize) if err != nil { return toObjectErr(err, bucket, object) } @@ -288,7 +293,7 @@ func (xl xlObjects) getObject(bucket, object string, startOffset int64, length i checksums[index] = checksumInfo.Hash } - file, err := storage.ReadFile(writer, bucket, pathJoin(object, partName), partOffset, readSize, partSize, checksums, algorithm, xlMeta.Erasure.BlockSize) + file, err := storage.ReadFile(ctx, writer, bucket, pathJoin(object, partName), partOffset, readSize, partSize, checksums, algorithm, xlMeta.Erasure.BlockSize) if err != nil { return toObjectErr(err, bucket, object) } @@ -306,7 +311,7 @@ func (xl xlObjects) getObject(bucket, object string, startOffset int64, length i } // getObjectInfoDir - This getObjectInfo is specific to object directory lookup. -func (xl xlObjects) getObjectInfoDir(bucket, object string) (oi ObjectInfo, err error) { +func (xl xlObjects) getObjectInfoDir(ctx context.Context, bucket, object string) (oi ObjectInfo, err error) { var wg = &sync.WaitGroup{} errs := make([]error, len(xl.getDisks())) @@ -338,7 +343,7 @@ func (xl xlObjects) getObjectInfoDir(bucket, object string) (oi ObjectInfo, err wg.Wait() readQuorum := len(xl.getDisks()) / 2 - return dirObjectInfo(bucket, object, 0, map[string]string{}), reduceReadQuorumErrs(errs, objectOpIgnoredErrs, readQuorum) + return dirObjectInfo(bucket, object, 0, map[string]string{}), reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum) } // GetObjectInfo - reads object metadata and replies back ObjectInfo. @@ -350,18 +355,18 @@ func (xl xlObjects) GetObjectInfo(ctx context.Context, bucket, object string) (o } defer objectLock.RUnlock() - if err := checkGetObjArgs(bucket, object); err != nil { + if err := checkGetObjArgs(ctx, bucket, object); err != nil { return oi, err } if hasSuffix(object, slashSeparator) { - if oi, e = xl.getObjectInfoDir(bucket, object); e != nil { + if oi, e = xl.getObjectInfoDir(ctx, bucket, object); e != nil { return oi, toObjectErr(e, bucket, object) } return oi, nil } - info, err := xl.getObjectInfo(bucket, object) + info, err := xl.getObjectInfo(ctx, bucket, object) if err != nil { return oi, toObjectErr(err, bucket, object) } @@ -370,9 +375,9 @@ func (xl xlObjects) GetObjectInfo(ctx context.Context, bucket, object string) (o } // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo. -func (xl xlObjects) getObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) { +func (xl xlObjects) getObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error) { // Read metadata associated with the object from all disks. - metaArr, errs := readAllXLMetadata(xl.getDisks(), bucket, object) + metaArr, errs := readAllXLMetadata(ctx, xl.getDisks(), bucket, object) // get Quorum for this object readQuorum, _, err := objectQuorumFromMeta(xl, metaArr, errs) @@ -380,7 +385,7 @@ func (xl xlObjects) getObjectInfo(bucket, object string) (objInfo ObjectInfo, er return objInfo, toObjectErr(err, bucket, object) } - if reducedErr := reduceReadQuorumErrs(errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil { + if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil { return objInfo, toObjectErr(reducedErr, bucket, object) } @@ -391,7 +396,7 @@ func (xl xlObjects) getObjectInfo(bucket, object string) (objInfo ObjectInfo, er modTime, _ := commonTime(modTimes) // Pick latest valid metadata. - xlMeta, err := pickValidXLMeta(metaArr, modTime) + xlMeta, err := pickValidXLMeta(ctx, metaArr, modTime) if err != nil { return objInfo, err } @@ -428,7 +433,7 @@ func undoRename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry str // rename - common function that renamePart and renameObject use to rename // the respective underlying storage layer representations. -func rename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, isDir bool, writeQuorum int) ([]StorageAPI, error) { +func rename(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, isDir bool, writeQuorum int) ([]StorageAPI, error) { // Initialize sync waitgroup. var wg = &sync.WaitGroup{} @@ -451,7 +456,8 @@ func rename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, defer wg.Done() err := disk.RenameFile(srcBucket, srcEntry, dstBucket, dstEntry) if err != nil && err != errFileNotFound { - errs[index] = errors.Trace(err) + logger.LogIf(ctx, err) + errs[index] = err } }(index, disk) } @@ -461,7 +467,7 @@ func rename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, // We can safely allow RenameFile errors up to len(xl.getDisks()) - writeQuorum // otherwise return failure. Cleanup successful renames. - err := reduceWriteQuorumErrs(errs, objectOpIgnoredErrs, writeQuorum) + err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum) if errors.Cause(err) == errXLWriteQuorum { // Undo all the partial rename operations. undoRename(disks, srcBucket, srcEntry, dstBucket, dstEntry, isDir, errs) @@ -473,18 +479,18 @@ func rename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, // across all disks in parallel. Additionally if we have errors and do // not have a readQuorum partially renamed files are renamed back to // its proper location. -func renamePart(disks []StorageAPI, srcBucket, srcPart, dstBucket, dstPart string, quorum int) ([]StorageAPI, error) { +func renamePart(ctx context.Context, disks []StorageAPI, srcBucket, srcPart, dstBucket, dstPart string, quorum int) ([]StorageAPI, error) { isDir := false - return rename(disks, srcBucket, srcPart, dstBucket, dstPart, isDir, quorum) + return rename(ctx, disks, srcBucket, srcPart, dstBucket, dstPart, isDir, quorum) } // renameObject - renames all source objects to destination object // across all disks in parallel. Additionally if we have errors and do // not have a readQuorum partially renamed files are renamed back to // its proper location. -func renameObject(disks []StorageAPI, srcBucket, srcObject, dstBucket, dstObject string, quorum int) ([]StorageAPI, error) { +func renameObject(ctx context.Context, disks []StorageAPI, srcBucket, srcObject, dstBucket, dstObject string, quorum int) ([]StorageAPI, error) { isDir := true - return rename(disks, srcBucket, srcObject, dstBucket, dstObject, isDir, quorum) + return rename(ctx, disks, srcBucket, srcObject, dstBucket, dstObject, isDir, quorum) } // PutObject - creates an object upon reading from the input stream @@ -493,7 +499,7 @@ func renameObject(disks []StorageAPI, srcBucket, srcObject, dstBucket, dstObject // object operations. func (xl xlObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) { // Validate put object input args. - if err = checkPutObjectArgs(bucket, object, xl, data.Size()); err != nil { + if err = checkPutObjectArgs(ctx, bucket, object, xl, data.Size()); err != nil { return ObjectInfo{}, err } // Lock the object. @@ -502,11 +508,11 @@ func (xl xlObjects) PutObject(ctx context.Context, bucket string, object string, return objInfo, err } defer objectLock.Unlock() - return xl.putObject(bucket, object, data, metadata) + return xl.putObject(ctx, bucket, object, data, metadata) } // putObject wrapper for xl PutObject -func (xl xlObjects) putObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) { +func (xl xlObjects) putObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) { uniqueID := mustGetUUID() tempObj := uniqueID @@ -525,7 +531,7 @@ func (xl xlObjects) putObject(bucket string, object string, data *hash.Reader, m // Delete temporary object in the event of failure. // If PutObject succeeded there would be no temporary // object to delete. - defer xl.deleteObject(minioMetaTmpBucket, tempObj) + defer xl.deleteObject(ctx, minioMetaTmpBucket, tempObj) // This is a special case with size as '0' and object ends with // a slash separator, we treat it like a valid operation and @@ -534,16 +540,16 @@ func (xl xlObjects) putObject(bucket string, object string, data *hash.Reader, m // Check if an object is present as one of the parent dir. // -- FIXME. (needs a new kind of lock). // -- FIXME (this also causes performance issue when disks are down). - if xl.parentDirIsObject(bucket, path.Dir(object)) { - return ObjectInfo{}, toObjectErr(errors.Trace(errFileAccessDenied), bucket, object) + if xl.parentDirIsObject(ctx, bucket, path.Dir(object)) { + return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object) } - if err = xl.putObjectDir(minioMetaTmpBucket, tempObj, writeQuorum); err != nil { - return ObjectInfo{}, toObjectErr(errors.Trace(err), bucket, object) + if err = xl.putObjectDir(ctx, minioMetaTmpBucket, tempObj, writeQuorum); err != nil { + return ObjectInfo{}, toObjectErr(err, bucket, object) } // Rename the successfully written temporary object to final location. - if _, err = renameObject(xl.getDisks(), minioMetaTmpBucket, tempObj, bucket, object, writeQuorum); err != nil { + if _, err = renameObject(ctx, xl.getDisks(), minioMetaTmpBucket, tempObj, bucket, object, writeQuorum); err != nil { return ObjectInfo{}, toObjectErr(err, bucket, object) } @@ -551,20 +557,21 @@ func (xl xlObjects) putObject(bucket string, object string, data *hash.Reader, m } // Validate put object input args. - if err = checkPutObjectArgs(bucket, object, xl, data.Size()); err != nil { + if err = checkPutObjectArgs(ctx, bucket, object, xl, data.Size()); err != nil { return ObjectInfo{}, err } // Validate input data size and it can never be less than zero. if data.Size() < 0 { - return ObjectInfo{}, toObjectErr(errors.Trace(errInvalidArgument)) + logger.LogIf(ctx, errInvalidArgument) + return ObjectInfo{}, toObjectErr(errInvalidArgument) } // Check if an object is present as one of the parent dir. // -- FIXME. (needs a new kind of lock). // -- FIXME (this also causes performance issue when disks are down). - if xl.parentDirIsObject(bucket, path.Dir(object)) { - return ObjectInfo{}, toObjectErr(errors.Trace(errFileAccessDenied), bucket, object) + if xl.parentDirIsObject(ctx, bucket, path.Dir(object)) { + return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object) } // Limit the reader to its provided size if specified. @@ -586,7 +593,7 @@ func (xl xlObjects) putObject(bucket string, object string, data *hash.Reader, m // Total size of the written object var sizeWritten int64 - storage, err := NewErasureStorage(onlineDisks, xlMeta.Erasure.DataBlocks, xlMeta.Erasure.ParityBlocks, xlMeta.Erasure.BlockSize) + storage, err := NewErasureStorage(ctx, onlineDisks, xlMeta.Erasure.DataBlocks, xlMeta.Erasure.ParityBlocks, xlMeta.Erasure.BlockSize) if err != nil { return ObjectInfo{}, toObjectErr(err, bucket, object) } @@ -604,7 +611,7 @@ func (xl xlObjects) putObject(bucket string, object string, data *hash.Reader, m // Calculate the size of the current part. var curPartSize int64 - curPartSize, err = calculatePartSizeFromIdx(data.Size(), globalPutPartSize, partIdx) + curPartSize, err = calculatePartSizeFromIdx(ctx, data.Size(), globalPutPartSize, partIdx) if err != nil { return ObjectInfo{}, toObjectErr(err, bucket, object) } @@ -613,7 +620,7 @@ func (xl xlObjects) putObject(bucket string, object string, data *hash.Reader, m // This is only an optimization. var curPartReader io.Reader if curPartSize > 0 { - pErr := xl.prepareFile(minioMetaTmpBucket, tempErasureObj, curPartSize, storage.disks, xlMeta.Erasure.BlockSize, xlMeta.Erasure.DataBlocks, writeQuorum) + pErr := xl.prepareFile(ctx, minioMetaTmpBucket, tempErasureObj, curPartSize, storage.disks, xlMeta.Erasure.BlockSize, xlMeta.Erasure.DataBlocks, writeQuorum) if pErr != nil { return ObjectInfo{}, toObjectErr(pErr, bucket, object) } @@ -625,7 +632,7 @@ func (xl xlObjects) putObject(bucket string, object string, data *hash.Reader, m curPartReader = reader } - file, erasureErr := storage.CreateFile(curPartReader, minioMetaTmpBucket, + file, erasureErr := storage.CreateFile(ctx, curPartReader, minioMetaTmpBucket, tempErasureObj, buffer, DefaultBitrotAlgorithm, writeQuorum) if erasureErr != nil { return ObjectInfo{}, toObjectErr(erasureErr, minioMetaTmpBucket, tempErasureObj) @@ -634,7 +641,8 @@ func (xl xlObjects) putObject(bucket string, object string, data *hash.Reader, m // Should return IncompleteBody{} error when reader has fewer bytes // than specified in request header. if file.Size < curPartSize { - return ObjectInfo{}, errors.Trace(IncompleteBody{}) + logger.LogIf(ctx, IncompleteBody{}) + return ObjectInfo{}, IncompleteBody{} } // Update the total written size @@ -669,12 +677,12 @@ func (xl xlObjects) putObject(bucket string, object string, data *hash.Reader, m newUniqueID := mustGetUUID() // Delete successfully renamed object. - defer xl.deleteObject(minioMetaTmpBucket, newUniqueID) + defer xl.deleteObject(ctx, minioMetaTmpBucket, newUniqueID) // NOTE: Do not use online disks slice here. // The reason is that existing object should be purged // regardless of `xl.json` status and rolled back in case of errors. - _, err = renameObject(xl.getDisks(), bucket, object, minioMetaTmpBucket, newUniqueID, writeQuorum) + _, err = renameObject(ctx, xl.getDisks(), bucket, object, minioMetaTmpBucket, newUniqueID, writeQuorum) if err != nil { return ObjectInfo{}, toObjectErr(err, bucket, object) } @@ -689,19 +697,20 @@ func (xl xlObjects) putObject(bucket string, object string, data *hash.Reader, m } // Write unique `xl.json` for each disk. - if onlineDisks, err = writeUniqueXLMetadata(onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, writeQuorum); err != nil { + if onlineDisks, err = writeUniqueXLMetadata(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, writeQuorum); err != nil { return ObjectInfo{}, toObjectErr(err, bucket, object) } // Deny if WORM is enabled if globalWORMEnabled { if xl.isObject(bucket, object) { - return ObjectInfo{}, errors.Trace(ObjectAlreadyExists{Bucket: bucket, Object: object}) + logger.LogIf(ctx, ObjectAlreadyExists{Bucket: bucket, Object: object}) + return ObjectInfo{}, ObjectAlreadyExists{Bucket: bucket, Object: object} } } // Rename the successfully written temporary object to final location. - if _, err = renameObject(onlineDisks, minioMetaTmpBucket, tempObj, bucket, object, writeQuorum); err != nil { + if _, err = renameObject(ctx, onlineDisks, minioMetaTmpBucket, tempObj, bucket, object, writeQuorum); err != nil { return ObjectInfo{}, toObjectErr(err, bucket, object) } @@ -728,7 +737,7 @@ func (xl xlObjects) putObject(bucket string, object string, data *hash.Reader, m // deleteObject - wrapper for delete object, deletes an object from // all the disks in parallel, including `xl.json` associated with the // object. -func (xl xlObjects) deleteObject(bucket, object string) error { +func (xl xlObjects) deleteObject(ctx context.Context, bucket, object string) error { // Initialize sync waitgroup. var wg = &sync.WaitGroup{} @@ -739,13 +748,13 @@ func (xl xlObjects) deleteObject(bucket, object string) error { if !isDir { // Read metadata associated with the object from all disks. - metaArr, errs := readAllXLMetadata(xl.getDisks(), bucket, object) + metaArr, errs := readAllXLMetadata(ctx, xl.getDisks(), bucket, object) // get Quorum for this object _, writeQuorum, err = objectQuorumFromMeta(xl, metaArr, errs) if err != nil { return err } - err = reduceWriteQuorumErrs(errs, objectOpIgnoredErrs, writeQuorum) + err = reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum) if err != nil { return err } @@ -759,7 +768,8 @@ func (xl xlObjects) deleteObject(bucket, object string) error { for index, disk := range xl.getDisks() { if disk == nil { - dErrs[index] = errors.Trace(errDiskNotFound) + logger.LogIf(ctx, errDiskNotFound) + dErrs[index] = errDiskNotFound continue } wg.Add(1) @@ -771,7 +781,7 @@ func (xl xlObjects) deleteObject(bucket, object string) error { // and will succeed only if that directory is empty. e = disk.DeleteFile(bucket, object) } else { - e = cleanupDir(disk, bucket, object) + e = cleanupDir(ctx, disk, bucket, object) } if e != nil && errors.Cause(e) != errVolumeNotFound { dErrs[index] = e @@ -783,7 +793,7 @@ func (xl xlObjects) deleteObject(bucket, object string) error { wg.Wait() // return errors if any during deletion - return reduceWriteQuorumErrs(dErrs, objectOpIgnoredErrs, writeQuorum) + return reduceWriteQuorumErrs(ctx, dErrs, objectOpIgnoredErrs, writeQuorum) } // DeleteObject - deletes an object, this call doesn't necessary reply @@ -797,24 +807,25 @@ func (xl xlObjects) DeleteObject(ctx context.Context, bucket, object string) (er } defer objectLock.Unlock() - if err = checkDelObjArgs(bucket, object); err != nil { + if err = checkDelObjArgs(ctx, bucket, object); err != nil { return err } if hasSuffix(object, slashSeparator) { // Delete the object on all disks. - if err = xl.deleteObject(bucket, object); err != nil { + if err = xl.deleteObject(ctx, bucket, object); err != nil { return toObjectErr(err, bucket, object) } } // Validate object exists. if !xl.isObject(bucket, object) { - return errors.Trace(ObjectNotFound{bucket, object}) + logger.LogIf(ctx, ObjectNotFound{bucket, object}) + return ObjectNotFound{bucket, object} } // else proceed to delete the object. // Delete the object on all disks. - if err = xl.deleteObject(bucket, object); err != nil { + if err = xl.deleteObject(ctx, bucket, object); err != nil { return toObjectErr(err, bucket, object) } diff --git a/cmd/xl-v1-object_test.go b/cmd/xl-v1-object_test.go index 4657b5b66..a0a3aa6cd 100644 --- a/cmd/xl-v1-object_test.go +++ b/cmd/xl-v1-object_test.go @@ -302,7 +302,7 @@ func TestHealing(t *testing.T) { } disk := xl.storageDisks[0] - xlMetaPreHeal, err := readXLMeta(disk, bucket, object) + xlMetaPreHeal, err := readXLMeta(context.Background(), disk, bucket, object) if err != nil { t.Fatal(err) } @@ -319,7 +319,7 @@ func TestHealing(t *testing.T) { t.Fatal(err) } - xlMetaPostHeal, err := readXLMeta(disk, bucket, object) + xlMetaPostHeal, err := readXLMeta(context.Background(), disk, bucket, object) if err != nil { t.Fatal(err) } @@ -333,7 +333,7 @@ func TestHealing(t *testing.T) { // gone down when an object was replaced by a new object. xlMetaOutDated := xlMetaPreHeal xlMetaOutDated.Stat.ModTime = time.Now() - err = writeXLMetadata(disk, bucket, object, xlMetaOutDated) + err = writeXLMetadata(context.Background(), disk, bucket, object, xlMetaOutDated) if err != nil { t.Fatal(err) } @@ -343,7 +343,7 @@ func TestHealing(t *testing.T) { t.Fatal(err) } - xlMetaPostHeal, err = readXLMeta(disk, bucket, object) + xlMetaPostHeal, err = readXLMeta(context.Background(), disk, bucket, object) if err != nil { t.Fatal(err) } diff --git a/cmd/xl-v1-utils.go b/cmd/xl-v1-utils.go index 565948276..5325981b3 100644 --- a/cmd/xl-v1-utils.go +++ b/cmd/xl-v1-utils.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "encoding/hex" "errors" "hash/crc32" @@ -24,6 +25,7 @@ import ( "sync" "time" + "github.com/minio/minio/cmd/logger" errors2 "github.com/minio/minio/pkg/errors" "github.com/tidwall/gjson" ) @@ -36,7 +38,6 @@ import ( // maximal values would occur quorum or more number of times. func reduceErrs(errs []error, ignoredErrs []error) (maxCount int, maxErr error) { errorCounts := make(map[error]int) - errs = errors2.Causes(errs) for _, err := range errs { if errors2.IsErrIgnored(err, ignoredErrs...) { continue @@ -63,34 +64,26 @@ func reduceErrs(errs []error, ignoredErrs []error) (maxCount int, maxErr error) // reduceQuorumErrs behaves like reduceErrs by only for returning // values of maximally occurring errors validated against a generic // quorum number that can be read or write quorum depending on usage. -// Additionally a special error is provided to be returned in case -// quorum is not satisfied. -func reduceQuorumErrs(errs []error, ignoredErrs []error, quorum int, quorumErr error) (maxErr error) { - var maxCount int - maxCount, maxErr = reduceErrs(errs, ignoredErrs) - switch { - case maxErr == nil && maxCount >= quorum: - // Success in quorum. - case maxErr != nil && maxCount >= quorum: - // Errors in quorum. - maxErr = errors2.Trace(maxErr, errs...) - default: - // No quorum satisfied. - maxErr = errors2.Trace(quorumErr, errs...) +func reduceQuorumErrs(ctx context.Context, errs []error, ignoredErrs []error, quorum int, quorumErr error) error { + maxCount, maxErr := reduceErrs(errs, ignoredErrs) + if maxCount >= quorum { + logger.LogIf(ctx, maxErr) + return maxErr } - return + logger.LogIf(ctx, quorumErr) + return quorumErr } // reduceReadQuorumErrs behaves like reduceErrs but only for returning // values of maximally occurring errors validated against readQuorum. -func reduceReadQuorumErrs(errs []error, ignoredErrs []error, readQuorum int) (maxErr error) { - return reduceQuorumErrs(errs, ignoredErrs, readQuorum, errXLReadQuorum) +func reduceReadQuorumErrs(ctx context.Context, errs []error, ignoredErrs []error, readQuorum int) (maxErr error) { + return reduceQuorumErrs(ctx, errs, ignoredErrs, readQuorum, errXLReadQuorum) } // reduceWriteQuorumErrs behaves like reduceErrs but only for returning // values of maximally occurring errors validated against writeQuorum. -func reduceWriteQuorumErrs(errs []error, ignoredErrs []error, writeQuorum int) (maxErr error) { - return reduceQuorumErrs(errs, ignoredErrs, writeQuorum, errXLWriteQuorum) +func reduceWriteQuorumErrs(ctx context.Context, errs []error, ignoredErrs []error, writeQuorum int) (maxErr error) { + return reduceQuorumErrs(ctx, errs, ignoredErrs, writeQuorum, errXLWriteQuorum) } // Similar to 'len(slice)' but returns the actual elements count @@ -153,7 +146,7 @@ func parseXLRelease(xlMetaBuf []byte) string { return gjson.GetBytes(xlMetaBuf, "minio.release").String() } -func parseXLErasureInfo(xlMetaBuf []byte) (ErasureInfo, error) { +func parseXLErasureInfo(ctx context.Context, xlMetaBuf []byte) (ErasureInfo, error) { erasure := ErasureInfo{} erasureResult := gjson.GetBytes(xlMetaBuf, "erasure") // parse the xlV1Meta.Erasure.Distribution. @@ -177,11 +170,13 @@ func parseXLErasureInfo(xlMetaBuf []byte) (ErasureInfo, error) { for i, v := range checkSumsResult { algorithm := BitrotAlgorithmFromString(v.Get("algorithm").String()) if !algorithm.Available() { - return erasure, errors2.Trace(errBitrotHashAlgoInvalid) + logger.LogIf(ctx, errBitrotHashAlgoInvalid) + return erasure, errBitrotHashAlgoInvalid } hash, err := hex.DecodeString(v.Get("hash").String()) if err != nil { - return erasure, errors2.Trace(err) + logger.LogIf(ctx, err) + return erasure, err } checkSums[i] = ChecksumInfo{Name: v.Get("name").String(), Algorithm: algorithm, Hash: hash} } @@ -215,7 +210,7 @@ func parseXLMetaMap(xlMetaBuf []byte) map[string]string { } // Constructs XLMetaV1 using `gjson` lib to retrieve each field. -func xlMetaV1UnmarshalJSON(xlMetaBuf []byte) (xlMeta xlMetaV1, e error) { +func xlMetaV1UnmarshalJSON(ctx context.Context, xlMetaBuf []byte) (xlMeta xlMetaV1, e error) { // obtain version. xlMeta.Version = parseXLVersion(xlMetaBuf) // obtain format. @@ -223,12 +218,13 @@ func xlMetaV1UnmarshalJSON(xlMetaBuf []byte) (xlMeta xlMetaV1, e error) { // Parse xlMetaV1.Stat . stat, err := parseXLStat(xlMetaBuf) if err != nil { + logger.LogIf(ctx, err) return xlMeta, err } xlMeta.Stat = stat // parse the xlV1Meta.Erasure fields. - xlMeta.Erasure, err = parseXLErasureInfo(xlMetaBuf) + xlMeta.Erasure, err = parseXLErasureInfo(ctx, xlMetaBuf) if err != nil { return xlMeta, err } @@ -244,11 +240,12 @@ func xlMetaV1UnmarshalJSON(xlMetaBuf []byte) (xlMeta xlMetaV1, e error) { } // read xl.json from the given disk, parse and return xlV1MetaV1.Parts. -func readXLMetaParts(disk StorageAPI, bucket string, object string) ([]objectPartInfo, map[string]string, error) { +func readXLMetaParts(ctx context.Context, disk StorageAPI, bucket string, object string) ([]objectPartInfo, map[string]string, error) { // Reads entire `xl.json`. xlMetaBuf, err := disk.ReadAll(bucket, path.Join(object, xlMetaJSONFile)) if err != nil { - return nil, nil, errors2.Trace(err) + logger.LogIf(ctx, err) + return nil, nil, err } // obtain xlMetaV1{}.Partsusing `github.com/tidwall/gjson`. @@ -259,11 +256,12 @@ func readXLMetaParts(disk StorageAPI, bucket string, object string) ([]objectPar } // read xl.json from the given disk and parse xlV1Meta.Stat and xlV1Meta.Meta using gjson. -func readXLMetaStat(disk StorageAPI, bucket string, object string) (si statInfo, mp map[string]string, e error) { +func readXLMetaStat(ctx context.Context, disk StorageAPI, bucket string, object string) (si statInfo, mp map[string]string, e error) { // Reads entire `xl.json`. xlMetaBuf, err := disk.ReadAll(bucket, path.Join(object, xlMetaJSONFile)) if err != nil { - return si, nil, errors2.Trace(err) + logger.LogIf(ctx, err) + return si, nil, err } // obtain version. @@ -275,7 +273,8 @@ func readXLMetaStat(disk StorageAPI, bucket string, object string) (si statInfo, // Validate if the xl.json we read is sane, return corrupted format. if !isXLMetaFormatValid(xlVersion, xlFormat) { // For version mismatchs and unrecognized format, return corrupted format. - return si, nil, errors2.Trace(errCorruptedFormat) + logger.LogIf(ctx, errCorruptedFormat) + return si, nil, errCorruptedFormat } // obtain xlMetaV1{}.Meta using `github.com/tidwall/gjson`. @@ -284,7 +283,8 @@ func readXLMetaStat(disk StorageAPI, bucket string, object string) (si statInfo, // obtain xlMetaV1{}.Stat using `github.com/tidwall/gjson`. xlStat, err := parseXLStat(xlMetaBuf) if err != nil { - return si, nil, errors2.Trace(err) + logger.LogIf(ctx, err) + return si, nil, err } // Return structured `xl.json`. @@ -292,16 +292,17 @@ func readXLMetaStat(disk StorageAPI, bucket string, object string) (si statInfo, } // readXLMeta reads `xl.json` and returns back XL metadata structure. -func readXLMeta(disk StorageAPI, bucket string, object string) (xlMeta xlMetaV1, err error) { +func readXLMeta(ctx context.Context, disk StorageAPI, bucket string, object string) (xlMeta xlMetaV1, err error) { // Reads entire `xl.json`. xlMetaBuf, err := disk.ReadAll(bucket, path.Join(object, xlMetaJSONFile)) if err != nil { - return xlMetaV1{}, errors2.Trace(err) + logger.LogIf(ctx, err) + return xlMetaV1{}, err } // obtain xlMetaV1{} using `github.com/tidwall/gjson`. - xlMeta, err = xlMetaV1UnmarshalJSON(xlMetaBuf) + xlMeta, err = xlMetaV1UnmarshalJSON(ctx, xlMetaBuf) if err != nil { - return xlMetaV1{}, errors2.Trace(err) + return xlMetaV1{}, err } // Return structured `xl.json`. return xlMeta, nil @@ -309,7 +310,7 @@ func readXLMeta(disk StorageAPI, bucket string, object string) (xlMeta xlMetaV1, // Reads all `xl.json` metadata as a xlMetaV1 slice. // Returns error slice indicating the failed metadata reads. -func readAllXLMetadata(disks []StorageAPI, bucket, object string) ([]xlMetaV1, []error) { +func readAllXLMetadata(ctx context.Context, disks []StorageAPI, bucket, object string) ([]xlMetaV1, []error) { errs := make([]error, len(disks)) metadataArray := make([]xlMetaV1, len(disks)) var wg = &sync.WaitGroup{} @@ -324,7 +325,7 @@ func readAllXLMetadata(disks []StorageAPI, bucket, object string) ([]xlMetaV1, [ go func(index int, disk StorageAPI) { defer wg.Done() var err error - metadataArray[index], err = readXLMeta(disk, bucket, object) + metadataArray[index], err = readXLMeta(ctx, disk, bucket, object) if err != nil { errs[index] = err return @@ -384,7 +385,7 @@ func unshuffleIndex(n int, distribution []int) int { // the corresponding error in errs slice is not nil func evalDisks(disks []StorageAPI, errs []error) []StorageAPI { if len(errs) != len(disks) { - errorIf(errors.New("unexpected disks/errors slice length"), "unable to evaluate internal disks") + logger.LogIf(context.Background(), errors.New("unexpected disks/errors slice length")) return nil } newDisks := make([]StorageAPI, len(disks)) @@ -406,15 +407,18 @@ var ( // calculatePartSizeFromIdx calculates the part size according to input index. // returns error if totalSize is -1, partSize is 0, partIndex is 0. -func calculatePartSizeFromIdx(totalSize int64, partSize int64, partIndex int) (currPartSize int64, err error) { +func calculatePartSizeFromIdx(ctx context.Context, totalSize int64, partSize int64, partIndex int) (currPartSize int64, err error) { if totalSize < 0 { - return 0, errors2.Trace(errInvalidArgument) + logger.LogIf(ctx, errInvalidArgument) + return 0, errInvalidArgument } if partSize == 0 { - return 0, errors2.Trace(errPartSizeZero) + logger.LogIf(ctx, errPartSizeZero) + return 0, errPartSizeZero } if partIndex < 1 { - return 0, errors2.Trace(errPartSizeIndex) + logger.LogIf(ctx, errPartSizeIndex) + return 0, errPartSizeIndex } if totalSize > 0 { // Compute the total count of parts diff --git a/cmd/xl-v1-utils_test.go b/cmd/xl-v1-utils_test.go index ec558de26..e07172d2a 100644 --- a/cmd/xl-v1-utils_test.go +++ b/cmd/xl-v1-utils_test.go @@ -18,6 +18,7 @@ package cmd import ( "bytes" + "context" "encoding/hex" "encoding/json" "reflect" @@ -91,11 +92,11 @@ func TestReduceErrs(t *testing.T) { } // Validates list of all the testcases for returning valid errors. for i, testCase := range testCases { - gotErr := reduceReadQuorumErrs(testCase.errs, testCase.ignoredErrs, 5) + gotErr := reduceReadQuorumErrs(context.Background(), testCase.errs, testCase.ignoredErrs, 5) if errors.Cause(gotErr) != testCase.err { t.Errorf("Test %d : expected %s, got %s", i+1, testCase.err, gotErr) } - gotNewErr := reduceWriteQuorumErrs(testCase.errs, testCase.ignoredErrs, 6) + gotNewErr := reduceWriteQuorumErrs(context.Background(), testCase.errs, testCase.ignoredErrs, 6) if errors.Cause(gotNewErr) != errXLWriteQuorum { t.Errorf("Test %d : expected %s, got %s", i+1, errXLWriteQuorum, gotErr) } @@ -312,7 +313,7 @@ func TestGetXLMetaV1GJson1(t *testing.T) { t.Errorf("Unmarshalling failed: %v", err) } - gjsonXLMeta, err := xlMetaV1UnmarshalJSON(xlMetaJSON) + gjsonXLMeta, err := xlMetaV1UnmarshalJSON(context.Background(), xlMetaJSON) if err != nil { t.Errorf("gjson parsing of XLMeta failed: %v", err) } @@ -329,7 +330,7 @@ func TestGetXLMetaV1GJson10(t *testing.T) { if err := json.Unmarshal(xlMetaJSON, &unMarshalXLMeta); err != nil { t.Errorf("Unmarshalling failed: %v", err) } - gjsonXLMeta, err := xlMetaV1UnmarshalJSON(xlMetaJSON) + gjsonXLMeta, err := xlMetaV1UnmarshalJSON(context.Background(), xlMetaJSON) if err != nil { t.Errorf("gjson parsing of XLMeta failed: %v", err) } @@ -359,7 +360,7 @@ func TestGetPartSizeFromIdx(t *testing.T) { } for i, testCase := range testCases { - s, err := calculatePartSizeFromIdx(testCase.totalSize, testCase.partSize, testCase.partIndex) + s, err := calculatePartSizeFromIdx(context.Background(), testCase.totalSize, testCase.partSize, testCase.partIndex) if err != nil { t.Errorf("Test %d: Expected to pass but failed. %s", i+1, err) } @@ -383,7 +384,7 @@ func TestGetPartSizeFromIdx(t *testing.T) { } for i, testCaseFailure := range testCasesFailure { - _, err := calculatePartSizeFromIdx(testCaseFailure.totalSize, testCaseFailure.partSize, testCaseFailure.partIndex) + _, err := calculatePartSizeFromIdx(context.Background(), testCaseFailure.totalSize, testCaseFailure.partSize, testCaseFailure.partIndex) if err == nil { t.Errorf("Test %d: Expected to failed but passed. %s", i+1, err) } diff --git a/cmd/xl-v1.go b/cmd/xl-v1.go index 0f835fb06..c26f539ec 100644 --- a/cmd/xl-v1.go +++ b/cmd/xl-v1.go @@ -21,6 +21,7 @@ import ( "sort" "time" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/bpool" "github.com/minio/minio/pkg/disk" "github.com/minio/minio/pkg/errors" @@ -140,7 +141,7 @@ func getDisksInfo(disks []StorageAPI) (disksInfo []disk.Info, onlineDisks int, o } info, err := storageDisk.DiskInfo() if err != nil { - errorIf(err, "Unable to fetch disk info for %#v", storageDisk) + logger.LogIf(context.Background(), err) if errors.IsErr(err, baseErrs...) { offlineDisks++ continue diff --git a/docs/shared-backend/DESIGN.md b/docs/shared-backend/DESIGN.md index fe2f3218d..507a3895f 100644 --- a/docs/shared-backend/DESIGN.md +++ b/docs/shared-backend/DESIGN.md @@ -84,7 +84,7 @@ GetObject() holds a read lock on `fs.json`. fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fsMetaJSONFile) rlk, err := fs.rwPool.Open(fsMetaPath) if err != nil { - return toObjectErr(errors.Trace(err), bucket, object) + return toObjectErr(err, bucket, object) } defer rlk.Close() diff --git a/docs/zh_CN/shared-backend/DESIGN.md b/docs/zh_CN/shared-backend/DESIGN.md index 2e3768811..4683a1e5f 100644 --- a/docs/zh_CN/shared-backend/DESIGN.md +++ b/docs/zh_CN/shared-backend/DESIGN.md @@ -88,7 +88,7 @@ GetObject()持有`fs.json`的一个读锁。 fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fsMetaJSONFile) rlk, err := fs.rwPool.Open(fsMetaPath) if err != nil { - return toObjectErr(errors.Trace(err), bucket, object) + return toObjectErr(err, bucket, object) } defer rlk.Close()