locks: Ensure local lock removal after a failed refresh (#12979) (#13183)

In the event when a lock is not refreshed in the cluster, this latter
will be automatically removed in the subsequent cleanup of non
refreshed locks routine, but it forgot to clean the local server,
hence having the same weird stale locks present.

This commit will remove the lock locally also in remote nodes, if
removing a lock from a remote node will fail, it will be anyway
removed later in the locks cleanup routine.

Makefile

@@ -71,9 +71,12 @@ build: checks
 	@echo "Building minio binary to './minio'"
 	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 
-docker: checks
+hotfix: LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \
+    sed 's#RELEASE\.\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)T\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)Z#\1-\2-\3T\4:\5:\6Z#'))
+hotfix: install
+
+docker: checks hotfix
 	@echo "Building minio docker image '$(TAG)'"
-	@GOOS=linux GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@docker build -t $(TAG) . -f Dockerfile.dev
 
 # Builds minio and installs it to $GOPATH/bin.

buildscripts/gen-ldflags.go

@@ -44,10 +44,21 @@ func releaseTag(version string) string {
 		relPrefix = prefix
 	}
 
+	relSuffix := ""
+	if hotfix := os.Getenv("MINIO_HOTFIX"); hotfix != "" {
+		relSuffix = hotfix
+	}
+
 	relTag := strings.Replace(version, " ", "-", -1)
 	relTag = strings.Replace(relTag, ":", "-", -1)
 	relTag = strings.Replace(relTag, ",", "", -1)
-	return relPrefix + "." + relTag
+	relTag = relPrefix + "." + relTag
+
+	if relSuffix != "" {
+		relTag += "." + relSuffix
+	}
+
+	return relTag
 }
 
 // commitID returns the abbreviated commit-id hash of the last commit.
@@ -68,5 +79,12 @@ func commitID() string {
 }
 
 func main() {
-	fmt.Println(genLDFlags(time.Now().UTC().Format(time.RFC3339)))
+	var version string
+	if len(os.Args) > 1 {
+		version = os.Args[1]
+	} else {
+		version = time.Now().UTC().Format(time.RFC3339)
+	}
+
+	fmt.Println(genLDFlags(version))
 }

cmd/acl-handlers.go

@@ -61,7 +61,7 @@ type accessControlPolicy struct {
 func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketACL")
 
-	defer logger.AuditLog(w, r, "PutBucketACL", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -125,7 +125,7 @@ func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.
 func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketACL")
 
-	defer logger.AuditLog(w, r, "GetBucketACL", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -176,7 +176,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 func (api objectAPIHandlers) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutObjectACL")
 
-	defer logger.AuditLog(w, r, "PutObjectACL", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -240,7 +240,7 @@ func (api objectAPIHandlers) PutObjectACLHandler(w http.ResponseWriter, r *http.
 func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetObjectACL")
 
-	defer logger.AuditLog(w, r, "GetObjectACL", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]

cmd/admin-bucket-handlers.go

@@ -44,7 +44,7 @@ const (
 func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketQuotaConfig")
 
-	defer logger.AuditLog(w, r, "PutBucketQuotaConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketQuotaAdminAction)
 	if objectAPI == nil {
@@ -90,7 +90,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketQuotaConfig")
 
-	defer logger.AuditLog(w, r, "GetBucketQuotaConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetBucketQuotaAdminAction)
 	if objectAPI == nil {
@@ -125,7 +125,7 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
 func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetBucketTarget")
 
-	defer logger.AuditLog(w, r, "SetBucketTarget", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -214,7 +214,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListBucketTargets")
 
-	defer logger.AuditLog(w, r, "ListBucketTargets", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	arnType := vars["type"]
@@ -250,7 +250,7 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
 func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RemoveBucketTarget")
 
-	defer logger.AuditLog(w, r, "RemoveBucketTarget", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	arn := vars["arn"]

cmd/admin-handlers-config-kv.go

@@ -62,7 +62,7 @@ func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *htt
 func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteConfigKV")
 
-	defer logger.AuditLog(w, r, "DeleteConfigKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -104,7 +104,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetConfigKV")
 
-	defer logger.AuditLog(w, r, "SetConfigKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -165,7 +165,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetConfigKV")
 
-	defer logger.AuditLog(w, r, "GetConfigKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -203,7 +203,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ClearConfigHistoryKV")
 
-	defer logger.AuditLog(w, r, "ClearConfigHistoryKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -240,7 +240,7 @@ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *
 func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RestoreConfigHistoryKV")
 
-	defer logger.AuditLog(w, r, "RestoreConfigHistoryKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -288,7 +288,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListConfigHistoryKV")
 
-	defer logger.AuditLog(w, r, "ListConfigHistoryKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -328,7 +328,7 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
 func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HelpConfigKV")
 
-	defer logger.AuditLog(w, r, "HelpHistoryKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -356,7 +356,7 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetConfig")
 
-	defer logger.AuditLog(w, r, "SetConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -413,7 +413,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetConfig")
 
-	defer logger.AuditLog(w, r, "GetConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {

cmd/admin-handlers-users.go

@@ -22,6 +22,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"strings"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
@@ -55,7 +56,7 @@ func validateAdminUsersReq(ctx context.Context, w http.ResponseWriter, r *http.R
 func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RemoveUser")
 
-	defer logger.AuditLog(w, r, "RemoveUser", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.DeleteUserAdminAction)
 	if objectAPI == nil {
@@ -93,7 +94,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListUsers")
 
-	defer logger.AuditLog(w, r, "ListUsers", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.ListUsersAdminAction)
 	if objectAPI == nil {
@@ -127,7 +128,7 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetUserInfo")
 
-	defer logger.AuditLog(w, r, "GetUserInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetUserAdminAction)
 	if objectAPI == nil {
@@ -156,7 +157,7 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "UpdateGroupMembers")
 
-	defer logger.AuditLog(w, r, "UpdateGroupMembers", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.AddUserToGroupAdminAction)
 	if objectAPI == nil {
@@ -201,7 +202,7 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetGroup")
 
-	defer logger.AuditLog(w, r, "GetGroup", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetGroupAdminAction)
 	if objectAPI == nil {
@@ -230,7 +231,7 @@ func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListGroups")
 
-	defer logger.AuditLog(w, r, "ListGroups", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListGroupsAdminAction)
 	if objectAPI == nil {
@@ -256,7 +257,7 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetGroupStatus")
 
-	defer logger.AuditLog(w, r, "SetGroupStatus", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.EnableGroupAdminAction)
 	if objectAPI == nil {
@@ -293,7 +294,7 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetUserStatus")
 
-	defer logger.AuditLog(w, r, "SetUserStatus", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.EnableUserAdminAction)
 	if objectAPI == nil {
@@ -328,7 +329,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddUser")
 
-	defer logger.AuditLog(w, r, "AddUser", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.CreateUserAdminAction)
 	if objectAPI == nil {
@@ -383,7 +384,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddServiceAccount")
 
-	defer logger.AuditLog(w, r, "AddServiceAccount", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -462,7 +463,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListServiceAccounts")
 
-	defer logger.AuditLog(w, r, "ListServiceAccounts", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -517,7 +518,7 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteServiceAccount")
 
-	defer logger.AuditLog(w, r, "DeleteServiceAccount", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -576,7 +577,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 func (a adminAPIHandlers) AccountUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AccountUsageInfo")
 
-	defer logger.AuditLog(w, r, "AccountUsageInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -627,10 +628,29 @@ func (a adminAPIHandlers) AccountUsageInfoHandler(w http.ResponseWriter, r *http
 		return rd, wr
 	}
 
-	buckets, err := objectAPI.ListBuckets(ctx)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
+	var (
+		buckets []BucketInfo
+		err     error
+	)
+
+	q := r.URL.Query()
+
+	prefixUsageEnabled := q.Get("prefix-usage") == "true"
+	selectedBucket := q.Get("bucket")
+
+	if selectedBucket != "" {
+		bucket, err := objectAPI.GetBucketInfo(ctx, selectedBucket)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		buckets = append(buckets, bucket)
+	} else {
+		buckets, err = objectAPI.ListBuckets(ctx)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
 	}
 
 	// Load the latest calculated data usage
@@ -649,26 +669,59 @@ func (a adminAPIHandlers) AccountUsageInfoHandler(w http.ResponseWriter, r *http
 		AccountName: accountName,
 	}
 
+	type bucketAccessInfo struct {
+		info        BucketInfo
+		read, write bool
+	}
+
+	var allowedAccessBuckets []bucketAccessInfo
 	for _, bucket := range buckets {
 		rd, wr := isAllowedAccess(bucket.Name)
 		if rd || wr {
-			var size uint64
-			// Fetch the data usage of the current bucket
-			if !dataUsageInfo.LastUpdate.IsZero() {
-				size = dataUsageInfo.BucketsUsage[bucket.Name].Size
-			}
-			acctInfo.Buckets = append(acctInfo.Buckets, madmin.BucketUsageInfo{
-				Name:    bucket.Name,
-				Created: bucket.Created,
-				Size:    size,
-				Access: madmin.AccountAccess{
-					Read:  rd,
-					Write: wr,
-				},
-			})
+			allowedAccessBuckets = append(allowedAccessBuckets, bucketAccessInfo{info: bucket, read: rd, write: wr})
 		}
 	}
 
+	var pathsUsage map[string]uint64
+	if prefixUsageEnabled {
+		pathsUsage, err = loadPathsUsageFromBackend(ctx, objectAPI, buckets)
+		if err != nil {
+			logger.LogIf(ctx, err)
+		}
+	}
+
+	for _, bucket := range allowedAccessBuckets {
+		var size uint64
+		// Fetch the data usage of the current bucket
+		if !dataUsageInfo.LastUpdate.IsZero() {
+			size = dataUsageInfo.BucketsUsage[bucket.info.Name].Size
+		}
+
+		bucketUsage := madmin.BucketUsageInfo{
+			Name:    bucket.info.Name,
+			Created: bucket.info.Created,
+			Size:    size,
+			Access: madmin.AccountAccess{
+				Read:  bucket.read,
+				Write: bucket.write,
+			},
+		}
+
+		if prefixUsageEnabled {
+			// Update prefixes usage for this bucket
+			bucketUsage.PrefixesUsage = make(map[string]uint64)
+			bucketPrefix := bucket.info.Name + slashSeparator
+			for prefix, usage := range pathsUsage {
+				if strings.HasPrefix(prefix, bucketPrefix) {
+					prefix := prefix[len(bucketPrefix):]
+					bucketUsage.PrefixesUsage[prefix] = usage
+				}
+			}
+		}
+
+		acctInfo.Buckets = append(acctInfo.Buckets, bucketUsage)
+	}
+
 	usageInfoJSON, err := json.Marshal(acctInfo)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -682,7 +735,7 @@ func (a adminAPIHandlers) AccountUsageInfoHandler(w http.ResponseWriter, r *http
 func (a adminAPIHandlers) InfoCannedPolicyV2(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "InfoCannedPolicyV2")
 
-	defer logger.AuditLog(w, r, "InfoCannedPolicyV2", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
 	if objectAPI == nil {
@@ -709,7 +762,7 @@ func (a adminAPIHandlers) InfoCannedPolicyV2(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "InfoCannedPolicy")
 
-	defer logger.AuditLog(w, r, "InfoCannedPolicy", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
 	if objectAPI == nil {
@@ -733,7 +786,7 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) ListCannedPoliciesV2(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListCannedPoliciesV2")
 
-	defer logger.AuditLog(w, r, "ListCannedPoliciesV2", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
@@ -767,7 +820,7 @@ func (a adminAPIHandlers) ListCannedPoliciesV2(w http.ResponseWriter, r *http.Re
 func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListCannedPolicies")
 
-	defer logger.AuditLog(w, r, "ListCannedPolicies", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
@@ -801,7 +854,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RemoveCannedPolicy")
 
-	defer logger.AuditLog(w, r, "RemoveCannedPolicy", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.DeletePolicyAdminAction)
 	if objectAPI == nil {
@@ -829,7 +882,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddCannedPolicy")
 
-	defer logger.AuditLog(w, r, "AddCannedPolicy", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.CreatePolicyAdminAction)
 	if objectAPI == nil {
@@ -881,7 +934,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetPolicyForUserOrGroup")
 
-	defer logger.AuditLog(w, r, "SetPolicyForUserOrGroup", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.AttachPolicyAdminAction)
 	if objectAPI == nil {

cmd/admin-handlers.go

@@ -41,6 +41,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/logger/message/log"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/dsync"
 	"github.com/minio/minio/pkg/handlers"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
@@ -77,7 +78,7 @@ func updateServer(u *url.URL, sha256Sum []byte, lrTime time.Time, mode string) (
 func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServerUpdate")
 
-	defer logger.AuditLog(w, r, "ServerUpdate", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerUpdateAdminAction)
 	if objectAPI == nil {
@@ -186,7 +187,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "Service")
 
-	defer logger.AuditLog(w, r, "Service", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	action := vars["action"]
@@ -256,9 +257,10 @@ type ServerHTTPAPIStats struct {
 // ServerHTTPStats holds all type of http operations performed to/from the server
 // including their average execution time.
 type ServerHTTPStats struct {
-	CurrentS3Requests ServerHTTPAPIStats `json:"currentS3Requests"`
-	TotalS3Requests   ServerHTTPAPIStats `json:"totalS3Requests"`
-	TotalS3Errors     ServerHTTPAPIStats `json:"totalS3Errors"`
+	CurrentS3Requests   ServerHTTPAPIStats `json:"currentS3Requests"`
+	TotalS3Requests     ServerHTTPAPIStats `json:"totalS3Requests"`
+	TotalS3Errors       ServerHTTPAPIStats `json:"totalS3Errors"`
+	TotalClientsInQueue int64              `json:"totalClientsInQueue"`
 }
 
 // ServerInfoData holds storage, connections and other
@@ -282,7 +284,7 @@ type ServerInfo struct {
 func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StorageInfo")
 
-	defer logger.AuditLog(w, r, "StorageInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.StorageInfoAdminAction)
 	if objectAPI == nil {
@@ -325,7 +327,7 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DataUsageInfo")
 
-	defer logger.AuditLog(w, r, "DataUsageInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DataUsageInfoAdminAction)
 	if objectAPI == nil {
@@ -374,10 +376,10 @@ func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
 		for _, locks := range peerLock.Locks {
 			for k, v := range locks {
 				for _, lockReqInfo := range v {
-					if val, ok := entryMap[lockReqInfo.UID]; ok {
+					if val, ok := entryMap[k]; ok {
 						val.ServerList = append(val.ServerList, peerLock.Addr)
 					} else {
-						entryMap[lockReqInfo.UID] = lriToLockEntry(lockReqInfo, k, peerLock.Addr)
+						entryMap[k] = lriToLockEntry(lockReqInfo, k, peerLock.Addr)
 					}
 				}
 			}
@@ -407,7 +409,7 @@ type PeerLocks struct {
 func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "TopLocks")
 
-	defer logger.AuditLog(w, r, "TopLocks", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.TopLocksAdminAction)
 	if objectAPI == nil {
@@ -445,6 +447,45 @@ func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
+// ForceUnlockHandler force unlocks requested resource
+func (a adminAPIHandlers) ForceUnlockHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ForceUnlock")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ForceUnlockAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	z, ok := objectAPI.(*erasureServerSets)
+	if !ok {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+
+	var args dsync.LockArgs
+	lockersMap := make(map[string]dsync.NetLocker)
+	for _, path := range strings.Split(vars["paths"], ",") {
+		if path == "" {
+			continue
+		}
+		args.Resources = append(args.Resources, path)
+		lockers, _ := z.serverSets[0].getHashedSet(path).getLockers()
+		for _, locker := range lockers {
+			if locker != nil {
+				lockersMap[locker.String()] = locker
+			}
+		}
+	}
+
+	for _, locker := range lockersMap {
+		locker.ForceUnlock(ctx, args)
+	}
+}
+
 // StartProfilingResult contains the status of the starting
 // profiling action in a given server
 type StartProfilingResult struct {
@@ -459,7 +500,7 @@ type StartProfilingResult struct {
 func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StartProfiling")
 
-	defer logger.AuditLog(w, r, "StartProfiling", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
 	if objectAPI == nil {
@@ -557,7 +598,7 @@ func (f dummyFileInfo) Sys() interface{}   { return f.sys }
 func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DownloadProfiling")
 
-	defer logger.AuditLog(w, r, "DownloadProfiling", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
 	if objectAPI == nil {
@@ -651,7 +692,7 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
 func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "Heal")
 
-	defer logger.AuditLog(w, r, "Heal", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
 	if objectAPI == nil {
@@ -862,7 +903,7 @@ func getAggregatedBackgroundHealState(ctx context.Context) (madmin.BgHealState,
 func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HealBackgroundStatus")
 
-	defer logger.AuditLog(w, r, "HealBackgroundStatus", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
 	if objectAPI == nil {
@@ -1074,7 +1115,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ConsoleLog")
 
-	defer logger.AuditLog(w, r, "ConsoleLog", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConsoleLogAdminAction)
 	if objectAPI == nil {
@@ -1145,7 +1186,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 // KMSCreateKeyHandler - POST /minio/admin/v3/kms/key/create?key-id=<master-key-id>
 func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSCreateKey")
-	defer logger.AuditLog(w, r, "KMSCreateKey", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSCreateKeyAdminAction)
 	if objectAPI == nil {
@@ -1168,7 +1209,7 @@ func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSKeyStatus")
 
-	defer logger.AuditLog(w, r, "KMSKeyStatus", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
 	if objectAPI == nil {
@@ -1241,7 +1282,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "OBDInfo")
 
-	defer logger.AuditLog(w, r, "OBDInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.OBDInfoAdminAction)
 	if objectAPI == nil {
@@ -1284,12 +1325,14 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 	deadlinedCtx, cancel := context.WithTimeout(ctx, deadline)
 	defer cancel()
 
-	nsLock := objectAPI.NewNSLock(ctx, minioMetaBucket, "obd-in-progress")
-	if err := nsLock.GetLock(newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock
+	nsLock := objectAPI.NewNSLock(minioMetaBucket, "obd-in-progress")
+	lkctx, err := nsLock.GetLock(ctx, newDynamicTimeout(deadline, deadline))
+	if err != nil { // returns a locked lock
 		errResp(err)
 		return
 	}
-	defer nsLock.Unlock()
+	ctx = lkctx.Context()
+	defer nsLock.Unlock(lkctx.Cancel)
 
 	go func() {
 		defer close(obdInfoCh)
@@ -1444,7 +1487,7 @@ func (a adminAPIHandlers) BandwidthMonitorHandler(w http.ResponseWriter, r *http
 func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServerInfo")
 
-	defer logger.AuditLog(w, r, "ServerInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction)
 	if objectAPI == nil {

cmd/admin-router.go

@@ -197,6 +197,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		// Top locks
 		if globalIsDistErasure {
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
+			adminRouter.Methods(http.MethodPost).Path(adminVersion+"/force-unlock").
+				Queries("paths", "{paths:.*}").HandlerFunc(httpTraceHdrs(adminAPI.ForceUnlockHandler))
 		}
 
 		// HTTP Trace

cmd/background-newdisks-heal-ops.go

@@ -176,7 +176,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerSets, bgSeq *
 						logger.Info("Healing disk '%s' on %s zone complete", disk, humanize.Ordinal(i+1))
 
 						if err := disk.DeleteFile(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix),
-							healingTrackerFilename); err != nil && !errors.Is(err, errFileNotFound) {
+							healingTrackerFilename, false); err != nil && !errors.Is(err, errFileNotFound) {
 							logger.LogIf(ctx, err)
 							continue
 						}

cmd/bucket-encryption-handlers.go

@@ -37,7 +37,7 @@ const (
 func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketEncryption")
 
-	defer logger.AuditLog(w, r, "PutBucketEncryption", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -102,7 +102,7 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketEncryption")
 
-	defer logger.AuditLog(w, r, "GetBucketEncryption", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -145,7 +145,7 @@ func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketEncryption")
 
-	defer logger.AuditLog(w, r, "DeleteBucketEncryption", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {

cmd/bucket-handlers.go

@@ -157,7 +157,7 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketLocation")
 
-	defer logger.AuditLog(w, r, "GetBucketLocation", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -205,7 +205,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListMultipartUploads")
 
-	defer logger.AuditLog(w, r, "ListMultipartUploads", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -260,7 +260,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListBuckets")
 
-	defer logger.AuditLog(w, r, "ListBuckets", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -347,7 +347,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteMultipleObjects")
 
-	defer logger.AuditLog(w, r, "DeleteMultipleObjects", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -383,6 +383,15 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
+	toObjectsNames := func(input []ObjectToDelete) (output []string) {
+		output = make([]string, len(input))
+		for i, obj := range input {
+			output[i] = obj.ObjectName
+		}
+		return
+	}
+	logger.GetReqInfo(ctx).ObjectNames = toObjectsNames(deleteObjects.Objects)
+
 	// Before proceeding validate if bucket exists.
 	_, err := objectAPI.GetBucketInfo(ctx, bucket)
 	if err != nil {
@@ -518,7 +527,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucket")
 
-	defer logger.AuditLog(w, r, "PutBucket", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -649,7 +658,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PostPolicyBucket")
 
-	defer logger.AuditLog(w, r, "PostPolicyBucket", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -913,7 +922,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HeadBucket")
 
-	defer logger.AuditLog(w, r, "HeadBucket", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -943,7 +952,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucket")
 
-	defer logger.AuditLog(w, r, "DeleteBucket", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1031,7 +1040,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketObjectLockConfig")
 
-	defer logger.AuditLog(w, r, "PutBucketObjectLockConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1087,7 +1096,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketObjectLockConfig")
 
-	defer logger.AuditLog(w, r, "GetBucketObjectLockConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1125,7 +1134,7 @@ func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWri
 func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketTagging")
 
-	defer logger.AuditLog(w, r, "PutBucketTagging", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1169,7 +1178,7 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 func (api objectAPIHandlers) GetBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketTagging")
 
-	defer logger.AuditLog(w, r, "GetBucketTagging", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1207,7 +1216,7 @@ func (api objectAPIHandlers) GetBucketTaggingHandler(w http.ResponseWriter, r *h
 func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketTagging")
 
-	defer logger.AuditLog(w, r, "DeleteBucketTagging", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1237,7 +1246,7 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 // Add a replication configuration on the specified bucket as specified in https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketReplication.html
 func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketReplicationConfig")
-	defer logger.AuditLog(w, r, "PutBucketReplicationConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1307,7 +1316,7 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 func (api objectAPIHandlers) GetBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketReplicationConfig")
 
-	defer logger.AuditLog(w, r, "GetBucketReplicationConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1348,7 +1357,7 @@ func (api objectAPIHandlers) GetBucketReplicationConfigHandler(w http.ResponseWr
 // ----------
 func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketReplicationConfig")
-	defer logger.AuditLog(w, r, "DeleteBucketReplicationConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 

cmd/bucket-lifecycle-handlers.go

@@ -38,7 +38,7 @@ const (
 func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketLifecycle")
 
-	defer logger.AuditLog(w, r, "PutBucketLifecycle", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -97,7 +97,7 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketLifecycle")
 
-	defer logger.AuditLog(w, r, "GetBucketLifecycle", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -139,7 +139,7 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketLifecycle")
 
-	defer logger.AuditLog(w, r, "DeleteBucketLifecycle", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {

cmd/bucket-listobjects-handlers.go

@@ -83,7 +83,7 @@ func validateListObjectsArgs(marker, delimiter, encodingType string, maxKeys int
 func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectVersions")
 
-	defer logger.AuditLog(w, r, "ListObjectVersions", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -153,7 +153,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV2M")
 
-	defer logger.AuditLog(w, r, "ListObjectsV2M", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -230,7 +230,7 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV2")
 
-	defer logger.AuditLog(w, r, "ListObjectsV2", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -359,7 +359,7 @@ func proxyRequestByStringHash(ctx context.Context, w http.ResponseWriter, r *htt
 func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV1")
 
-	defer logger.AuditLog(w, r, "ListObjectsV1", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]

cmd/bucket-notification-handlers.go

@@ -39,7 +39,7 @@ const (
 func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketNotification")
 
-	defer logger.AuditLog(w, r, "GetBucketNotification", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
@@ -111,7 +111,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketNotification")
 
-	defer logger.AuditLog(w, r, "PutBucketNotification", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {

cmd/bucket-policy-handlers.go

@@ -40,7 +40,7 @@ const (
 func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketPolicy")
 
-	defer logger.AuditLog(w, r, "PutBucketPolicy", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -106,7 +106,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketPolicy")
 
-	defer logger.AuditLog(w, r, "DeleteBucketPolicy", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -141,7 +141,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketPolicy")
 
-	defer logger.AuditLog(w, r, "GetBucketPolicy", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {

cmd/bucket-targets.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"net/http"
 	"sync"
-	"time"
 
 	minio "github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
@@ -282,7 +281,7 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*m
 	creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "")
 
 	getRemoteTargetInstanceTransportOnce.Do(func() {
-		getRemoteTargetInstanceTransport = newGatewayHTTPTransport(1 * time.Hour)
+		getRemoteTargetInstanceTransport = NewRemoteTargetHTTPTransport()
 	})
 
 	core, err := miniogo.NewCore(tcfg.Endpoint, &miniogo.Options{

cmd/bucket-versioning-handler.go

@@ -40,7 +40,7 @@ const (
 func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketVersioning")
 
-	defer logger.AuditLog(w, r, "PutBucketVersioning", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -98,7 +98,7 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketVersioning")
 
-	defer logger.AuditLog(w, r, "GetBucketVersioning", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]

cmd/config-current.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"crypto/tls"
 	"fmt"
 	"strings"
 	"sync"
@@ -271,7 +272,9 @@ func validateConfig(s config.Config, setDriveCount int) error {
 		}
 	}
 	{
-		kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewGatewayHTTPTransport())
+		kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), newCustomHTTPTransportWithHTTP2(&tls.Config{
+			RootCAs: globalRootCAs,
+		}, defaultDialTimeout)())
 		if err != nil {
 			return err
 		}
@@ -443,7 +446,9 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 		logger.LogIf(ctx, fmt.Errorf("Unable to read heal config: %w", err))
 	}
 
-	kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewGatewayHTTPTransport())
+	kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), newCustomHTTPTransportWithHTTP2(&tls.Config{
+		RootCAs: globalRootCAs,
+	}, defaultDialTimeout)())
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to setup KMS config: %w", err))
 	}

cmd/crypto/kes.go

@@ -177,11 +177,10 @@ func (kes *kesService) CreateKey(keyID string) error { return kes.client.CreateK
 // named key referenced by keyID. It also binds the generated key
 // cryptographically to the provided context.
 func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
-	var context bytes.Buffer
-	ctx.WriteTo(&context)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	var plainKey []byte
-	plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context.Bytes())
+	plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context)
 	if err != nil {
 		return key, nil, err
 	}
@@ -200,11 +199,10 @@ func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 // The context must be same context as the one provided while
 // generating the plaintext key / sealedKey.
 func (kes *kesService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
-	var context bytes.Buffer
-	ctx.WriteTo(&context)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	var plainKey []byte
-	plainKey, err = kes.client.DecryptDataKey(keyID, sealedKey, context.Bytes())
+	plainKey, err = kes.client.DecryptDataKey(keyID, sealedKey, context)
 	if err != nil {
 		return key, err
 	}
@@ -415,7 +413,7 @@ func (c *kesClient) postRetry(path string, body io.ReadSeeker, limit int64) (io.
 		}
 
 		// If the error is not temp. / retryable => fail the request immediately.
-		if !xnet.IsNetworkOrHostDown(err) &&
+		if !xnet.IsNetworkOrHostDown(err, false) &&
 			!errors.Is(err, io.EOF) &&
 			!errors.Is(err, io.ErrUnexpectedEOF) &&
 			!errors.Is(err, context.DeadlineExceeded) {

cmd/crypto/key.go

@@ -103,7 +103,6 @@ func (key ObjectKey) Seal(extKey, iv [32]byte, domain, bucket, object string) Se
 func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucket, object string) error {
 	var (
 		unsealConfig sio.Config
-		decryptedKey bytes.Buffer
 	)
 	switch sealedKey.Algorithm {
 	default:
@@ -122,10 +121,9 @@ func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucke
 		unsealConfig = sio.Config{MinVersion: sio.Version10, Key: sha.Sum(nil)}
 	}
 
-	if n, err := sio.Decrypt(&decryptedKey, bytes.NewReader(sealedKey.Key[:]), unsealConfig); n != 32 || err != nil {
+	if out, err := sio.DecryptBuffer(key[:0], sealedKey.Key[:], unsealConfig); len(out) != 32 || err != nil {
 		return ErrSecretKeyMismatch
 	}
-	copy(key[:], decryptedKey.Bytes())
 	return nil
 }
 
@@ -165,11 +163,7 @@ func (key ObjectKey) UnsealETag(etag []byte) ([]byte, error) {
 	if !IsETagSealed(etag) {
 		return etag, nil
 	}
-	var buffer bytes.Buffer
 	mac := hmac.New(sha256.New, key[:])
 	mac.Write([]byte("SSE-etag"))
-	if _, err := sio.Decrypt(&buffer, bytes.NewReader(etag), sio.Config{Key: mac.Sum(nil)}); err != nil {
-		return nil, err
-	}
-	return buffer.Bytes(), nil
+	return sio.DecryptBuffer(make([]byte, 0, len(etag)), etag, sio.Config{Key: mac.Sum(nil)})
 }

cmd/crypto/kms.go

@@ -39,6 +39,8 @@ type Context map[string]string
 //
 // WriteTo sorts the context keys and writes the sorted
 // key-value pairs as canonical JSON object to w.
+//
+// Note that neither keys nor values are escaped for JSON.
 func (c Context) WriteTo(w io.Writer) (n int64, err error) {
 	sortedKeys := make(sort.StringSlice, 0, len(c))
 	for k := range c {
@@ -67,6 +69,53 @@ func (c Context) WriteTo(w io.Writer) (n int64, err error) {
 	return n + int64(nn), err
 }
 
+// AppendTo appends the context in a canonical from to dst.
+//
+// AppendTo sorts the context keys and writes the sorted
+// key-value pairs as canonical JSON object to w.
+//
+// Note that neither keys nor values are escaped for JSON.
+func (c Context) AppendTo(dst []byte) (output []byte) {
+	if len(c) == 0 {
+		return append(dst, '{', '}')
+	}
+
+	// out should not escape.
+	out := bytes.NewBuffer(dst)
+
+	// No need to copy+sort
+	if len(c) == 1 {
+		for k, v := range c {
+			out.WriteString(`{"`)
+			out.WriteString(k)
+			out.WriteString(`":"`)
+			out.WriteString(v)
+			out.WriteString(`"}`)
+		}
+		return out.Bytes()
+	}
+
+	sortedKeys := make([]string, 0, len(c))
+	for k := range c {
+		sortedKeys = append(sortedKeys, k)
+	}
+	sort.Strings(sortedKeys)
+
+	out.WriteByte('{')
+	for i, k := range sortedKeys {
+		out.WriteByte('"')
+		out.WriteString(k)
+		out.WriteString(`":"`)
+		out.WriteString(c[k])
+		out.WriteByte('"')
+		if i < len(sortedKeys)-1 {
+			out.WriteByte(',')
+		}
+	}
+	out.WriteByte('}')
+	return out.Bytes()
+}
+
 // KMS represents an active and authenticted connection
 // to a Key-Management-Service. It supports generating
 // data key generation and unsealing of KMS-generated
@@ -155,13 +204,12 @@ func (kms *masterKeyKMS) Info() (info KMSInfo) {
 
 func (kms *masterKeyKMS) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
 	var (
-		buffer     bytes.Buffer
 		derivedKey = kms.deriveKey(keyID, ctx)
 	)
-	if n, err := sio.Decrypt(&buffer, bytes.NewReader(sealedKey), sio.Config{Key: derivedKey[:]}); err != nil || n != 32 {
+	out, err := sio.DecryptBuffer(key[:0], sealedKey, sio.Config{Key: derivedKey[:]})
+	if err != nil || len(out) != 32 {
 		return key, err // TODO(aead): upgrade sio to use sio.Error
 	}
-	copy(key[:], buffer.Bytes())
 	return key, nil
 }
 
@@ -171,7 +219,7 @@ func (kms *masterKeyKMS) deriveKey(keyID string, context Context) (key [32]byte)
 	}
 	mac := hmac.New(sha256.New, kms.masterKey[:])
 	mac.Write([]byte(keyID))
-	context.WriteTo(mac)
+	mac.Write(context.AppendTo(make([]byte, 0, 128)))
 	mac.Sum(key[:0])
 	return key
 }

cmd/crypto/kms_test.go

@@ -16,6 +16,7 @@ package crypto
 
 import (
 	"bytes"
+	"fmt"
 	"path"
 	"strings"
 	"testing"
@@ -83,3 +84,32 @@ func TestContextWriteTo(t *testing.T) {
 		}
 	}
 }
+
+func TestContextAppendTo(t *testing.T) {
+	for i, test := range contextWriteToTests {
+		dst := make([]byte, 0, 1024)
+		dst = test.Context.AppendTo(dst)
+		if s := string(dst); s != test.ExpectedJSON {
+			t.Errorf("Test %d: JSON representation differ - got: '%s' want: '%s'", i, s, test.ExpectedJSON)
+		}
+		// Append one more
+		dst = test.Context.AppendTo(dst)
+		if s := string(dst); s != test.ExpectedJSON+test.ExpectedJSON {
+			t.Errorf("Test %d: JSON representation differ - got: '%s' want: '%s'", i, s, test.ExpectedJSON+test.ExpectedJSON)
+		}
+	}
+}
+
+func BenchmarkContext_AppendTo(b *testing.B) {
+	tests := []Context{{}, {"bucket": "warp-benchmark-bucket"}, {"0": "1", "-": "2", ".": "#"}, {"34trg": "dfioutr89", "ikjfdghkjf": "jkedfhgfjkhg", "sdfhsdjkh": "if88889", "asddsirfh804": "kjfdshgdfuhgfg78-45604586#$%"}}
+	for _, test := range tests {
+		b.Run(fmt.Sprintf("%d-elems", len(test)), func(b *testing.B) {
+			dst := make([]byte, 0, 1024)
+			b.ReportAllocs()
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				dst = test.AppendTo(dst[:0])
+			}
+		})
+	}
+}

cmd/crypto/metadata.go

@@ -204,15 +204,17 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 	}
 
 	// Check whether all extracted values are well-formed
-	iv, err := base64.StdEncoding.DecodeString(b64IV)
-	if err != nil || len(iv) != 32 {
+	var iv [32]byte
+	n, err := base64.StdEncoding.Decode(iv[:], []byte(b64IV))
+	if err != nil || n != 32 {
 		return keyID, kmsKey, sealedKey, errInvalidInternalIV
 	}
 	if algorithm != SealAlgorithm {
 		return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
 	}
-	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
-	if err != nil || len(encryptedKey) != 64 {
+	var encryptedKey [64]byte
+	n, err = base64.StdEncoding.Decode(encryptedKey[:], []byte(b64SealedKey))
+	if err != nil || n != 64 {
 		return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
 	}
 	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
@@ -223,8 +225,8 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 	}
 
 	sealedKey.Algorithm = algorithm
-	copy(sealedKey.IV[:], iv)
-	copy(sealedKey.Key[:], encryptedKey)
+	sealedKey.IV = iv
+	sealedKey.Key = encryptedKey
 	return keyID, kmsKey, sealedKey, nil
 }
 

cmd/crypto/vault.go

@@ -15,7 +15,6 @@
 package crypto
 
 import (
-	"bytes"
 	"encoding/base64"
 	"errors"
 	"fmt"
@@ -224,11 +223,10 @@ func (v *vaultService) CreateKey(keyID string) error {
 // named key referenced by keyID. It also binds the generated key
 // cryptographically to the provided context.
 func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
-	var contextStream bytes.Buffer
-	ctx.WriteTo(&contextStream)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	payload := map[string]interface{}{
-		"context": base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+		"context": base64.StdEncoding.EncodeToString(context),
 	}
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/datakey/plaintext/%s", keyID), payload)
 	if err != nil {
@@ -260,12 +258,11 @@ func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 // The context must be same context as the one provided while
 // generating the plaintext key / sealedKey.
 func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
-	var contextStream bytes.Buffer
-	ctx.WriteTo(&contextStream)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	payload := map[string]interface{}{
 		"ciphertext": string(sealedKey),
-		"context":    base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+		"context":    base64.StdEncoding.EncodeToString(context),
 	}
 
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/decrypt/%s", keyID), payload)
@@ -294,12 +291,11 @@ func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k
 // The context must be same context as the one provided while
 // generating the plaintext key / sealedKey.
 func (v *vaultService) UpdateKey(keyID string, sealedKey []byte, ctx Context) (rotatedKey []byte, err error) {
-	var contextStream bytes.Buffer
-	ctx.WriteTo(&contextStream)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	payload := map[string]interface{}{
 		"ciphertext": string(sealedKey),
-		"context":    base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+		"context":    base64.StdEncoding.EncodeToString(context),
 	}
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/rewrap/%s", keyID), payload)
 	if err != nil {

cmd/data-crawler.go

@@ -31,6 +31,7 @@ import (
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/config/heal"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/cmd/logger/message/audit"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/bucket/replication"
 	"github.com/minio/minio/pkg/color"
@@ -69,14 +70,16 @@ func initDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 // There should only ever be one crawler running per cluster.
 func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 	// Make sure only 1 crawler is running on the cluster.
-	locker := objAPI.NewNSLock(ctx, minioMetaBucket, "runDataCrawler.lock")
+	locker := objAPI.NewNSLock(minioMetaBucket, "runDataCrawler.lock")
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	for {
-		err := locker.GetLock(dataCrawlerLeaderLockTimeout)
+		lkctx, err := locker.GetLock(ctx, dataCrawlerLeaderLockTimeout)
 		if err != nil {
 			time.Sleep(time.Duration(r.Float64() * float64(dataCrawlStartDelay)))
 			continue
 		}
+		ctx = lkctx.Context()
+		defer lkctx.Cancel()
 		break
 		// No unlock for "leader" lock.
 	}
@@ -489,7 +492,10 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 			// Dynamic time delay.
 			t := UTCNow()
 
-			err = objAPI.HealObjects(ctx, bucket, prefix, madmin.HealOpts{Recursive: true, Remove: healDeleteDangling},
+			err = objAPI.HealObjects(ctx, bucket, prefix, madmin.HealOpts{
+				Recursive: true,
+				Remove:    healDeleteDangling,
+			},
 				func(bucket, object, versionID string) error {
 					// Wait for each heal as per crawler frequency.
 					sleepDuration(time.Since(t), f.dataUsageCrawlMult)
@@ -755,6 +761,9 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 		return size
 	}
 
+	// Send audit for the lifecycle delete operation
+	auditLogLifecycle(ctx, i.bucket, i.objectPath())
+
 	eventName := event.ObjectRemovedDelete
 	if obj.DeleteMarker {
 		eventName = event.ObjectRemovedDeleteMarkerCreated
@@ -794,3 +803,13 @@ func (i *crawlItem) healReplication(ctx context.Context, o ObjectLayer, meta act
 		globalReplicationState.queueReplicaTask(meta.oi)
 	}
 }
+
+func auditLogLifecycle(ctx context.Context, bucket, object string) {
+	entry := audit.NewEntry(globalDeploymentID)
+	entry.Trigger = "internal-scanner"
+	entry.API.Name = "DeleteObject"
+	entry.API.Bucket = bucket
+	entry.API.Object = object
+	ctx = logger.SetAuditEntry(ctx, &entry)
+	logger.AuditLog(ctx, nil, nil, nil)
+}

cmd/data-usage-cache.go

@@ -297,6 +297,18 @@ func (h dataUsageHash) Key() string {
 	return string(h)
 }
 
+func (d *dataUsageCache) flattenChildrens(root dataUsageEntry) (m map[string]dataUsageEntry) {
+	m = make(map[string]dataUsageEntry)
+	for id := range root.Children {
+		e := d.Cache[id]
+		if len(e.Children) > 0 {
+			e = d.flatten(e)
+		}
+		m[id] = e
+	}
+	return m
+}
+
 // flatten all children of the root into the root element and return it.
 func (d *dataUsageCache) flatten(root dataUsageEntry) dataUsageEntry {
 	for id := range root.Children {

cmd/data-usage.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/cmd/logger"
@@ -61,6 +62,35 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, gui <-chan
 	}
 }
 
+// loadPathsUsageFromBackend returns prefix usages found in passed buckets
+//   e.g.:  /testbucket/prefix => 355601334
+func loadPathsUsageFromBackend(ctx context.Context, objAPI ObjectLayer, buckets []BucketInfo) (map[string]uint64, error) {
+	z, ok := objAPI.(*erasureServerSets)
+	if !ok {
+		return nil, errors.New("prefix usage is not supported")
+	}
+
+	cache := dataUsageCache{}
+
+	m := make(map[string]uint64)
+	for _, bucket := range buckets {
+		for _, pool := range z.serverSets {
+			for _, er := range pool.sets {
+				// Load bucket usage prefixes
+				if err := cache.load(ctx, er, bucket.Name+slashSeparator+dataUsageCacheName); err == nil {
+					if root := cache.find(bucket.Name); root != nil {
+						for id, usageInfo := range cache.flattenChildrens(*root) {
+							m[id] += uint64(usageInfo.Size)
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return m, nil
+}
+
 func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
 	var dataUsageInfoJSON bytes.Buffer
 

cmd/disk-cache-backend.go

@@ -142,7 +142,7 @@ type diskCache struct {
 	// nsMutex namespace lock
 	nsMutex *nsLockMap
 	// Object functions pointing to the corresponding functions of backend implementation.
-	NewNSLockFn func(ctx context.Context, cachePath string) RWLocker
+	NewNSLockFn func(cachePath string) RWLocker
 }
 
 // Inits the disk cache dir if it is not initialized already.
@@ -175,8 +175,8 @@ func newDiskCache(ctx context.Context, dir string, config cache.Config) (*diskCa
 	}
 	go cache.purgeWait(ctx)
 	cache.diskSpaceAvailable(0) // update if cache usage is already high.
-	cache.NewNSLockFn = func(ctx context.Context, cachePath string) RWLocker {
-		return cache.nsMutex.NewNSLock(ctx, nil, cachePath, "")
+	cache.NewNSLockFn = func(cachePath string) RWLocker {
+		return cache.nsMutex.NewNSLock(nil, cachePath, "")
 	}
 	return &cache, nil
 }
@@ -419,12 +419,13 @@ func (c *diskCache) Stat(ctx context.Context, bucket, object string) (oi ObjectI
 // if partial object is cached.
 func (c *diskCache) statCachedMeta(ctx context.Context, cacheObjPath string) (meta *cacheMeta, partial bool, numHits int, err error) {
 
-	cLock := c.NewNSLockFn(ctx, cacheObjPath)
-	if err = cLock.GetRLock(globalOperationTimeout); err != nil {
+	cLock := c.NewNSLockFn(cacheObjPath)
+	lkctx, err := cLock.GetRLock(ctx, globalOperationTimeout)
+	if err != nil {
 		return
 	}
-
-	defer cLock.RUnlock()
+	ctx = lkctx.Context()
+	defer cLock.RUnlock(lkctx.Cancel)
 	return c.statCache(ctx, cacheObjPath)
 }
 
@@ -501,11 +502,13 @@ func (c *diskCache) statCache(ctx context.Context, cacheObjPath string) (meta *c
 // incHitsOnly is true if metadata update is incrementing only the hit counter
 func (c *diskCache) SaveMetadata(ctx context.Context, bucket, object string, meta map[string]string, actualSize int64, rs *HTTPRangeSpec, rsFileName string, incHitsOnly bool) error {
 	cachedPath := getCacheSHADir(c.dir, bucket, object)
-	cLock := c.NewNSLockFn(ctx, cachedPath)
-	if err := cLock.GetLock(globalOperationTimeout); err != nil {
+	cLock := c.NewNSLockFn(cachedPath)
+	lkctx, err := cLock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
 		return err
 	}
-	defer cLock.Unlock()
+	ctx = lkctx.Context()
+	defer cLock.Unlock(lkctx.Cancel)
 	return c.saveMetadata(ctx, bucket, object, meta, actualSize, rs, rsFileName, incHitsOnly)
 }
 
@@ -666,11 +669,13 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
 		return errDiskFull
 	}
 	cachePath := getCacheSHADir(c.dir, bucket, object)
-	cLock := c.NewNSLockFn(ctx, cachePath)
-	if err := cLock.GetLock(globalOperationTimeout); err != nil {
+	cLock := c.NewNSLockFn(cachePath)
+	lkctx, err := cLock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
 		return err
 	}
-	defer cLock.Unlock()
+	ctx = lkctx.Context()
+	defer cLock.Unlock(lkctx.Cancel)
 
 	meta, _, numHits, err := c.statCache(ctx, cachePath)
 	// Case where object not yet cached
@@ -866,12 +871,14 @@ func (c *diskCache) bitrotReadFromCache(ctx context.Context, filePath string, of
 // Get returns ObjectInfo and reader for object from disk cache
 func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, numHits int, err error) {
 	cacheObjPath := getCacheSHADir(c.dir, bucket, object)
-	cLock := c.NewNSLockFn(ctx, cacheObjPath)
-	if err := cLock.GetRLock(globalOperationTimeout); err != nil {
+	cLock := c.NewNSLockFn(cacheObjPath)
+	lkctx, err := cLock.GetRLock(ctx, globalOperationTimeout)
+	if err != nil {
 		return nil, numHits, err
 	}
+	ctx = lkctx.Context()
+	defer cLock.RUnlock(lkctx.Cancel)
 
-	defer cLock.RUnlock()
 	var objInfo ObjectInfo
 	var rngInfo RangeInfo
 	if objInfo, rngInfo, numHits, err = c.statRange(ctx, bucket, object, rs); err != nil {
@@ -930,11 +937,12 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 
 // Deletes the cached object
 func (c *diskCache) delete(ctx context.Context, cacheObjPath string) (err error) {
-	cLock := c.NewNSLockFn(ctx, cacheObjPath)
-	if err := cLock.GetLock(globalOperationTimeout); err != nil {
+	cLock := c.NewNSLockFn(cacheObjPath)
+	lkctx, err := cLock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
 		return err
 	}
-	defer cLock.Unlock()
+	defer cLock.Unlock(lkctx.Cancel)
 	return removeAll(cacheObjPath)
 }
 

cmd/dummy-handlers.go

@@ -32,7 +32,7 @@ import (
 func (api objectAPIHandlers) GetBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketWebsite")
 
-	defer logger.AuditLog(w, r, "GetBucketWebsite", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -64,7 +64,7 @@ func (api objectAPIHandlers) GetBucketWebsiteHandler(w http.ResponseWriter, r *h
 func (api objectAPIHandlers) GetBucketAccelerateHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketAccelerate")
 
-	defer logger.AuditLog(w, r, "GetBucketAccelerate", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -97,7 +97,7 @@ func (api objectAPIHandlers) GetBucketAccelerateHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketRequestPaymentHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketRequestPayment")
 
-	defer logger.AuditLog(w, r, "GetBucketRequestPayment", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -131,7 +131,7 @@ func (api objectAPIHandlers) GetBucketRequestPaymentHandler(w http.ResponseWrite
 func (api objectAPIHandlers) GetBucketLoggingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketLogging")
 
-	defer logger.AuditLog(w, r, "GetBucketLogging", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -170,7 +170,7 @@ func (api objectAPIHandlers) DeleteBucketWebsiteHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketCors")
 
-	defer logger.AuditLog(w, r, "GetBucketCors", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]

cmd/encryption-v1_test.go

@@ -19,10 +19,14 @@ package cmd
 import (
 	"bytes"
 	"encoding/base64"
+	"encoding/json"
+	"fmt"
 	"net/http"
+	"os"
 	"testing"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/klauspost/compress/zstd"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/sio"
@@ -622,3 +626,89 @@ func TestGetDefaultOpts(t *testing.T) {
 		}
 	}
 }
+func Test_decryptObjectInfo(t *testing.T) {
+	var testSet []struct {
+		Bucket  string
+		Name    string
+		UserDef map[string]string
+	}
+	file, err := os.Open("testdata/decryptObjectInfo.json.zst")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+	dec, err := zstd.NewReader(file)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer dec.Close()
+	js := json.NewDecoder(dec)
+	err = js.Decode(&testSet)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	os.Setenv("MINIO_KMS_MASTER_KEY", "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574")
+	defer os.Setenv("MINIO_KMS_MASTER_KEY", "")
+	GlobalKMS, err = crypto.NewKMS(crypto.KMSConfig{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var dst [32]byte
+	for i := range testSet {
+		t.Run(fmt.Sprint("case-", i), func(t *testing.T) {
+			test := &testSet[i]
+			_, err := decryptObjectInfo(dst[:], test.Bucket, test.Name, test.UserDef)
+			if err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
+func Benchmark_decryptObjectInfo(b *testing.B) {
+	var testSet []struct {
+		Bucket  string
+		Name    string
+		UserDef map[string]string
+	}
+	file, err := os.Open("testdata/decryptObjectInfo.json.zst")
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer file.Close()
+	dec, err := zstd.NewReader(file)
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer dec.Close()
+	js := json.NewDecoder(dec)
+	err = js.Decode(&testSet)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	os.Setenv("MINIO_KMS_MASTER_KEY", "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574")
+	defer os.Setenv("MINIO_KMS_MASTER_KEY", "")
+	GlobalKMS, err = crypto.NewKMS(crypto.KMSConfig{})
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	b.SetBytes(int64(len(testSet)))
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			var dst [32]byte
+			for i := range testSet {
+				test := &testSet[i]
+				_, err := decryptObjectInfo(dst[:], test.Bucket, test.Name, test.UserDef)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		}
+	})
+}

cmd/endpoint.go

@@ -17,7 +17,9 @@
 package cmd
 
 import (
+	"context"
 	"crypto/tls"
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -33,6 +35,7 @@ import (
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/cmd/config"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"
 	"github.com/minio/minio/pkg/env"
@@ -744,6 +747,72 @@ func GetProxyEndpointLocalIndex(proxyEps []ProxyEndpoint) int {
 	return -1
 }
 
+func httpDo(clnt *http.Client, req *http.Request, f func(*http.Response, error) error) error {
+	ctx, cancel := context.WithTimeout(GlobalContext, 200*time.Millisecond)
+	defer cancel()
+
+	// Run the HTTP request in a goroutine and pass the response to f.
+	c := make(chan error, 1)
+	req = req.WithContext(ctx)
+	go func() { c <- f(clnt.Do(req)) }()
+	select {
+	case <-ctx.Done():
+		<-c // Wait for f to return.
+		return ctx.Err()
+	case err := <-c:
+		return err
+	}
+}
+
+func getOnlineProxyEndpointIdx() int {
+	type reqIndex struct {
+		Request *http.Request
+		Idx     int
+	}
+
+	proxyRequests := make(map[*http.Client]reqIndex, len(globalProxyEndpoints))
+	for i, proxyEp := range globalProxyEndpoints {
+		proxyEp := proxyEp
+		serverURL := &url.URL{
+			Scheme: proxyEp.Scheme,
+			Host:   proxyEp.Host,
+			Path:   pathJoin(healthCheckPathPrefix, healthCheckLivenessPath),
+		}
+
+		req, err := http.NewRequest(http.MethodGet, serverURL.String(), nil)
+		if err != nil {
+			continue
+		}
+
+		proxyRequests[&http.Client{
+			Transport: proxyEp.Transport,
+		}] = reqIndex{
+			Request: req,
+			Idx:     i,
+		}
+	}
+
+	for c, r := range proxyRequests {
+		if err := httpDo(c, r.Request, func(resp *http.Response, err error) error {
+			if err != nil {
+				return err
+			}
+			xhttp.DrainBody(resp.Body)
+			if resp.StatusCode != http.StatusOK {
+				return errors.New(resp.Status)
+			}
+			if v := resp.Header.Get(xhttp.MinIOServerStatus); v == unavailable {
+				return errors.New(v)
+			}
+			return nil
+		}); err != nil {
+			continue
+		}
+		return r.Idx
+	}
+	return -1
+}
+
 // GetProxyEndpoints - get all endpoints that can be used to proxy list request.
 func GetProxyEndpoints(endpointServerSets EndpointServerSets) ([]ProxyEndpoint, error) {
 	var proxyEps []ProxyEndpoint

cmd/erasure-healing-common.go

@@ -167,7 +167,14 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			// consider the offline disks as consistent.
 			continue
 		}
+		if len(meta.Erasure.Distribution) != len(onlineDisks) {
+			// Erasure distribution seems to have lesser
+			// number of items than number of online disks.
+			inconsistent++
+			continue
+		}
 		if meta.Erasure.Distribution[i] != meta.Erasure.Index {
+			// Mismatch indexes with distribution order
 			inconsistent++
 		}
 	}
@@ -193,6 +200,16 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			if !meta.IsValid() {
 				continue
 			}
+
+			if len(meta.Erasure.Distribution) != len(onlineDisks) {
+				// Erasure distribution is not the same as onlineDisks
+				// attempt a fix if possible, assuming other entries
+				// might have the right erasure distribution.
+				partsMetadata[i] = FileInfo{}
+				dataErrs[i] = errFileCorrupt
+				continue
+			}
+
 			// Since erasure.Distribution is trustable we can fix the mismatching erasure.Index
 			if meta.Erasure.Distribution[i] != meta.Erasure.Index {
 				partsMetadata[i] = FileInfo{}

cmd/erasure-healing.go

@@ -518,7 +518,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 				wg.Add(1)
 				go func(index int, disk StorageAPI) {
 					defer wg.Done()
-					_ = disk.DeleteFile(ctx, bucket, object)
+					_ = disk.DeleteFile(ctx, bucket, object, false)
 				}(index, disk)
 			}
 			wg.Wait()

cmd/erasure-metadata-utils.go

@@ -140,7 +140,7 @@ func readVersionFromDisks(ctx context.Context, disks []StorageAPI, bucket, objec
 			}
 			metadataArray[index], err = disks[index].ReadVersion(ctx, bucket, object, versionID, checkDataDir)
 			if err != nil {
-				if err != errFileNotFound && err != errVolumeNotFound && err != errFileVersionNotFound {
+				if err != errDiskNotFound && err != errFileNotFound && err != errVolumeNotFound && err != errFileVersionNotFound {
 					logger.GetReqInfo(ctx).AppendTags("disk", disks[index].String())
 					logger.LogIf(ctx, err)
 				}

cmd/erasure-metadata.go

@@ -90,9 +90,12 @@ func (fi FileInfo) IsValid() bool {
 	}
 	dataBlocks := fi.Erasure.DataBlocks
 	parityBlocks := fi.Erasure.ParityBlocks
+	correctIndexes := (fi.Erasure.Index > 0 &&
+		fi.Erasure.Index <= dataBlocks+parityBlocks &&
+		len(fi.Erasure.Distribution) == (dataBlocks+parityBlocks))
 	return ((dataBlocks >= parityBlocks) &&
 		(dataBlocks != 0) && (parityBlocks != 0) &&
-		(fi.Erasure.Index > 0 && fi.Erasure.Distribution != nil))
+		correctIndexes)
 }
 
 // ToObjectInfo - Converts metadata to object info.

cmd/erasure-multipart.go

@@ -81,7 +81,7 @@ func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir stri
 			// Ignoring failure to remove parts that weren't present in CompleteMultipartUpload
 			// requests. xl.meta is the authoritative source of truth on which parts constitute
 			// the object. The presence of parts that don't belong in the object doesn't affect correctness.
-			_ = storageDisks[index].DeleteFile(context.TODO(), minioMetaMultipartBucket, curpartPath)
+			_ = storageDisks[index].DeleteFile(context.TODO(), minioMetaMultipartBucket, curpartPath, false)
 			return nil
 		}, index)
 	}
@@ -355,14 +355,23 @@ func (er erasureObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObjec
 //
 // Implements S3 compatible Upload Part API.
 func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (pi PartInfo, err error) {
-	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err = uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+	partIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID, strconv.Itoa(partID)))
+	plkctx, err := partIDLock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
 		return PartInfo{}, err
 	}
-	readLocked := true
+	pctx := plkctx.Context()
+	defer partIDLock.Unlock(plkctx.Cancel)
+
+	uploadIDRLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
+	rlkctx, err := uploadIDRLock.GetRLock(pctx, globalOperationTimeout)
+	if err != nil {
+		return PartInfo{}, err
+	}
+	rctx := rlkctx.Context()
 	defer func() {
-		if readLocked {
-			uploadIDLock.RUnlock()
+		if uploadIDRLock != nil {
+			uploadIDRLock.RUnlock(rlkctx.Cancel)
 		}
 	}()
 
@@ -378,21 +387,25 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
 
 	// Validates if upload ID exists.
-	if err = er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
+	if err = er.checkUploadIDExists(rctx, bucket, object, uploadID); err != nil {
 		return pi, toObjectErr(err, bucket, object, uploadID)
 	}
 
 	// Read metadata associated with the object from all disks.
-	partsMetadata, errs = readAllFileInfo(ctx, er.getDisks(), minioMetaMultipartBucket,
+	partsMetadata, errs = readAllFileInfo(rctx, er.getDisks(), minioMetaMultipartBucket,
 		uploadIDPath, "")
 
+	// Unlock upload id locks before, so others can get it.
+	uploadIDRLock.RUnlock(rlkctx.Cancel)
+	uploadIDRLock = nil
+
 	// get Quorum for this object
-	_, writeQuorum, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
+	_, writeQuorum, err := objectQuorumFromMeta(pctx, er, partsMetadata, errs)
 	if err != nil {
 		return pi, toObjectErr(err, bucket, object)
 	}
 
-	reducedErr := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
+	reducedErr := reduceWriteQuorumErrs(pctx, errs, objectOpIgnoredErrs, writeQuorum)
 	if reducedErr == errErasureWriteQuorum {
 		return pi, toObjectErr(reducedErr, bucket, object)
 	}
@@ -401,7 +414,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	onlineDisks, modTime := listOnlineDisks(er.getDisks(), partsMetadata, errs)
 
 	// Pick one from the first valid metadata.
-	fi, err := pickValidFileInfo(ctx, partsMetadata, modTime, writeQuorum)
+	fi, err := pickValidFileInfo(pctx, partsMetadata, modTime, writeQuorum)
 	if err != nil {
 		return pi, err
 	}
@@ -418,7 +431,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	// Delete the temporary object part. If PutObjectPart succeeds there would be nothing to delete.
 	defer er.deleteObject(context.Background(), minioMetaTmpBucket, tmpPart, writeQuorum)
 
-	erasure, err := NewErasure(ctx, fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks, fi.Erasure.BlockSize)
+	erasure, err := NewErasure(pctx, fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks, fi.Erasure.BlockSize)
 	if err != nil {
 		return pi, toObjectErr(err, bucket, object)
 	}
@@ -447,7 +460,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, tmpPartPath, erasure.ShardFileSize(data.Size()), DefaultBitrotAlgorithm, erasure.ShardSize())
 	}
 
-	n, err := erasure.Encode(ctx, data, writers, buffer, writeQuorum)
+	n, err := erasure.Encode(pctx, data, writers, buffer, writeQuorum)
 	closeBitrotWriters(writers)
 	if err != nil {
 		return pi, toObjectErr(err, bucket, object)
@@ -465,30 +478,30 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		}
 	}
 
-	// Unlock here before acquiring write locks all concurrent
-	// PutObjectParts would serialize here updating `xl.meta`
-	uploadIDLock.RUnlock()
-	readLocked = false
-	if err = uploadIDLock.GetLock(globalOperationTimeout); err != nil {
+	// Acquire write lock to update metadata.
+	uploadIDWLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
+	wlkctx, err := uploadIDWLock.GetLock(pctx, globalOperationTimeout)
+	if err != nil {
 		return PartInfo{}, err
 	}
-	defer uploadIDLock.Unlock()
+	wctx := wlkctx.Context()
+	defer uploadIDWLock.Unlock(wlkctx.Cancel)
 
 	// Validates if upload ID exists.
-	if err = er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
+	if err = er.checkUploadIDExists(wctx, bucket, object, uploadID); err != nil {
 		return pi, toObjectErr(err, bucket, object, uploadID)
 	}
 
 	// Rename temporary part file to its final location.
 	partPath := pathJoin(uploadIDPath, fi.DataDir, partSuffix)
-	onlineDisks, err = rename(ctx, onlineDisks, minioMetaTmpBucket, tmpPartPath, minioMetaMultipartBucket, partPath, false, writeQuorum, nil)
+	onlineDisks, err = rename(wctx, onlineDisks, minioMetaTmpBucket, tmpPartPath, minioMetaMultipartBucket, partPath, false, writeQuorum, nil)
 	if err != nil {
 		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
 	}
 
 	// Read metadata again because it might be updated with parallel upload of another part.
-	partsMetadata, errs = readAllFileInfo(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, "")
-	reducedErr = reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
+	partsMetadata, errs = readAllFileInfo(wctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, "")
+	reducedErr = reduceWriteQuorumErrs(wctx, errs, objectOpIgnoredErrs, writeQuorum)
 	if reducedErr == errErasureWriteQuorum {
 		return pi, toObjectErr(reducedErr, bucket, object)
 	}
@@ -497,7 +510,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	onlineDisks, modTime = listOnlineDisks(onlineDisks, partsMetadata, errs)
 
 	// Pick one from the first valid metadata.
-	fi, err = pickValidFileInfo(ctx, partsMetadata, modTime, writeQuorum)
+	fi, err = pickValidFileInfo(wctx, partsMetadata, modTime, writeQuorum)
 	if err != nil {
 		return pi, err
 	}
@@ -525,7 +538,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	}
 
 	// Writes update `xl.meta` format for each disk.
-	if _, err = writeUniqueFileInfo(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, partsMetadata, writeQuorum); err != nil {
+	if _, err = writeUniqueFileInfo(wctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, partsMetadata, writeQuorum); err != nil {
 		return pi, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
 	}
 
@@ -550,11 +563,13 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 		UploadID: uploadID,
 	}
 
-	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err := uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
+	lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
+	if err != nil {
 		return MultipartInfo{}, err
 	}
-	defer uploadIDLock.RUnlock()
+	ctx = lkctx.Context()
+	defer uploadIDLock.RUnlock(lkctx.Cancel)
 
 	if err := er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return result, toObjectErr(err, bucket, object, uploadID)
@@ -597,12 +612,14 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 // Implements S3 compatible ListObjectParts API. The resulting
 // ListPartsInfo structure is marshaled directly into XML and
 // replied back to the client.
-func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int, opts ObjectOptions) (result ListPartsInfo, e error) {
-	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err := uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int, opts ObjectOptions) (result ListPartsInfo, err error) {
+	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
+	lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
+	if err != nil {
 		return ListPartsInfo{}, err
 	}
-	defer uploadIDLock.RUnlock()
+	ctx = lkctx.Context()
+	defer uploadIDLock.RUnlock(lkctx.Cancel)
 
 	if err := er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return result, toObjectErr(err, bucket, object, uploadID)
@@ -691,13 +708,15 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart, opts ObjectOptions) (oi ObjectInfo, err error) {
 	// Hold read-locks to verify uploaded parts, also disallows
 	// parallel part uploads as well.
-	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err = uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
+	rlkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
+	if err != nil {
 		return oi, err
 	}
-	defer uploadIDLock.RUnlock()
+	rctx := rlkctx.Context()
+	defer uploadIDLock.RUnlock(rlkctx.Cancel)
 
-	if err = er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
+	if err = er.checkUploadIDExists(rctx, bucket, object, uploadID); err != nil {
 		return oi, toObjectErr(err, bucket, object, uploadID)
 	}
 
@@ -717,15 +736,15 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	storageDisks := er.getDisks()
 
 	// Read metadata associated with the object from all disks.
-	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "")
+	partsMetadata, errs := readAllFileInfo(rctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "")
 
 	// get Quorum for this object
-	_, writeQuorum, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
+	_, writeQuorum, err := objectQuorumFromMeta(rctx, er, partsMetadata, errs)
 	if err != nil {
 		return oi, toObjectErr(err, bucket, object)
 	}
 
-	reducedErr := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
+	reducedErr := reduceWriteQuorumErrs(rctx, errs, objectOpIgnoredErrs, writeQuorum)
 	if reducedErr == errErasureWriteQuorum {
 		return oi, toObjectErr(reducedErr, bucket, object)
 	}
@@ -739,7 +758,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	var objectActualSize int64
 
 	// Pick one from the first valid metadata.
-	fi, err := pickValidFileInfo(ctx, partsMetadata, modTime, writeQuorum)
+	fi, err := pickValidFileInfo(rctx, partsMetadata, modTime, writeQuorum)
 	if err != nil {
 		return oi, err
 	}
@@ -825,6 +844,15 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		partsMetadata[index].Parts = fi.Parts
 	}
 
+	// Hold namespace to complete the transaction
+	lk := er.NewNSLock(bucket, object)
+	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return oi, err
+	}
+	ctx = lkctx.Context()
+	defer lk.Unlock(lkctx.Cancel)
+
 	// Write final `xl.meta` at uploadID location
 	if onlineDisks, err = writeUniqueFileInfo(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, partsMetadata, writeQuorum); err != nil {
 		return oi, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
@@ -843,13 +871,6 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 	}
 
-	// Hold namespace to complete the transaction
-	lk := er.NewNSLock(ctx, bucket, object)
-	if err = lk.GetLock(globalOperationTimeout); err != nil {
-		return oi, err
-	}
-	defer lk.Unlock()
-
 	// Rename the multipart object to final location.
 	if onlineDisks, err = renameData(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath,
 		fi.DataDir, bucket, object, writeQuorum, nil); err != nil {
@@ -885,12 +906,14 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 // All parts are purged from all disks and reference to the uploadID
 // would be removed from the system, rollback is not possible on this
 // operation.
-func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error {
-	lk := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err := lk.GetLock(globalOperationTimeout); err != nil {
+func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (err error) {
+	lk := er.NewNSLock(bucket, pathJoin(object, uploadID))
+	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
 		return err
 	}
-	defer lk.Unlock()
+	ctx = lkctx.Context()
+	defer lk.Unlock(lkctx.Cancel)
 
 	// Validates if upload ID exists.
 	if err := er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {

cmd/erasure-object.go

@@ -41,18 +41,20 @@ var objectOpIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied, errUnform
 // CopyObject - copy object source object to destination object.
 // if source object and destination object are same we only
 // update metadata.
-func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, e error) {
+func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, err error) {
 	// This call shouldn't be used for anything other than metadata updates or adding self referential versions.
 	if !srcInfo.metadataOnly {
 		return oi, NotImplemented{}
 	}
 
 	defer ObjectPathUpdated(path.Join(dstBucket, dstObject))
-	lk := er.NewNSLock(ctx, dstBucket, dstObject)
-	if err := lk.GetLock(globalOperationTimeout); err != nil {
+	lk := er.NewNSLock(dstBucket, dstObject)
+	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
 		return oi, err
 	}
-	defer lk.Unlock()
+	ctx = lkctx.Context()
+	defer lk.Unlock(lkctx.Cancel)
 
 	// Read metadata associated with the object from all disks.
 	storageDisks := er.getDisks()
@@ -135,18 +137,22 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 
 	// Acquire lock
 	if lockType != noLock {
-		lock := er.NewNSLock(ctx, bucket, object)
+		lock := er.NewNSLock(bucket, object)
 		switch lockType {
 		case writeLock:
-			if err = lock.GetLock(globalOperationTimeout); err != nil {
+			lkctx, err := lock.GetLock(ctx, globalOperationTimeout)
+			if err != nil {
 				return nil, err
 			}
-			nsUnlocker = lock.Unlock
+			ctx = lkctx.Context()
+			nsUnlocker = func() { lock.Unlock(lkctx.Cancel) }
 		case readLock:
-			if err = lock.GetRLock(globalOperationTimeout); err != nil {
+			lkctx, err := lock.GetRLock(ctx, globalOperationTimeout)
+			if err != nil {
 				return nil, err
 			}
-			nsUnlocker = lock.RUnlock
+			ctx = lkctx.Context()
+			nsUnlocker = func() { lock.RUnlock(lkctx.Cancel) }
 		}
 		unlockOnDefer = true
 	}
@@ -194,13 +200,15 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (er erasureObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error {
+func (er erasureObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) {
 	// Lock the object before reading.
-	lk := er.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalOperationTimeout); err != nil {
+	lk := er.NewNSLock(bucket, object)
+	lkctx, err := lk.GetRLock(ctx, globalOperationTimeout)
+	if err != nil {
 		return err
 	}
-	defer lk.RUnlock()
+	ctx = lkctx.Context()
+	defer lk.RUnlock(lkctx.Cancel)
 
 	// Start offset cannot be negative.
 	if startOffset < 0 {
@@ -344,11 +352,13 @@ func (er erasureObjects) getObject(ctx context.Context, bucket, object string, s
 // GetObjectInfo - reads object metadata and replies back ObjectInfo.
 func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (info ObjectInfo, err error) {
 	// Lock the object before reading.
-	lk := er.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalOperationTimeout); err != nil {
+	lk := er.NewNSLock(bucket, object)
+	lkctx, err := lk.GetRLock(ctx, globalOperationTimeout)
+	if err != nil {
 		return ObjectInfo{}, err
 	}
-	defer lk.RUnlock()
+	ctx = lkctx.Context()
+	defer lk.RUnlock(lkctx.Cancel)
 
 	return er.getObjectInfo(ctx, bucket, object, opts)
 }
@@ -365,6 +375,24 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	}
 
 	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
+		if reducedErr == errErasureReadQuorum && bucket != minioMetaBucket {
+			if _, ok := isObjectDangling(metaArr, errs, nil); ok {
+				reducedErr = errFileNotFound
+				if opts.VersionID != "" {
+					reducedErr = errFileVersionNotFound
+				}
+				// Remove the dangling object only when:
+				//  - This is a non versioned bucket
+				//  - This is a versioned bucket and the version ID is passed, the reason
+				//    is that we cannot fetch the ID of the latest version when we don't trust xl.meta
+				if !opts.Versioned || opts.VersionID != "" {
+					er.deleteObjectVersion(ctx, bucket, object, 1, FileInfo{
+						Name:      object,
+						VersionID: opts.VersionID,
+					})
+				}
+			}
+		}
 		return fi, nil, nil, toObjectErr(reducedErr, bucket, object)
 	}
 
@@ -632,11 +660,13 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		return ObjectInfo{}, IncompleteBody{Bucket: bucket, Object: object}
 	}
 
-	lk := er.NewNSLock(ctx, bucket, object)
-	if err := lk.GetLock(globalOperationTimeout); err != nil {
+	lk := er.NewNSLock(bucket, object)
+	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
 		return ObjectInfo{}, err
 	}
-	defer lk.Unlock()
+	ctx = lkctx.Context()
+	defer lk.Unlock(lkctx.Cancel)
 
 	for i, w := range writers {
 		if w == nil {
@@ -704,6 +734,28 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	return fi.ToObjectInfo(bucket, object), nil
 }
 
+func (er erasureObjects) deletePrefix(ctx context.Context, bucket, prefix string) error {
+	disks := er.getDisks()
+	g := errgroup.WithNErrs(len(disks))
+
+	for index := range disks {
+		index := index
+		g.Go(func() error {
+			if disks[index] == nil {
+				return nil
+			}
+			return disks[index].DeleteFile(ctx, bucket, prefix, true)
+		}, index)
+	}
+
+	for _, err := range g.Wait() {
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object string, writeQuorum int, fi FileInfo) error {
 	disks := er.getDisks()
 
@@ -768,6 +820,18 @@ func (er erasureObjects) deleteObject(ctx context.Context, bucket, object string
 	return reduceWriteQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, writeQuorum)
 }
 
+func (er erasureObjects) getConnectedDisks() []StorageAPI {
+	var storageDisks []StorageAPI
+	for _, d := range er.getDisks() {
+		if d == nil || !d.IsOnline() {
+			storageDisks = append(storageDisks, nil)
+		} else {
+			storageDisks = append(storageDisks, d)
+		}
+	}
+	return storageDisks
+}
+
 // DeleteObjects deletes objects/versions in bulk, this function will still automatically split objects list
 // into smaller bulks if some object names are found to be duplicated in the delete list, splitting
 // into smaller bulks will avoid holding twice the write lock of the duplicated object names.
@@ -776,7 +840,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 	dobjects := make([]DeletedObject, len(objects))
 	writeQuorums := make([]int, len(objects))
 
-	storageDisks := er.getDisks()
+	storageDisks := er.getConnectedDisks()
 
 	for i := range objects {
 		// Assume (N/2 + 1) quorums for all objects
@@ -892,6 +956,12 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 // any error as it is not necessary for the handler to reply back a
 // response to the client request.
 func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+
+	if opts.DeletePrefix {
+		err := er.deletePrefix(ctx, bucket, object)
+		return ObjectInfo{}, err
+	}
+
 	goi, gerr := er.GetObjectInfo(ctx, bucket, object, opts)
 	if gerr != nil && goi.Name == "" {
 		switch gerr.(type) {
@@ -902,13 +972,15 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	}
 
 	// Acquire a write lock before deleting the object.
-	lk := er.NewNSLock(ctx, bucket, object)
-	if err = lk.GetLock(globalDeleteOperationTimeout); err != nil {
+	lk := er.NewNSLock(bucket, object)
+	lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
+	if err != nil {
 		return ObjectInfo{}, err
 	}
-	defer lk.Unlock()
+	ctx = lkctx.Context()
+	defer lk.Unlock(lkctx.Cancel)
 
-	storageDisks := er.getDisks()
+	storageDisks := er.getConnectedDisks()
 	writeQuorum := len(storageDisks)/2 + 1
 
 	if opts.VersionID == "" {

cmd/erasure-server-sets.go

@@ -38,6 +38,8 @@ import (
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
+var errErasureListingQuorum = errors.New("Listing failed. Insufficient number of disks online")
+
 type erasureServerSets struct {
 	GatewayUnsupported
 
@@ -83,14 +85,15 @@ func newErasureServerSets(ctx context.Context, endpointServerSets EndpointServer
 		if err != nil {
 			return nil, err
 		}
+		z.serverSets[i].zoneIndex = i
 	}
 	ctx, z.shutdown = context.WithCancel(ctx)
 	go intDataUpdateTracker.start(ctx, localDrives...)
 	return z, nil
 }
 
-func (z *erasureServerSets) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
-	return z.serverSets[0].NewNSLock(ctx, bucket, objects...)
+func (z *erasureServerSets) NewNSLock(bucket string, objects ...string) RWLocker {
+	return z.serverSets[0].NewNSLock(bucket, objects...)
 }
 
 func (z *erasureServerSets) GetAllLockers() []dsync.NetLocker {
@@ -535,11 +538,27 @@ func (z *erasureServerSets) PutObject(ctx context.Context, bucket string, object
 	return z.serverSets[idx].PutObject(ctx, bucket, object, data, opts)
 }
 
+func (z *erasureServerSets) deletePrefix(ctx context.Context, bucket string, prefix string, opts ObjectOptions) error {
+	for _, zone := range z.serverSets {
+		_, err := zone.DeleteObject(ctx, bucket, prefix, opts)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func (z *erasureServerSets) DeleteObject(ctx context.Context, bucket string, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	if err = checkDelObjArgs(ctx, bucket, object); err != nil {
 		return objInfo, err
 	}
 
+	if opts.DeletePrefix {
+		err := z.deletePrefix(ctx, bucket, object, opts)
+		return ObjectInfo{}, err
+	}
+
 	object = encodeDirObject(object)
 
 	if z.SingleZone() {
@@ -570,14 +589,16 @@ func (z *erasureServerSets) DeleteObjects(ctx context.Context, bucket string, ob
 	}
 
 	// Acquire a bulk write lock across 'objects'
-	multiDeleteLock := z.NewNSLock(ctx, bucket, objSets.ToSlice()...)
-	if err := multiDeleteLock.GetLock(globalOperationTimeout); err != nil {
+	multiDeleteLock := z.NewNSLock(bucket, objSets.ToSlice()...)
+	lkctx, err := multiDeleteLock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
 		for i := range derrs {
 			derrs[i] = err
 		}
 		return nil, derrs
 	}
-	defer multiDeleteLock.Unlock()
+	ctx = lkctx.Context()
+	defer multiDeleteLock.Unlock(lkctx.Cancel)
 
 	for _, zone := range z.serverSets {
 		deletedObjects, errs := zone.DeleteObjects(ctx, bucket, objects, opts)
@@ -688,22 +709,32 @@ func (z *erasureServerSets) listObjectsNonSlash(ctx context.Context, bucket, pre
 		serverSetsEntriesValid = append(serverSetsEntriesValid, make([]bool, len(entryChs)))
 	}
 
+	var prevEntryName string
 	for {
 		if len(objInfos) == maxKeys {
 			break
 		}
 
-		result, quorumCount, zoneIndex, ok := lexicallySortedEntryZone(serverSetsEntryChs, serverSetsEntriesInfos, serverSetsEntriesValid)
+		result, isDir, quorumCount, zoneIndex, listingOK, ok := lexicallySortedEntryZone(serverSetsEntryChs, serverSetsEntriesInfos, serverSetsEntriesValid)
 		if !ok {
 			eof = true
 			break
 		}
 
+		if listingOK < serverSetsListTolerancePerSet[zoneIndex] {
+			return loi, errErasureListingQuorum
+		}
+
 		if quorumCount < serverSetsListTolerancePerSet[zoneIndex] {
 			// Skip entries which are not found on upto expected tolerance
 			continue
 		}
 
+		if isDir && result.Name == prevEntryName {
+			continue
+		}
+		prevEntryName = result.Name
+
 		var objInfo ObjectInfo
 
 		index := strings.Index(strings.TrimPrefix(result.Name, prefix), delimiter)
@@ -803,7 +834,10 @@ func (z *erasureServerSets) listObjectsSplunk(ctx context.Context, bucket, prefi
 		}
 	}
 
-	entries := mergeServerSetsEntriesCh(serverSetsEntryChs, maxKeys, serverSetsListTolerancePerSet)
+	entries, err := mergeServerSetsEntriesCh(serverSetsEntryChs, maxKeys, serverSetsListTolerancePerSet)
+	if err != nil {
+		return loi, err
+	}
 	if len(entries.Files) == 0 {
 		return loi, nil
 	}
@@ -899,7 +933,10 @@ func (z *erasureServerSets) listObjects(ctx context.Context, bucket, prefix, mar
 		}
 	}
 
-	entries := mergeServerSetsEntriesCh(serverSetsEntryChs, maxKeys, serverSetsListTolerancePerSet)
+	entries, err := mergeServerSetsEntriesCh(serverSetsEntryChs, maxKeys, serverSetsListTolerancePerSet)
+	if err != nil {
+		return loi, err
+	}
 	if len(entries.Files) == 0 {
 		return loi, nil
 	}
@@ -911,6 +948,12 @@ func (z *erasureServerSets) listObjects(ctx context.Context, bucket, prefix, mar
 
 	for _, entry := range entries.Files {
 		objInfo := entry.ToObjectInfo(entry.Volume, entry.Name)
+		// Always avoid including the marker in the result, this is
+		// needed to avoid including dir__XLDIR__ and dir/ twice in
+		// different listing pages
+		if objInfo.Name == marker {
+			continue
+		}
 		if HasSuffix(objInfo.Name, SlashSeparator) && !recursive {
 			loi.Prefixes = append(loi.Prefixes, objInfo.Name)
 			continue
@@ -933,9 +976,15 @@ func (z *erasureServerSets) listObjects(ctx context.Context, bucket, prefix, mar
 // again to list the next entry. It is callers responsibility
 // if the caller wishes to list N entries to call lexicallySortedEntry
 // N times until this boolean is 'false'.
-func lexicallySortedEntryZone(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileInfo, zoneEntriesValid [][]bool) (FileInfo, int, int, bool) {
+func lexicallySortedEntryZone(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileInfo, zoneEntriesValid [][]bool) (FileInfo, bool, int, int, int, bool) {
+	var listingOK int
+
 	for i, entryChs := range zoneEntryChs {
 		for j := range entryChs {
+			if entryChs[j].Err != nil {
+				continue
+			}
+			listingOK++
 			zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
 		}
 	}
@@ -955,10 +1004,9 @@ func lexicallySortedEntryZone(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileI
 	}
 
 	var lentry FileInfo
-	var found bool
+	var found, isDir bool
 	var zoneIndex = -1
-	// TODO: following loop can be merged with above
-	// loop, explore this possibility.
+	var setIndex = -1
 	for i, entriesValid := range zoneEntriesValid {
 		for j, valid := range entriesValid {
 			if !valid {
@@ -968,6 +1016,7 @@ func lexicallySortedEntryZone(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileI
 				lentry = zoneEntries[i][j]
 				found = true
 				zoneIndex = i
+				setIndex = zoneEntryChs[i][j].SetIndex
 				continue
 			}
 			str1 := zoneEntries[i][j].Name
@@ -982,6 +1031,7 @@ func lexicallySortedEntryZone(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileI
 			if str1 < str2 {
 				lentry = zoneEntries[i][j]
 				zoneIndex = i
+				setIndex = zoneEntryChs[i][j].SetIndex
 			}
 		}
 	}
@@ -989,7 +1039,15 @@ func lexicallySortedEntryZone(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileI
 	// We haven't been able to find any least entry,
 	// this would mean that we don't have valid entry.
 	if !found {
-		return lentry, 0, zoneIndex, isTruncated
+		return lentry, false, 0, zoneIndex, listingOK, isTruncated
+	}
+
+	if HasSuffix(lentry.Name, slashSeparator) {
+		isDir = true
+	}
+
+	if HasSuffix(lentry.Name, globalDirSuffix) {
+		lentry.Name = strings.TrimSuffix(lentry.Name, globalDirSuffix) + slashSeparator
 	}
 
 	lexicallySortedEntryCount := 0
@@ -999,11 +1057,27 @@ func lexicallySortedEntryZone(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileI
 				continue
 			}
 
-			// Entries are duplicated across disks,
-			// we should simply skip such entries.
-			if lentry.Name == zoneEntries[i][j].Name && lentry.ModTime.Equal(zoneEntries[i][j].ModTime) {
-				lexicallySortedEntryCount++
-				continue
+			zoneEntryName := zoneEntries[i][j].Name
+			zoneEntryObjDir := false
+			if HasSuffix(zoneEntryName, globalDirSuffix) {
+				zoneEntryName = strings.TrimSuffix(zoneEntryName, globalDirSuffix) + slashSeparator
+				zoneEntryObjDir = true
+			}
+
+			if lentry.Name == zoneEntryName {
+				if isDir && zoneEntryObjDir {
+					// Deduplicate directory & object-directory
+					lexicallySortedEntryCount++
+					continue
+				}
+
+				// Entries are duplicated across disks, we should simply skip such entries.
+				if zoneIndex == zoneEntryChs[i][j].ZoneIndex && setIndex == zoneEntryChs[i][j].SetIndex {
+					if isDir || lentry.ModTime.Equal(zoneEntries[i][j].ModTime) {
+						lexicallySortedEntryCount++
+						continue
+					}
+				}
 			}
 
 			// Push all entries which are lexically higher
@@ -1012,11 +1086,7 @@ func lexicallySortedEntryZone(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileI
 		}
 	}
 
-	if HasSuffix(lentry.Name, globalDirSuffix) {
-		lentry.Name = strings.TrimSuffix(lentry.Name, globalDirSuffix) + slashSeparator
-	}
-
-	return lentry, lexicallySortedEntryCount, zoneIndex, isTruncated
+	return lentry, isDir, lexicallySortedEntryCount, zoneIndex, listingOK, isTruncated
 }
 
 // Calculate least entry across serverSets and across multiple FileInfoVersions
@@ -1026,9 +1096,15 @@ func lexicallySortedEntryZone(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileI
 // again to list the next entry. It is callers responsibility
 // if the caller wishes to list N entries to call lexicallySortedEntry
 // N times until this boolean is 'false'.
-func lexicallySortedEntryZoneVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneEntries [][]FileInfoVersions, zoneEntriesValid [][]bool) (FileInfoVersions, int, int, bool) {
+func lexicallySortedEntryZoneVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneEntries [][]FileInfoVersions, zoneEntriesValid [][]bool) (FileInfoVersions, bool, int, int, int, bool) {
+	var listingOK int
+
 	for i, entryChs := range zoneEntryChs {
 		for j := range entryChs {
+			if entryChs[j].Err != nil {
+				continue
+			}
+			listingOK++
 			zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
 		}
 	}
@@ -1048,8 +1124,9 @@ func lexicallySortedEntryZoneVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneE
 	}
 
 	var lentry FileInfoVersions
-	var found bool
+	var found, isDir bool
 	var zoneIndex = -1
+	var setIndex = -1
 	for i, entriesValid := range zoneEntriesValid {
 		for j, valid := range entriesValid {
 			if !valid {
@@ -1059,6 +1136,7 @@ func lexicallySortedEntryZoneVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneE
 				lentry = zoneEntries[i][j]
 				found = true
 				zoneIndex = i
+				setIndex = zoneEntryChs[i][j].SetIndex
 				continue
 			}
 			str1 := zoneEntries[i][j].Name
@@ -1072,7 +1150,8 @@ func lexicallySortedEntryZoneVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneE
 
 			if str1 < str2 {
 				lentry = zoneEntries[i][j]
-				zoneIndex = i
+				zoneIndex = zoneEntryChs[i][j].ZoneIndex
+				setIndex = zoneEntryChs[i][j].SetIndex
 			}
 		}
 	}
@@ -1080,7 +1159,15 @@ func lexicallySortedEntryZoneVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneE
 	// We haven't been able to find any least entry,
 	// this would mean that we don't have valid entry.
 	if !found {
-		return lentry, 0, zoneIndex, isTruncated
+		return lentry, false, 0, zoneIndex, listingOK, isTruncated
+	}
+
+	if HasSuffix(lentry.Name, slashSeparator) {
+		isDir = true
+	}
+
+	if HasSuffix(lentry.Name, globalDirSuffix) {
+		lentry.Name = strings.TrimSuffix(lentry.Name, globalDirSuffix) + slashSeparator
 	}
 
 	lexicallySortedEntryCount := 0
@@ -1090,11 +1177,27 @@ func lexicallySortedEntryZoneVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneE
 				continue
 			}
 
-			// Entries are duplicated across disks,
-			// we should simply skip such entries.
-			if lentry.Name == zoneEntries[i][j].Name && lentry.LatestModTime.Equal(zoneEntries[i][j].LatestModTime) {
-				lexicallySortedEntryCount++
-				continue
+			zoneEntryName := zoneEntries[i][j].Name
+			zoneEntryObjDir := false
+			if HasSuffix(zoneEntryName, globalDirSuffix) {
+				zoneEntryName = strings.TrimSuffix(zoneEntryName, globalDirSuffix) + slashSeparator
+				zoneEntryObjDir = true
+			}
+
+			if lentry.Name == zoneEntryName {
+				// Deduplicat diretory and objet directories in case of non recursive listing
+				if isDir && zoneEntryObjDir {
+					lexicallySortedEntryCount++
+					continue
+				}
+
+				// Entries are duplicated across disks, we should simply skip such entries.
+				if zoneIndex == zoneEntryChs[i][j].ZoneIndex && setIndex == zoneEntryChs[i][j].SetIndex {
+					if isDir || lentry.LatestModTime.Equal(zoneEntries[i][j].LatestModTime) {
+						lexicallySortedEntryCount++
+						continue
+					}
+				}
 			}
 
 			// Push all entries which are lexically higher
@@ -1103,15 +1206,11 @@ func lexicallySortedEntryZoneVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneE
 		}
 	}
 
-	if HasSuffix(lentry.Name, globalDirSuffix) {
-		lentry.Name = strings.TrimSuffix(lentry.Name, globalDirSuffix) + slashSeparator
-	}
-
-	return lentry, lexicallySortedEntryCount, zoneIndex, isTruncated
+	return lentry, isDir, lexicallySortedEntryCount, zoneIndex, listingOK, isTruncated
 }
 
 // mergeServerSetsEntriesVersionsCh - merges FileInfoVersions channel to entries upto maxKeys.
-func mergeServerSetsEntriesVersionsCh(serverSetsEntryChs [][]FileInfoVersionsCh, maxKeys int, serverSetsListTolerancePerSet []int) (entries FilesInfoVersions) {
+func mergeServerSetsEntriesVersionsCh(serverSetsEntryChs [][]FileInfoVersionsCh, maxKeys int, serverSetsListTolerancePerSet []int) (entries FilesInfoVersions, err error) {
 	var i = 0
 	serverSetsEntriesInfos := make([][]FileInfoVersions, 0, len(serverSetsEntryChs))
 	serverSetsEntriesValid := make([][]bool, 0, len(serverSetsEntryChs))
@@ -1120,18 +1219,28 @@ func mergeServerSetsEntriesVersionsCh(serverSetsEntryChs [][]FileInfoVersionsCh,
 		serverSetsEntriesValid = append(serverSetsEntriesValid, make([]bool, len(entryChs)))
 	}
 
+	var prevEntryName string
 	for {
-		fi, quorumCount, zoneIndex, ok := lexicallySortedEntryZoneVersions(serverSetsEntryChs, serverSetsEntriesInfos, serverSetsEntriesValid)
+		fi, isDir, quorumCount, zoneIndex, listingOK, ok := lexicallySortedEntryZoneVersions(serverSetsEntryChs, serverSetsEntriesInfos, serverSetsEntriesValid)
 		if !ok {
 			// We have reached EOF across all entryChs, break the loop.
 			break
 		}
 
+		if listingOK < serverSetsListTolerancePerSet[zoneIndex] {
+			return entries, errErasureListingQuorum
+		}
+
 		if quorumCount < serverSetsListTolerancePerSet[zoneIndex] {
 			// Skip entries which are not found upto the expected tolerance
 			continue
 		}
 
+		if isDir && prevEntryName == fi.Name {
+			continue
+		}
+		prevEntryName = fi.Name
+
 		entries.FilesVersions = append(entries.FilesVersions, fi)
 		i++
 		if i == maxKeys {
@@ -1139,11 +1248,11 @@ func mergeServerSetsEntriesVersionsCh(serverSetsEntryChs [][]FileInfoVersionsCh,
 			break
 		}
 	}
-	return entries
+	return entries, nil
 }
 
 // mergeServerSetsEntriesCh - merges FileInfo channel to entries upto maxKeys.
-func mergeServerSetsEntriesCh(serverSetsEntryChs [][]FileInfoCh, maxKeys int, serverSetsListTolerancePerSet []int) (entries FilesInfo) {
+func mergeServerSetsEntriesCh(serverSetsEntryChs [][]FileInfoCh, maxKeys int, serverSetsListTolerancePerSet []int) (entries FilesInfo, err error) {
 	var i = 0
 	serverSetsEntriesInfos := make([][]FileInfo, 0, len(serverSetsEntryChs))
 	serverSetsEntriesValid := make([][]bool, 0, len(serverSetsEntryChs))
@@ -1151,22 +1260,28 @@ func mergeServerSetsEntriesCh(serverSetsEntryChs [][]FileInfoCh, maxKeys int, se
 		serverSetsEntriesInfos = append(serverSetsEntriesInfos, make([]FileInfo, len(entryChs)))
 		serverSetsEntriesValid = append(serverSetsEntriesValid, make([]bool, len(entryChs)))
 	}
-	var prevEntry string
+
+	var prevEntryName string
 	for {
-		fi, quorumCount, zoneIndex, ok := lexicallySortedEntryZone(serverSetsEntryChs, serverSetsEntriesInfos, serverSetsEntriesValid)
+		fi, isDir, quorumCount, zoneIndex, listingOK, ok := lexicallySortedEntryZone(serverSetsEntryChs, serverSetsEntriesInfos, serverSetsEntriesValid)
 		if !ok {
 			// We have reached EOF across all entryChs, break the loop.
 			break
 		}
 
+		if listingOK < serverSetsListTolerancePerSet[zoneIndex] {
+			return entries, errErasureListingQuorum
+		}
+
 		if quorumCount < serverSetsListTolerancePerSet[zoneIndex] {
 			// Skip entries which are not found upto configured tolerance.
 			continue
 		}
 
-		if HasSuffix(fi.Name, slashSeparator) && fi.Name == prevEntry {
+		if isDir && prevEntryName == fi.Name {
 			continue
 		}
+		prevEntryName = fi.Name
 
 		entries.Files = append(entries.Files, fi)
 		i++
@@ -1174,9 +1289,8 @@ func mergeServerSetsEntriesCh(serverSetsEntryChs [][]FileInfoCh, maxKeys int, se
 			entries.IsTruncated = isTruncatedServerSets(serverSetsEntryChs, serverSetsEntriesInfos, serverSetsEntriesValid)
 			break
 		}
-		prevEntry = fi.Name
 	}
-	return entries
+	return entries, nil
 }
 
 func isTruncatedServerSets(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileInfo, zoneEntriesValid [][]bool) bool {
@@ -1305,7 +1419,10 @@ func (z *erasureServerSets) listObjectVersions(ctx context.Context, bucket, pref
 		}
 	}
 
-	entries := mergeServerSetsEntriesVersionsCh(serverSetsEntryChs, maxKeys, serverSetsListTolerancePerSet)
+	entries, err := mergeServerSetsEntriesVersionsCh(serverSetsEntryChs, maxKeys, serverSetsListTolerancePerSet)
+	if err != nil {
+		return loi, err
+	}
 	if len(entries.FilesVersions) == 0 {
 		return loi, nil
 	}
@@ -1318,6 +1435,12 @@ func (z *erasureServerSets) listObjectVersions(ctx context.Context, bucket, pref
 	for _, entry := range entries.FilesVersions {
 		for _, version := range entry.Versions {
 			objInfo := version.ToObjectInfo(bucket, entry.Name)
+			// Always avoid including the marker in the result, this is
+			// needed to avoid including dir__XLDIR__ and dir/ twice in
+			// different listing pages
+			if objInfo.Name == marker && objInfo.VersionID == versionMarker {
+				continue
+			}
 			if HasSuffix(objInfo.Name, SlashSeparator) && !recursive {
 				loi.Prefixes = append(loi.Prefixes, objInfo.Name)
 				continue
@@ -1377,8 +1500,28 @@ func (z *erasureServerSets) NewMultipartUpload(ctx context.Context, bucket, obje
 		return z.serverSets[0].NewMultipartUpload(ctx, bucket, object, opts)
 	}
 
-	// We don't know the exact size, so we ask for at least 1GiB file.
-	idx, err := z.getZoneIdx(ctx, bucket, object, opts, 1<<30)
+	ns := z.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object, "newMultipartObject.lck"))
+	lkctx, err := ns.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return "", err
+	}
+	ctx = lkctx.Context()
+	defer ns.Unlock(lkctx.Cancel)
+
+	for idx, zone := range z.serverSets {
+		result, err := zone.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
+		if err != nil {
+			return "", err
+		}
+		// If there is a multipart upload with the same bucket/object name,
+		// create the new multipart in the same pool, this will avoid
+		// creating two multiparts uploads in two different pools
+		if len(result.Uploads) != 0 {
+			return z.serverSets[idx].NewMultipartUpload(ctx, bucket, object, opts)
+		}
+	}
+
+	idx, err := z.getZoneIdx(ctx, bucket, object, ObjectOptions{}, 1<<30)
 	if err != nil {
 		return "", err
 	}
@@ -1522,17 +1665,9 @@ func (z *erasureServerSets) CompleteMultipartUpload(ctx context.Context, bucket,
 		return z.serverSets[0].CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 	}
 
-	// Purge any existing object.
 	for _, zone := range z.serverSets {
-		zone.DeleteObject(ctx, bucket, object, opts)
-	}
-
-	for _, zone := range z.serverSets {
-		result, err := zone.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
-		if err != nil {
-			return objInfo, err
-		}
-		if result.Lookup(uploadID) {
+		_, err = zone.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
+		if err == nil {
 			return zone.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 		}
 	}
@@ -1680,24 +1815,15 @@ func (z *erasureServerSets) ListBuckets(ctx context.Context) (buckets []BucketIn
 	return buckets, nil
 }
 
-func (z *erasureServerSets) ReloadFormat(ctx context.Context, dryRun bool) error {
-	// No locks needed since reload happens in HealFormat under
-	// write lock across all nodes.
-	for _, zone := range z.serverSets {
-		if err := zone.ReloadFormat(ctx, dryRun); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 func (z *erasureServerSets) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) {
 	// Acquire lock on format.json
-	formatLock := z.NewNSLock(ctx, minioMetaBucket, formatConfigFile)
-	if err := formatLock.GetLock(globalOperationTimeout); err != nil {
+	formatLock := z.NewNSLock(minioMetaBucket, formatConfigFile)
+	lkctx, err := formatLock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
 		return madmin.HealResultItem{}, err
 	}
-	defer formatLock.Unlock()
+	ctx = lkctx.Context()
+	defer formatLock.Unlock(lkctx.Cancel)
 
 	var r = madmin.HealResultItem{
 		Type:   madmin.HealItemMetadata,
@@ -1722,15 +1848,6 @@ func (z *erasureServerSets) HealFormat(ctx context.Context, dryRun bool) (madmin
 		r.After.Drives = append(r.After.Drives, result.After.Drives...)
 	}
 
-	// Healing succeeded notify the peers to reload format and re-initialize disks.
-	// We will not notify peers if healing is not required.
-	for _, nerr := range globalNotificationSys.ReloadFormat(dryRun) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
-
 	// No heal returned by all serverSets, return errNoHealRequired
 	if countNoHeal == len(z.serverSets) {
 		return r, errNoHealRequired
@@ -1799,17 +1916,25 @@ func (z *erasureServerSets) Walk(ctx context.Context, bucket, prefix string, res
 		go func() {
 			defer close(results)
 
+			var prevEntryName string
 			for {
-				entry, quorumCount, zoneIdx, ok := lexicallySortedEntryZoneVersions(serverSetsEntryChs, serverSetsEntriesInfos, serverSetsEntriesValid)
+				entry, isDir, quorumCount, zoneIdx, _, ok := lexicallySortedEntryZoneVersions(serverSetsEntryChs, serverSetsEntriesInfos, serverSetsEntriesValid)
 				if !ok {
 					// We have reached EOF across all entryChs, break the loop.
 					return
 				}
 
-				if quorumCount >= serverSetsListTolerancePerSet[zoneIdx] {
-					for _, version := range entry.Versions {
-						results <- version.ToObjectInfo(bucket, version.Name)
-					}
+				if quorumCount < serverSetsListTolerancePerSet[zoneIdx] {
+					continue
+				}
+
+				if isDir && prevEntryName == entry.Name {
+					continue
+				}
+				prevEntryName = entry.Name
+
+				for _, version := range entry.Versions {
+					results <- version.ToObjectInfo(bucket, version.Name)
 				}
 			}
 		}()
@@ -1832,16 +1957,24 @@ func (z *erasureServerSets) Walk(ctx context.Context, bucket, prefix string, res
 	go func() {
 		defer close(results)
 
+		var prevEntryName string
 		for {
-			entry, quorumCount, zoneIdx, ok := lexicallySortedEntryZone(serverSetsEntryChs, serverSetsEntriesInfos, serverSetsEntriesValid)
+			entry, isDir, quorumCount, zoneIdx, _, ok := lexicallySortedEntryZone(serverSetsEntryChs, serverSetsEntriesInfos, serverSetsEntriesValid)
 			if !ok {
 				// We have reached EOF across all entryChs, break the loop.
 				return
 			}
 
-			if quorumCount >= serverSetsListTolerancePerSet[zoneIdx] {
-				results <- entry.ToObjectInfo(bucket, entry.Name)
+			if quorumCount < serverSetsListTolerancePerSet[zoneIdx] {
+				continue
 			}
+
+			if isDir && entry.Name == prevEntryName {
+				continue
+			}
+			prevEntryName = entry.Name
+
+			results <- entry.ToObjectInfo(bucket, entry.Name)
 		}
 	}()
 
@@ -1852,75 +1985,67 @@ func (z *erasureServerSets) Walk(ctx context.Context, bucket, prefix string, res
 type HealObjectFn func(bucket, object, versionID string) error
 
 func (z *erasureServerSets) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObject HealObjectFn) error {
-	endWalkCh := make(chan struct{})
-	defer close(endWalkCh)
-
-	serverSetsEntryChs := make([][]FileInfoVersionsCh, 0, len(z.serverSets))
-	zoneDrivesPerSet := make([]int, 0, len(z.serverSets))
 
+	var skipped int
 	for _, zone := range z.serverSets {
-		serverSetsEntryChs = append(serverSetsEntryChs,
-			zone.startMergeWalksVersions(ctx, bucket, prefix, "", true, endWalkCh))
-		zoneDrivesPerSet = append(zoneDrivesPerSet, zone.setDriveCount)
-	}
+		entryChs := zone.startMergeWalksVersions(ctx, bucket, prefix, "", true, ctx.Done())
+		entriesInfos := make([]FileInfoVersions, len(entryChs))
+		entriesValid := make([]bool, len(entryChs))
 
-	serverSetsEntriesInfos := make([][]FileInfoVersions, 0, len(serverSetsEntryChs))
-	serverSetsEntriesValid := make([][]bool, 0, len(serverSetsEntryChs))
-	for _, entryChs := range serverSetsEntryChs {
-		serverSetsEntriesInfos = append(serverSetsEntriesInfos, make([]FileInfoVersions, len(entryChs)))
-		serverSetsEntriesValid = append(serverSetsEntriesValid, make([]bool, len(entryChs)))
-	}
+		for {
+			entry, quorumCount, ok := lexicallySortedEntryVersions(entryChs, entriesInfos, entriesValid)
+			if !ok {
+				skipped++
+				// calculate number of skips to return
+				// "NotFound" error at the end.
+				break
+			}
 
-	// If listing did not return any entries upon first attempt, we
-	// return `ObjectNotFound`, to indicate the caller for any
-	// actions they may want to take as if `prefix` is missing.
-	err := toObjectErr(errFileNotFound, bucket, prefix)
-	for {
-		entry, quorumCount, zoneIndex, ok := lexicallySortedEntryZoneVersions(serverSetsEntryChs, serverSetsEntriesInfos, serverSetsEntriesValid)
-		if !ok {
-			break
-		}
+			drivesPerSet := zone.setDriveCount
+			if quorumCount == drivesPerSet && opts.ScanMode == madmin.HealNormalScan {
+				// Skip good entries.
+				continue
+			}
 
-		// Indicate that first attempt was a success and subsequent loop
-		// knows that its not our first attempt at 'prefix'
-		err = nil
-
-		if zoneIndex >= len(zoneDrivesPerSet) || zoneIndex < 0 {
-			return fmt.Errorf("invalid zone index returned: %d", zoneIndex)
-		}
-
-		if quorumCount == zoneDrivesPerSet[zoneIndex] && opts.ScanMode == madmin.HealNormalScan {
-			// Skip good entries.
-			continue
-		}
-
-		for _, version := range entry.Versions {
-			if err := healObject(bucket, version.Name, version.VersionID); err != nil {
-				return toObjectErr(err, bucket, version.Name)
+			for _, version := range entry.Versions {
+				if err := healObject(bucket, version.Name, version.VersionID); err != nil {
+					return toObjectErr(err, bucket, version.Name)
+				}
 			}
 		}
 	}
 
-	return err
+	if skipped == len(z.serverSets) {
+		// If listing did not return any entries upon first attempt, we
+		// return `ObjectNotFound`, to indicate the caller for any
+		// actions they may want to take as if `prefix` is missing.
+		return toObjectErr(errFileNotFound, bucket, prefix)
+	}
+
+	return nil
 }
 
 func (z *erasureServerSets) HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
 	object = encodeDirObject(object)
 
-	lk := z.NewNSLock(ctx, bucket, object)
+	lk := z.NewNSLock(bucket, object)
 	if bucket == minioMetaBucket {
 		// For .minio.sys bucket heals we should hold write locks.
-		if err := lk.GetLock(globalOperationTimeout); err != nil {
+		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
 			return madmin.HealResultItem{}, err
 		}
-		defer lk.Unlock()
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx.Cancel)
 	} else {
 		// Lock the object before healing. Use read lock since healing
 		// will only regenerate parts & xl.meta of outdated disks.
-		if err := lk.GetRLock(globalOperationTimeout); err != nil {
+		lkctx, err := lk.GetRLock(ctx, globalOperationTimeout)
+		if err != nil {
 			return madmin.HealResultItem{}, err
 		}
-		defer lk.RUnlock()
+		ctx = lkctx.Context()
+		defer lk.RUnlock(lkctx.Cancel)
 	}
 
 	for _, zone := range z.serverSets {

cmd/erasure-sets.go

@@ -43,11 +43,6 @@ import (
 // setsDsyncLockers is encapsulated type for Close()
 type setsDsyncLockers [][]dsync.NetLocker
 
-// Information of a new disk connection
-type diskConnectInfo struct {
-	setIndex int
-}
-
 // erasureSets implements ObjectLayer combining a static list of erasure coded
 // object sets. NOTE: There is no dynamic scaling allowed or intended in
 // current design.
@@ -84,7 +79,10 @@ type erasureSets struct {
 	listTolerancePerSet     int
 
 	monitorContextCancel context.CancelFunc
-	disksConnectEvent    chan diskConnectInfo
+
+	// A channel to send the set index to the MRF when
+	// any disk belonging to that set is connected
+	setReconnectEvent chan int
 
 	// Distribution algorithm of choice.
 	distributionAlgo string
@@ -97,16 +95,26 @@ type erasureSets struct {
 	poolSplunk   *MergeWalkPool
 	poolVersions *MergeWalkVersionsPool
 
-	mrfMU         sync.Mutex
-	mrfOperations map[healSource]int
+	lastConnectDisksOpTime time.Time
+	mrfMU                  sync.Mutex
+	mrfOperations          map[healSource]int
+
+	zoneIndex int
 }
 
-func isEndpointConnected(diskMap map[string]StorageAPI, endpoint string) bool {
+// Return false if endpoint is not connected or has been reconnected after last check
+func isEndpointConnectionStable(diskMap map[string]StorageAPI, endpoint string, lastCheck time.Time) bool {
 	disk := diskMap[endpoint]
 	if disk == nil {
 		return false
 	}
-	return disk.IsOnline()
+	if !disk.IsOnline() {
+		return false
+	}
+	if disk.LastConn().After(lastCheck) {
+		return false
+	}
+	return true
 }
 
 func (s *erasureSets) getDiskMap() map[string]StorageAPI {
@@ -198,14 +206,19 @@ func findDiskIndex(refFormat, format *formatErasureV3) (int, int, error) {
 // connectDisks - attempt to connect all the endpoints, loads format
 // and re-arranges the disks in proper position.
 func (s *erasureSets) connectDisks() {
+	defer func() {
+		s.lastConnectDisksOpTime = time.Now()
+	}()
+
 	var wg sync.WaitGroup
+	var setsJustConnected = make([]bool, s.setCount)
 	diskMap := s.getDiskMap()
 	for _, endpoint := range s.endpoints {
 		diskPath := endpoint.String()
 		if endpoint.IsLocal {
 			diskPath = endpoint.Path
 		}
-		if isEndpointConnected(diskMap, diskPath) {
+		if isEndpointConnectionStable(diskMap, diskPath, s.lastConnectDisksOpTime) {
 			continue
 		}
 		wg.Add(1)
@@ -252,16 +265,29 @@ func (s *erasureSets) connectDisks() {
 			}
 			s.endpointStrings[setIndex*s.setDriveCount+diskIndex] = disk.String()
 			s.erasureDisksMu.Unlock()
-			go func(setIndex int) {
-				// Send a new disk connect event with a timeout
-				select {
-				case s.disksConnectEvent <- diskConnectInfo{setIndex: setIndex}:
-				case <-time.After(100 * time.Millisecond):
-				}
-			}(setIndex)
+			setsJustConnected[setIndex] = true
 		}(endpoint)
 	}
+
 	wg.Wait()
+
+	go func() {
+		idler := time.NewTimer(100 * time.Millisecond)
+		defer idler.Stop()
+
+		for setIndex, justConnected := range setsJustConnected {
+			if !justConnected {
+				continue
+			}
+
+			// Send a new set connect event with a timeout
+			idler.Reset(100 * time.Millisecond)
+			select {
+			case s.setReconnectEvent <- setIndex:
+			case <-idler.C:
+			}
+		}
+	}()
 }
 
 // monitorAndConnectEndpoints this is a monitoring loop to keep track of disconnected
@@ -364,7 +390,7 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 		setDriveCount:       setDriveCount,
 		listTolerancePerSet: listTolerancePerSet,
 		format:              format,
-		disksConnectEvent:   make(chan diskConnectInfo),
+		setReconnectEvent:   make(chan int),
 		distributionAlgo:    format.Erasure.DistributionAlgo,
 		deploymentID:        uuid.MustParse(format.ID),
 		pool:                NewMergeWalkPool(globalMergeLookupTimeout),
@@ -431,11 +457,11 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
-func (s *erasureSets) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
+func (s *erasureSets) NewNSLock(bucket string, objects ...string) RWLocker {
 	if len(objects) == 1 {
-		return s.getHashedSet(objects[0]).NewNSLock(ctx, bucket, objects...)
+		return s.getHashedSet(objects[0]).NewNSLock(bucket, objects...)
 	}
-	return s.getHashedSet("").NewNSLock(ctx, bucket, objects...)
+	return s.getHashedSet("").NewNSLock(bucket, objects...)
 }
 
 // SetDriveCount returns the current drives per set.
@@ -543,12 +569,12 @@ func (s *erasureSets) Shutdown(ctx context.Context) error {
 		}
 	}
 	select {
-	case _, ok := <-s.disksConnectEvent:
+	case _, ok := <-s.setReconnectEvent:
 		if ok {
-			close(s.disksConnectEvent)
+			close(s.setReconnectEvent)
 		}
 	default:
-		close(s.disksConnectEvent)
+		close(s.setReconnectEvent)
 	}
 	return nil
 }
@@ -731,8 +757,24 @@ func (s *erasureSets) GetObjectInfo(ctx context.Context, bucket, object string,
 	return s.getHashedSet(object).GetObjectInfo(ctx, bucket, object, opts)
 }
 
+func (s *erasureSets) deletePrefix(ctx context.Context, bucket string, prefix string, opts ObjectOptions) error {
+	for _, s := range s.sets {
+		_, err := s.DeleteObject(ctx, bucket, prefix, opts)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // DeleteObject - deletes an object from the hashedSet based on the object name.
 func (s *erasureSets) DeleteObject(ctx context.Context, bucket string, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+	if opts.DeletePrefix {
+		err := s.deletePrefix(ctx, bucket, object, opts)
+		return ObjectInfo{}, err
+	}
+
 	return s.getHashedSet(object).DeleteObject(ctx, bucket, object, opts)
 }
 
@@ -832,9 +874,14 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
 
 // FileInfoVersionsCh - file info versions channel
 type FileInfoVersionsCh struct {
-	Ch    chan FileInfoVersions
-	Prev  FileInfoVersions
-	Valid bool
+	Ch        chan FileInfoVersions
+	Prev      FileInfoVersions
+	Valid     bool
+	SetIndex  int
+	ZoneIndex int
+
+	// Is set when storage Walk() returns an error
+	Err error
 }
 
 // Pop - pops a cached entry if any, or from the cached channel.
@@ -855,9 +902,14 @@ func (f *FileInfoVersionsCh) Push(fi FileInfoVersions) {
 
 // FileInfoCh - file info channel
 type FileInfoCh struct {
-	Ch    chan FileInfo
-	Prev  FileInfo
-	Valid bool
+	Ch        chan FileInfo
+	Prev      FileInfo
+	Valid     bool
+	SetIndex  int
+	ZoneIndex int
+
+	// Is set when storage Walk() returns an error
+	Err error
 }
 
 // Pop - pops a cached entry if any, or from the cached channel.
@@ -884,8 +936,8 @@ func (f *FileInfoCh) Push(fi FileInfo) {
 // if the caller wishes to list N entries to call lexicallySortedEntry
 // N times until this boolean is 'false'.
 func lexicallySortedEntryVersions(entryChs []FileInfoVersionsCh, entries []FileInfoVersions, entriesValid []bool) (FileInfoVersions, int, bool) {
-	for j := range entryChs {
-		entries[j], entriesValid[j] = entryChs[j].Pop()
+	for i := range entryChs {
+		entries[i], entriesValid[i] = entryChs[i].Pop()
 	}
 
 	var isTruncated = false
@@ -899,6 +951,7 @@ func lexicallySortedEntryVersions(entryChs []FileInfoVersionsCh, entries []FileI
 
 	var lentry FileInfoVersions
 	var found bool
+	var setIndex = -1
 	for i, valid := range entriesValid {
 		if !valid {
 			continue
@@ -906,10 +959,12 @@ func lexicallySortedEntryVersions(entryChs []FileInfoVersionsCh, entries []FileI
 		if !found {
 			lentry = entries[i]
 			found = true
+			setIndex = entryChs[i].SetIndex
 			continue
 		}
 		if entries[i].Name < lentry.Name {
 			lentry = entries[i]
+			setIndex = entryChs[i].SetIndex
 		}
 	}
 
@@ -927,7 +982,7 @@ func lexicallySortedEntryVersions(entryChs []FileInfoVersionsCh, entries []FileI
 
 		// Entries are duplicated across disks,
 		// we should simply skip such entries.
-		if lentry.Name == entries[i].Name && lentry.LatestModTime.Equal(entries[i].LatestModTime) {
+		if lentry.Name == entries[i].Name && lentry.LatestModTime.Equal(entries[i].LatestModTime) && setIndex == entryChs[i].SetIndex {
 			lexicallySortedEntryCount++
 			continue
 		}
@@ -954,24 +1009,29 @@ func (s *erasureSets) startMergeWalksVersionsN(ctx context.Context, bucket, pref
 	var entryChs []FileInfoVersionsCh
 	var wg sync.WaitGroup
 	var mutex sync.Mutex
-	for _, set := range s.sets {
+	for i, set := range s.sets {
 		// Reset for the next erasure set.
 		for _, disk := range set.getLoadBalancedNDisks(ndisks) {
 			wg.Add(1)
-			go func(disk StorageAPI) {
+			go func(i int, disk StorageAPI) {
 				defer wg.Done()
 
 				entryCh, err := disk.WalkVersions(GlobalContext, bucket, prefix, marker, recursive, endWalkCh)
 				if err != nil {
+					mutex.Lock()
+					entryChs = append(entryChs, FileInfoVersionsCh{Err: err})
+					mutex.Unlock()
 					return
 				}
 
 				mutex.Lock()
 				entryChs = append(entryChs, FileInfoVersionsCh{
-					Ch: entryCh,
+					Ch:        entryCh,
+					SetIndex:  i,
+					ZoneIndex: s.zoneIndex,
 				})
 				mutex.Unlock()
-			}(disk)
+			}(i, disk)
 		}
 	}
 	wg.Wait()
@@ -984,11 +1044,11 @@ func (s *erasureSets) startMergeWalksN(ctx context.Context, bucket, prefix, mark
 	var entryChs []FileInfoCh
 	var wg sync.WaitGroup
 	var mutex sync.Mutex
-	for _, set := range s.sets {
+	for i, set := range s.sets {
 		// Reset for the next erasure set.
 		for _, disk := range set.getLoadBalancedNDisks(ndisks) {
 			wg.Add(1)
-			go func(disk StorageAPI) {
+			go func(i int, disk StorageAPI) {
 				defer wg.Done()
 
 				var entryCh chan FileInfo
@@ -999,15 +1059,19 @@ func (s *erasureSets) startMergeWalksN(ctx context.Context, bucket, prefix, mark
 					entryCh, err = disk.Walk(GlobalContext, bucket, prefix, marker, recursive, endWalkCh)
 				}
 				if err != nil {
-					// Disk walk returned error, ignore it.
+					mutex.Lock()
+					entryChs = append(entryChs, FileInfoCh{Err: err})
+					mutex.Unlock()
 					return
 				}
 				mutex.Lock()
 				entryChs = append(entryChs, FileInfoCh{
-					Ch: entryCh,
+					Ch:        entryCh,
+					SetIndex:  i,
+					ZoneIndex: s.zoneIndex,
 				})
 				mutex.Unlock()
-			}(disk)
+			}(i, disk)
 		}
 	}
 	wg.Wait()
@@ -1141,81 +1205,6 @@ func formatsToDrivesInfo(endpoints Endpoints, formats []*formatErasureV3, sErrs
 	return beforeDrives
 }
 
-// Reloads the format from the disk, usually called by a remote peer notifier while
-// healing in a distributed setup.
-func (s *erasureSets) ReloadFormat(ctx context.Context, dryRun bool) (err error) {
-	storageDisks, errs := initStorageDisksWithErrorsWithoutHealthCheck(s.endpoints)
-	for i, err := range errs {
-		if err != nil && err != errDiskNotFound {
-			return fmt.Errorf("Disk %s: %w", s.endpoints[i], err)
-		}
-	}
-	defer func(storageDisks []StorageAPI) {
-		if err != nil {
-			closeStorageDisks(storageDisks)
-		}
-	}(storageDisks)
-
-	formats, _ := loadFormatErasureAll(storageDisks, false)
-	if err = checkFormatErasureValues(formats, s.setDriveCount); err != nil {
-		return err
-	}
-
-	refFormat, err := getFormatErasureInQuorum(formats)
-	if err != nil {
-		return err
-	}
-
-	s.monitorContextCancel() // turn-off disk monitoring and replace format.
-
-	s.erasureDisksMu.Lock()
-
-	// Replace with new reference format.
-	s.format = refFormat
-
-	// Close all existing disks and reconnect all the disks.
-	for _, disk := range storageDisks {
-		if disk == nil {
-			continue
-		}
-
-		diskID, err := disk.GetDiskID()
-		if err != nil {
-			continue
-		}
-
-		m, n, err := findDiskIndexByDiskID(refFormat, diskID)
-		if err != nil {
-			continue
-		}
-
-		if s.erasureDisks[m][n] != nil {
-			s.erasureDisks[m][n].Close()
-		}
-
-		s.endpointStrings[m*s.setDriveCount+n] = disk.String()
-		if !disk.IsLocal() {
-			// Enable healthcheck disk for remote endpoint.
-			disk, err = newStorageAPI(disk.Endpoint())
-			if err != nil {
-				continue
-			}
-			disk.SetDiskID(diskID)
-		}
-
-		s.erasureDisks[m][n] = disk
-	}
-
-	s.erasureDisksMu.Unlock()
-
-	mctx, mctxCancel := context.WithCancel(GlobalContext)
-	s.monitorContextCancel = mctxCancel
-
-	go s.monitorAndConnectEndpoints(mctx, defaultMonitorConnectEndpointInterval)
-
-	return nil
-}
-
 // If it is a single node Erasure and all disks are root disks, it is most likely a test setup, else it is a production setup.
 // On a test setup we allow creation of format.json on root disks to help with dev/testing.
 func isTestSetup(infos []DiskInfo, errs []error) bool {
@@ -1335,13 +1324,8 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 			}
 		}
 
-		// Save formats `format.json` across all disks.
-		if err = saveFormatErasureAllWithErrs(ctx, storageDisks, sErrs, tmpNewFormats); err != nil {
-			return madmin.HealResultItem{}, err
-		}
-
-		refFormat, err = getFormatErasureInQuorum(tmpNewFormats)
-		if err != nil {
+		// Save new formats `format.json` on unformatted disks.
+		if err = saveUnformattedFormat(ctx, storageDisks, tmpNewFormats); err != nil {
 			return madmin.HealResultItem{}, err
 		}
 
@@ -1349,21 +1333,12 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 
 		s.erasureDisksMu.Lock()
 
-		// Replace with new reference format.
-		s.format = refFormat
-
-		// Disconnect/relinquish all existing disks, lockers and reconnect the disks, lockers.
-		for _, disk := range storageDisks {
-			if disk == nil {
+		for index, format := range tmpNewFormats {
+			if format == nil {
 				continue
 			}
 
-			diskID, err := disk.GetDiskID()
-			if err != nil {
-				continue
-			}
-
-			m, n, err := findDiskIndexByDiskID(refFormat, diskID)
+			m, n, err := findDiskIndexByDiskID(refFormat, format.Erasure.This)
 			if err != nil {
 				continue
 			}
@@ -1372,19 +1347,13 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 				s.erasureDisks[m][n].Close()
 			}
 
-			s.endpointStrings[m*s.setDriveCount+n] = disk.String()
-			if !disk.IsLocal() {
-				// Enable healthcheck disk for remote endpoint.
-				disk, err = newStorageAPI(disk.Endpoint())
-				if err != nil {
-					continue
-				}
-				disk.SetDiskID(diskID)
-			}
-			s.erasureDisks[m][n] = disk
-
+			s.erasureDisks[m][n] = storageDisks[index]
+			s.endpointStrings[m*s.setDriveCount+n] = storageDisks[index].String()
 		}
 
+		// Replace with new reference format.
+		s.format = refFormat
+
 		s.erasureDisksMu.Unlock()
 
 		mctx, mctxCancel := context.WithCancel(GlobalContext)
@@ -1488,6 +1457,7 @@ func (s *erasureSets) maintainMRFList() {
 			bucket:    fOp.bucket,
 			object:    fOp.object,
 			versionID: fOp.versionID,
+			opts:      &madmin.HealOpts{Remove: true},
 		}] = fOp.failedSet
 		s.mrfMU.Unlock()
 	}
@@ -1511,13 +1481,13 @@ func (s *erasureSets) healMRFRoutine() {
 		time.Sleep(time.Second)
 	}
 
-	for e := range s.disksConnectEvent {
+	for setIndex := range s.setReconnectEvent {
 		// Get the list of objects related the er.set
 		// to which the connected disk belongs.
 		var mrfOperations []healSource
 		s.mrfMU.Lock()
 		for k, v := range s.mrfOperations {
-			if v == e.setIndex {
+			if v == setIndex {
 				mrfOperations = append(mrfOperations, k)
 			}
 		}

cmd/erasure.go

@@ -68,8 +68,8 @@ type erasureObjects struct {
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
-func (er erasureObjects) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
-	return er.nsMutex.NewNSLock(ctx, er.getLockers, bucket, objects...)
+func (er erasureObjects) NewNSLock(bucket string, objects ...string) RWLocker {
+	return er.nsMutex.NewNSLock(er.getLockers, bucket, objects...)
 }
 
 // SetDriveCount returns the current drives per set.

cmd/format-erasure.go

@@ -24,6 +24,7 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"reflect"
 	"sync"
 
@@ -183,6 +184,13 @@ func formatGetBackendErasureVersion(formatPath string) (string, error) {
 // first before it V2 migrates to V3.
 func formatErasureMigrate(export string) error {
 	formatPath := pathJoin(export, minioMetaBucket, formatConfigFile)
+	if _, err := os.Stat(formatPath); err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("unable to access (%s) %w", formatPath, err)
+	}
+
 	version, err := formatGetBackendErasureVersion(formatPath)
 	if err != nil {
 		return err
@@ -360,7 +368,7 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) erro
 	tmpFormat := mustGetUUID()
 
 	// Purge any existing temporary file, okay to ignore errors here.
-	defer disk.DeleteFile(context.TODO(), minioMetaBucket, tmpFormat)
+	defer disk.DeleteFile(context.TODO(), minioMetaBucket, tmpFormat, false)
 
 	// write to unique file.
 	if err = disk.WriteAll(context.TODO(), minioMetaBucket, tmpFormat, bytes.NewReader(formatBytes)); err != nil {
@@ -704,28 +712,18 @@ func initErasureMetaVolumesInLocalDisks(storageDisks []StorageAPI, formats []*fo
 	return nil
 }
 
-// saveFormatErasureAllWithErrs - populates `format.json` on disks in its order.
+// saveUnformattedFormat - populates `format.json` on unformatted disks.
 // also adds `.healing.bin` on the disks which are being actively healed.
-func saveFormatErasureAllWithErrs(ctx context.Context, storageDisks []StorageAPI, fErrs []error, formats []*formatErasureV3) error {
-	g := errgroup.WithNErrs(len(storageDisks))
-
-	// Write `format.json` to all disks.
-	for index := range storageDisks {
-		index := index
-		g.Go(func() error {
-			if formats[index] == nil {
-				return errDiskNotFound
-			}
-			if errors.Is(fErrs[index], errUnformattedDisk) {
-				return saveFormatErasure(storageDisks[index], formats[index], true)
-			}
-			return nil
-		}, index)
+func saveUnformattedFormat(ctx context.Context, storageDisks []StorageAPI, formats []*formatErasureV3) error {
+	for index, format := range formats {
+		if format == nil {
+			continue
+		}
+		if err := saveFormatErasure(storageDisks[index], format, true); err != nil {
+			return err
+		}
 	}
-
-	writeQuorum := getWriteQuorum(len(storageDisks))
-	// Wait for the routines to finish.
-	return reduceWriteQuorumErrs(ctx, g.Wait(), nil, writeQuorum)
+	return nil
 }
 
 // saveFormatErasureAll - populates `format.json` on disks in its order.

cmd/fs-v1-multipart.go

@@ -714,11 +714,13 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	}
 
 	// Hold write lock on the object.
-	destLock := fs.NewNSLock(ctx, bucket, object)
-	if err = destLock.GetLock(globalOperationTimeout); err != nil {
+	destLock := fs.NewNSLock(bucket, object)
+	lkctx, err := destLock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
 		return oi, err
 	}
-	defer destLock.Unlock()
+	ctx = lkctx.Context()
+	defer destLock.Unlock(lkctx.Cancel)
 
 	bucketMetaDir := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix)
 	fsMetaPath := pathJoin(bucketMetaDir, bucket, object, fs.metaJSONFile)

cmd/fs-v1.go

@@ -186,9 +186,9 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
-func (fs *FSObjects) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
+func (fs *FSObjects) NewNSLock(bucket string, objects ...string) RWLocker {
 	// lockers are explicitly 'nil' for FS mode since there are only local lockers
-	return fs.nsMutex.NewNSLock(ctx, nil, bucket, objects...)
+	return fs.nsMutex.NewNSLock(nil, bucket, objects...)
 }
 
 // SetDriveCount no-op
@@ -588,7 +588,7 @@ func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string, forceDelet
 // CopyObject - copy object source object to destination object.
 // if source object and destination object are same we only
 // update metadata.
-func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, e error) {
+func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, err error) {
 	if srcOpts.VersionID != "" && srcOpts.VersionID != nullVersionID {
 		return oi, VersionNotFound{
 			Bucket:    srcBucket,
@@ -601,11 +601,13 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 	defer ObjectPathUpdated(path.Join(dstBucket, dstObject))
 
 	if !cpSrcDstSame {
-		objectDWLock := fs.NewNSLock(ctx, dstBucket, dstObject)
-		if err := objectDWLock.GetLock(globalOperationTimeout); err != nil {
+		objectDWLock := fs.NewNSLock(dstBucket, dstObject)
+		lkctx, err := objectDWLock.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
 			return oi, err
 		}
-		defer objectDWLock.Unlock()
+		ctx = lkctx.Context()
+		defer objectDWLock.Unlock(lkctx.Cancel)
 	}
 
 	atomic.AddInt64(&fs.activeIOCount, 1)
@@ -692,18 +694,22 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 
 	if lockType != noLock {
 		// Lock the object before reading.
-		lock := fs.NewNSLock(ctx, bucket, object)
+		lock := fs.NewNSLock(bucket, object)
 		switch lockType {
 		case writeLock:
-			if err = lock.GetLock(globalOperationTimeout); err != nil {
+			lkctx, err := lock.GetLock(ctx, globalOperationTimeout)
+			if err != nil {
 				return nil, err
 			}
-			nsUnlocker = lock.Unlock
+			ctx = lkctx.Context()
+			nsUnlocker = func() { lock.Unlock(lkctx.Cancel) }
 		case readLock:
-			if err = lock.GetRLock(globalOperationTimeout); err != nil {
+			lkctx, err := lock.GetRLock(ctx, globalOperationTimeout)
+			if err != nil {
 				return nil, err
 			}
-			nsUnlocker = lock.RUnlock
+			ctx = lkctx.Context()
+			nsUnlocker = func() { lock.RUnlock(lkctx.Cancel) }
 		}
 	}
 
@@ -785,12 +791,14 @@ func (fs *FSObjects) GetObject(ctx context.Context, bucket, object string, offse
 	}
 
 	// Lock the object before reading.
-	lk := fs.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalOperationTimeout); err != nil {
+	lk := fs.NewNSLock(bucket, object)
+	lkctx, err := lk.GetRLock(ctx, globalOperationTimeout)
+	if err != nil {
 		logger.LogIf(ctx, err)
 		return err
 	}
-	defer lk.RUnlock()
+	ctx = lkctx.Context()
+	defer lk.RUnlock(lkctx.Cancel)
 
 	atomic.AddInt64(&fs.activeIOCount, 1)
 	defer func() {
@@ -1011,13 +1019,15 @@ func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (
 }
 
 // getObjectInfoWithLock - reads object metadata and replies back ObjectInfo.
-func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
+func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object string) (oi ObjectInfo, err error) {
 	// Lock the object before reading.
-	lk := fs.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalOperationTimeout); err != nil {
+	lk := fs.NewNSLock(bucket, object)
+	lkctx, err := lk.GetRLock(ctx, globalOperationTimeout)
+	if err != nil {
 		return oi, err
 	}
-	defer lk.RUnlock()
+	ctx = lkctx.Context()
+	defer lk.RUnlock(lkctx.Cancel)
 
 	if err := checkGetObjArgs(ctx, bucket, object); err != nil {
 		return oi, err
@@ -1051,14 +1061,16 @@ func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string, o
 
 	oi, err := fs.getObjectInfoWithLock(ctx, bucket, object)
 	if err == errCorruptedFormat || err == io.EOF {
-		lk := fs.NewNSLock(ctx, bucket, object)
-		if err = lk.GetLock(globalOperationTimeout); err != nil {
+		lk := fs.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
 			return oi, toObjectErr(err, bucket, object)
 		}
+		ctx = lkctx.Context()
 
 		fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile)
 		err = fs.createFsJSON(object, fsMetaPath)
-		lk.Unlock()
+		lk.Unlock(lkctx.Cancel)
 		if err != nil {
 			return oi, toObjectErr(err, bucket, object)
 		}
@@ -1092,7 +1104,7 @@ func (fs *FSObjects) parentDirIsObject(ctx context.Context, bucket, parent strin
 // until EOF, writes data directly to configured filesystem path.
 // Additionally writes `fs.json` which carries the necessary metadata
 // for future object operations.
-func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
+func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	if opts.Versioned {
 		return objInfo, NotImplemented{}
 	}
@@ -1102,12 +1114,15 @@ func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string
 	}
 
 	// Lock the object.
-	lk := fs.NewNSLock(ctx, bucket, object)
-	if err := lk.GetLock(globalOperationTimeout); err != nil {
+	lk := fs.NewNSLock(bucket, object)
+	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
 		logger.LogIf(ctx, err)
 		return objInfo, err
 	}
-	defer lk.Unlock()
+	ctx = lkctx.Context()
+	defer lk.Unlock(lkctx.Cancel)
+
 	defer ObjectPathUpdated(path.Join(bucket, object))
 
 	atomic.AddInt64(&fs.activeIOCount, 1)
@@ -1284,11 +1299,13 @@ func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string, op
 	}
 
 	// Acquire a write lock before deleting the object.
-	lk := fs.NewNSLock(ctx, bucket, object)
-	if err = lk.GetLock(globalOperationTimeout); err != nil {
+	lk := fs.NewNSLock(bucket, object)
+	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
 		return objInfo, err
 	}
-	defer lk.Unlock()
+	ctx = lkctx.Context()
+	defer lk.Unlock(lkctx.Cancel)
 
 	if err = checkDelObjArgs(ctx, bucket, object); err != nil {
 		return objInfo, err
@@ -1535,12 +1552,6 @@ func (fs *FSObjects) DeleteObjectTags(ctx context.Context, bucket, object string
 	return fs.PutObjectTags(ctx, bucket, object, "", opts)
 }
 
-// ReloadFormat - no-op for fs, Valid only for Erasure.
-func (fs *FSObjects) ReloadFormat(ctx context.Context, dryRun bool) error {
-	logger.LogIf(ctx, NotImplemented{})
-	return NotImplemented{}
-}
-
 // HealFormat - no-op for fs, Valid only for Erasure.
 func (fs *FSObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) {
 	logger.LogIf(ctx, NotImplemented{})

cmd/gateway-common.go

@@ -270,7 +270,7 @@ func IsBackendOnline(ctx context.Context, clnt *http.Client, urlStr string) bool
 	resp, err := clnt.Do(req)
 	if err != nil {
 		clnt.CloseIdleConnections()
-		return !xnet.IsNetworkOrHostDown(err)
+		return !xnet.IsNetworkOrHostDown(err, false)
 	}
 	xhttp.DrainBody(resp.Body)
 	return true
@@ -291,7 +291,7 @@ func ErrorRespToObjectError(err error, params ...string) error {
 		object = params[1]
 	}
 
-	if xnet.IsNetworkOrHostDown(err) {
+	if xnet.IsNetworkOrHostDown(err, false) {
 		return BackendDown{}
 	}
 

cmd/gateway-main.go

@@ -51,8 +51,8 @@ type GatewayLocker struct {
 }
 
 // NewNSLock - implements gateway level locker
-func (l *GatewayLocker) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
-	return l.nsMutex.NewNSLock(ctx, nil, bucket, objects...)
+func (l *GatewayLocker) NewNSLock(bucket string, objects ...string) RWLocker {
+	return l.nsMutex.NewNSLock(nil, bucket, objects...)
 }
 
 // Walk - implements common gateway level Walker, to walk on all objects recursively at a prefix

cmd/gateway-unsupported.go

@@ -41,8 +41,8 @@ func (a GatewayUnsupported) CrawlAndGetDataUsage(ctx context.Context, bf *bloomF
 }
 
 // NewNSLock is a dummy stub for gateway.
-func (a GatewayUnsupported) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
-	logger.CriticalIf(ctx, errors.New("not implemented"))
+func (a GatewayUnsupported) NewNSLock(bucket string, objects ...string) RWLocker {
+	logger.CriticalIf(context.Background(), errors.New("not implemented"))
 	return nil
 }
 
@@ -160,11 +160,6 @@ func (a GatewayUnsupported) DeleteBucketSSEConfig(ctx context.Context, bucket st
 	return NotImplemented{}
 }
 
-// ReloadFormat - Not implemented stub.
-func (a GatewayUnsupported) ReloadFormat(ctx context.Context, dryRun bool) error {
-	return NotImplemented{}
-}
-
 // HealFormat - Not implemented stub
 func (a GatewayUnsupported) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) {
 	return madmin.HealResultItem{}, NotImplemented{}

cmd/generic-handlers.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"net/http"
 	"strings"
 	"time"
@@ -157,15 +158,48 @@ const (
 	loginPathPrefix         = SlashSeparator + "login"
 )
 
-// Adds redirect rules for incoming requests.
 type redirectHandler struct {
 	handler http.Handler
 }
 
-func setBrowserRedirectHandler(h http.Handler) http.Handler {
+func setRedirectHandler(h http.Handler) http.Handler {
 	return redirectHandler{handler: h}
 }
 
+// Adds redirect rules for incoming requests.
+type browserRedirectHandler struct {
+	handler http.Handler
+}
+
+func setBrowserRedirectHandler(h http.Handler) http.Handler {
+	return browserRedirectHandler{handler: h}
+}
+
+func shouldProxy() bool {
+	if newObjectLayerFn() == nil {
+		return true
+	}
+	return !globalIAMSys.Initialized()
+}
+
+func (h redirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if guessIsRPCReq(r) || guessIsBrowserReq(r) ||
+		guessIsHealthCheckReq(r) || guessIsMetricsReq(r) || isAdminReq(r) {
+		h.handler.ServeHTTP(w, r)
+		return
+	}
+	if shouldProxy() {
+		// if this server is still initializing, proxy the request
+		// to any other online servers to avoid 503 for any incoming
+		// API calls.
+		if idx := getOnlineProxyEndpointIdx(); idx >= 0 {
+			proxyRequest(context.TODO(), w, r, globalProxyEndpoints[idx])
+			return
+		}
+	}
+	h.handler.ServeHTTP(w, r)
+}
+
 // Fetch redirect location if urlPath satisfies certain
 // criteria. Some special names are considered to be
 // redirectable, this is purely internal function and
@@ -236,7 +270,7 @@ func guessIsRPCReq(req *http.Request) bool {
 		strings.HasPrefix(req.URL.Path, minioReservedBucketPath+SlashSeparator)
 }
 
-func (h redirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+func (h browserRedirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Re-direction is handled specifically for browser requests.
 	if guessIsBrowserReq(r) {
 		// Fetch the redirect location if any.

cmd/global-heal.go

@@ -148,7 +148,8 @@ func healErasureSet(ctx context.Context, setIndex int, buckets []BucketInfo, dis
 				}
 				mu.Lock()
 				entryChs = append(entryChs, FileInfoVersionsCh{
-					Ch: entryCh,
+					Ch:       entryCh,
+					SetIndex: setIndex,
 				})
 				mu.Unlock()
 			}()

cmd/handler-api.go

@@ -85,37 +85,48 @@ func (t *apiConfig) getClusterDeadline() time.Duration {
 	return t.clusterDeadline
 }
 
-func (t *apiConfig) getRequestsPool() (chan struct{}, <-chan time.Time) {
+func (t *apiConfig) getRequestsPool() (chan struct{}, time.Duration) {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 
 	if t.requestsPool == nil {
-		return nil, nil
+		return nil, 0
+	}
+	if t.requestsDeadline <= 0 {
+		return t.requestsPool, 0
 	}
 
-	return t.requestsPool, time.NewTimer(t.requestsDeadline).C
+	return t.requestsPool, t.requestsDeadline
 }
 
 // maxClients throttles the S3 API calls
 func maxClients(f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		pool, deadlineTimer := globalAPIConfig.getRequestsPool()
+		pool, deadlineTime := globalAPIConfig.getRequestsPool()
 		if pool == nil {
 			f.ServeHTTP(w, r)
 			return
 		}
 
+		timer := time.NewTimer(deadlineTime)
+		defer timer.Stop()
+
+		globalHTTPStats.addRequestsInQueue(1)
+
 		select {
 		case pool <- struct{}{}:
 			defer func() { <-pool }()
+			globalHTTPStats.addRequestsInQueue(-1)
 			f.ServeHTTP(w, r)
-		case <-deadlineTimer:
+		case <-timer.C:
 			// Send a http timeout message
 			writeErrorResponse(r.Context(), w,
 				errorCodes.ToAPIErr(ErrOperationMaxedOut),
 				r.URL, guessIsBrowserReq(r))
+			globalHTTPStats.addRequestsInQueue(-1)
 			return
 		case <-r.Context().Done():
+			globalHTTPStats.addRequestsInQueue(-1)
 			return
 		}
 	}

cmd/healthcheck-handler.go

@@ -20,31 +20,36 @@ import (
 	"context"
 	"net/http"
 	"strconv"
+
+	xhttp "github.com/minio/minio/cmd/http"
 )
 
+const unavailable = "offline"
+
 // ClusterCheckHandler returns if the server is ready for requests.
 func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ClusterCheckHandler")
 
-	objLayer := newObjectLayerFn()
-	// Service not initialized yet
-	if objLayer == nil {
+	if shouldProxy() {
+		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}
 
+	objLayer := newObjectLayerFn()
+
 	ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getClusterDeadline())
 	defer cancel()
 
 	opts := HealthOptions{Maintenance: r.URL.Query().Get("maintenance") == "true"}
 	result := objLayer.Health(ctx, opts)
 	if result.WriteQuorum > 0 {
-		w.Header().Set("X-Minio-Write-Quorum", strconv.Itoa(result.WriteQuorum))
+		w.Header().Set(xhttp.MinIOWriteQuorum, strconv.Itoa(result.WriteQuorum))
 	}
 	if !result.Healthy {
 		// return how many drives are being healed if any
 		if result.HealingDrives > 0 {
-			w.Header().Set("X-Minio-Healing-Drives", strconv.Itoa(result.HealingDrives))
+			w.Header().Set(xhttp.MinIOHealingDrives, strconv.Itoa(result.HealingDrives))
 		}
 		// As a maintenance call we are purposefully asked to be taken
 		// down, this is for orchestrators to know if we can safely
@@ -61,12 +66,19 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 
 // ReadinessCheckHandler Checks if the process is up. Always returns success.
 func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
-	// TODO: only implement this function to notify that this pod is
-	// busy, at a local scope in future, for now '200 OK'.
+	if shouldProxy() {
+		// Service not initialized yet
+		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
+	}
+
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }
 
 // LivenessCheckHandler - Checks if the process is up. Always returns success.
 func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
+	if shouldProxy() {
+		// Service not initialized yet
+		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
+	}
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }

cmd/http-stats.go

@@ -137,15 +137,18 @@ func (stats *HTTPAPIStats) Load() map[string]int {
 // HTTPStats holds statistics information about
 // HTTP requests made by all clients
 type HTTPStats struct {
-	currentS3Requests HTTPAPIStats
-	totalS3Requests   HTTPAPIStats
-	totalS3Errors     HTTPAPIStats
+	currentS3Requests   HTTPAPIStats
+	totalS3Requests     HTTPAPIStats
+	totalS3Errors       HTTPAPIStats
+	totalClientsInQueue int64
 }
 
 // Converts http stats into struct to be sent back to the client.
 func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats {
 	serverStats := ServerHTTPStats{}
 
+	serverStats.TotalClientsInQueue = atomic.LoadInt64(&st.totalClientsInQueue)
+
 	serverStats.CurrentS3Requests = ServerHTTPAPIStats{
 		APIStats: st.currentS3Requests.Load(),
 	}
@@ -160,6 +163,10 @@ func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats {
 	return serverStats
 }
 
+func (st *HTTPStats) addRequestsInQueue(i int64) {
+	atomic.AddInt64(&st.totalClientsInQueue, i)
+}
+
 // Update statistics from http request and response data
 func (st *HTTPStats) updateStats(api string, r *http.Request, w *logger.ResponseWriter, durationSecs float64) {
 	// A successful request has a 2xx response code

cmd/http/headers.go

@@ -126,6 +126,12 @@ const (
 
 	// Header indicates if the etag should be preserved by client
 	MinIOSourceETag = "x-minio-source-etag"
+
+	// Writes expected write quorum
+	MinIOWriteQuorum = "x-minio-write-quorum"
+
+	// Reports number of drives currently healing
+	MinIOHealingDrives = "x-minio-healing-drives"
 )
 
 // Common http query params S3 API

cmd/http/server.go

@@ -194,7 +194,7 @@ func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificat
 			// TLS hardening
 			PreferServerCipherSuites: true,
 			MinVersion:               tls.VersionTLS12,
-			NextProtos:               []string{"h2", "http/1.1"},
+			NextProtos:               []string{"http/1.1", "h2"},
 		}
 		tlsConfig.GetCertificate = getCert
 	}

cmd/iam.go

@@ -24,6 +24,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"sync"
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
@@ -201,6 +202,8 @@ func newMappedPolicy(policy string) MappedPolicy {
 
 // IAMSys - config system.
 type IAMSys struct {
+	sync.Mutex
+
 	usersSysType UsersSysType
 
 	// map of policy names to policy definitions
@@ -277,7 +280,7 @@ type IAMStorageAPI interface {
 // simplifies the implementation for group removal. This is called
 // only via IAM notifications.
 func (sys *IAMSys) LoadGroup(objAPI ObjectLayer, group string) error {
-	if objAPI == nil || sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -318,7 +321,7 @@ func (sys *IAMSys) LoadGroup(objAPI ObjectLayer, group string) error {
 
 // LoadPolicy - reloads a specific canned policy from backend disks or etcd.
 func (sys *IAMSys) LoadPolicy(objAPI ObjectLayer, policyName string) error {
-	if objAPI == nil || sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -336,7 +339,7 @@ func (sys *IAMSys) LoadPolicy(objAPI ObjectLayer, policyName string) error {
 // LoadPolicyMapping - loads the mapped policy for a user or group
 // from storage into server memory.
 func (sys *IAMSys) LoadPolicyMapping(objAPI ObjectLayer, userOrGroup string, isGroup bool) error {
-	if objAPI == nil || sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -362,7 +365,7 @@ func (sys *IAMSys) LoadPolicyMapping(objAPI ObjectLayer, userOrGroup string, isG
 
 // LoadUser - reloads a specific user from backend disks or etcd.
 func (sys *IAMSys) LoadUser(objAPI ObjectLayer, accessKey string, userType IAMUserType) error {
-	if objAPI == nil || sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -386,7 +389,7 @@ func (sys *IAMSys) LoadUser(objAPI ObjectLayer, accessKey string, userType IAMUs
 
 // LoadServiceAccount - reloads a specific service account from backend disks or etcd.
 func (sys *IAMSys) LoadServiceAccount(accessKey string) error {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -410,6 +413,9 @@ func (sys *IAMSys) doIAMConfigMigration(ctx context.Context) error {
 
 // InitStore initializes IAM stores
 func (sys *IAMSys) InitStore(objAPI ObjectLayer) {
+	sys.Lock()
+	defer sys.Unlock()
+
 	if globalEtcdClient == nil {
 		sys.store = newIAMObjectStore(objAPI)
 	} else {
@@ -421,6 +427,16 @@ func (sys *IAMSys) InitStore(objAPI ObjectLayer) {
 	}
 }
 
+// Initialized check if IAM is initialized
+func (sys *IAMSys) Initialized() bool {
+	if sys == nil {
+		return false
+	}
+	sys.Lock()
+	defer sys.Unlock()
+	return sys.store != nil
+}
+
 // Init - initializes config system by reading entries from config/iam
 func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 	retryCtx, cancel := context.WithCancel(ctx)
@@ -429,7 +445,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 	defer cancel()
 
 	// Hold the lock for migration only.
-	txnLk := objAPI.NewNSLock(retryCtx, minioMetaBucket, minioConfigPrefix+"/iam.lock")
+	txnLk := objAPI.NewNSLock(minioMetaBucket, minioConfigPrefix+"/iam.lock")
 
 	// Initializing IAM sub-system needs a retry mechanism for
 	// the following reasons:
@@ -446,7 +462,8 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 	for range retry.NewTimerWithJitter(retryCtx, time.Second, 5*time.Second, retry.MaxJitter) {
 		// let one of the server acquire the lock, if not let them timeout.
 		// which shall be retried again by this loop.
-		if err := txnLk.GetLock(iamLockTimeout); err != nil {
+		lkctx, err := txnLk.GetLock(retryCtx, iamLockTimeout)
+		if err != nil {
 			logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. trying to acquire lock")
 			continue
 		}
@@ -455,8 +472,8 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 			// ****  WARNING ****
 			// Migrating to encrypted backend on etcd should happen before initialization of
 			// IAM sub-system, make sure that we do not move the above codeblock elsewhere.
-			if err := migrateIAMConfigsEtcdToEncrypted(ctx, globalEtcdClient); err != nil {
-				txnLk.Unlock()
+			if err := migrateIAMConfigsEtcdToEncrypted(lkctx.Context(), globalEtcdClient); err != nil {
+				txnLk.Unlock(lkctx.Cancel)
 				logger.LogIf(ctx, fmt.Errorf("Unable to decrypt an encrypted ETCD backend for IAM users and policies: %w", err))
 				logger.LogIf(ctx, errors.New("IAM sub-system is partially initialized, some users may not be available"))
 				return
@@ -470,7 +487,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 
 		// Migrate IAM configuration, if necessary.
 		if err := sys.doIAMConfigMigration(ctx); err != nil {
-			txnLk.Unlock()
+			txnLk.Unlock(lkctx.Cancel)
 			if errors.Is(err, errDiskNotFound) ||
 				errors.Is(err, errConfigNotFound) ||
 				errors.Is(err, context.Canceled) ||
@@ -487,7 +504,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 		}
 
 		// Successfully migrated, proceed to load the users.
-		txnLk.Unlock()
+		txnLk.Unlock(lkctx.Cancel)
 		break
 	}
 
@@ -505,7 +522,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 
 // DeletePolicy - deletes a canned policy from backend or etcd.
 func (sys *IAMSys) DeletePolicy(policyName string) error {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -557,7 +574,7 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
 
 // InfoPolicy - expands the canned policy into its JSON structure.
 func (sys *IAMSys) InfoPolicy(policyName string) (iampolicy.Policy, error) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return iampolicy.Policy{}, errServerNotInitialized
 	}
 
@@ -574,7 +591,7 @@ func (sys *IAMSys) InfoPolicy(policyName string) (iampolicy.Policy, error) {
 
 // ListPolicies - lists all canned policies.
 func (sys *IAMSys) ListPolicies() (map[string]iampolicy.Policy, error) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return nil, errServerNotInitialized
 	}
 
@@ -595,7 +612,7 @@ func (sys *IAMSys) ListPolicies() (map[string]iampolicy.Policy, error) {
 
 // SetPolicy - sets a new name policy.
 func (sys *IAMSys) SetPolicy(policyName string, p iampolicy.Policy) error {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -616,7 +633,7 @@ func (sys *IAMSys) SetPolicy(policyName string, p iampolicy.Policy) error {
 
 // DeleteUser - delete user (only for long-term users not STS users).
 func (sys *IAMSys) DeleteUser(accessKey string) error {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -665,13 +682,14 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 	return err
 }
 
-// returns comma separated policy string, from an input policy
-// after validating if there are any current policies which exist
-// on MinIO corresponding to the input.
-func (sys *IAMSys) currentPolicies(policyName string) string {
-	if sys.store == nil {
+// CurrentPolicies - returns comma separated policy string, from
+// an input policy after validating if there are any current
+// policies which exist on MinIO corresponding to the input.
+func (sys *IAMSys) CurrentPolicies(policyName string) string {
+	if !sys.Initialized() {
 		return ""
 	}
+
 	sys.store.rlock()
 	defer sys.store.runlock()
 
@@ -688,7 +706,7 @@ func (sys *IAMSys) currentPolicies(policyName string) string {
 
 // SetTempUser - set temporary user credentials, these credentials have an expiry.
 func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyName string) error {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -737,7 +755,7 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyNa
 
 // ListUsers - list all users.
 func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return nil, errServerNotInitialized
 	}
 
@@ -773,7 +791,7 @@ func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
 
 // IsTempUser - returns if given key is a temporary user.
 func (sys *IAMSys) IsTempUser(name string) (bool, error) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return false, errServerNotInitialized
 	}
 
@@ -790,7 +808,7 @@ func (sys *IAMSys) IsTempUser(name string) (bool, error) {
 
 // IsServiceAccount - returns if given key is a service account
 func (sys *IAMSys) IsServiceAccount(name string) (bool, string, error) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return false, "", errServerNotInitialized
 	}
 
@@ -811,7 +829,7 @@ func (sys *IAMSys) IsServiceAccount(name string) (bool, string, error) {
 
 // GetUserInfo - get info on a user.
 func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return u, errServerNotInitialized
 	}
 
@@ -856,8 +874,7 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
 
 // SetUserStatus - sets current user status, supports disabled or enabled.
 func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus) error {
-
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -902,8 +919,7 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
 
 // NewServiceAccount - create a new service account
 func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, sessionPolicy *iampolicy.Policy) (auth.Credentials, error) {
-
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return auth.Credentials{}, errServerNotInitialized
 	}
 
@@ -969,8 +985,7 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, ses
 
 // ListServiceAccounts - lists all services accounts associated to a specific user
 func (sys *IAMSys) ListServiceAccounts(ctx context.Context, accessKey string) ([]string, error) {
-
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return nil, errServerNotInitialized
 	}
 
@@ -994,8 +1009,7 @@ func (sys *IAMSys) ListServiceAccounts(ctx context.Context, accessKey string) ([
 
 // GetServiceAccountParent - gets information about a service account
 func (sys *IAMSys) GetServiceAccountParent(ctx context.Context, accessKey string) (string, error) {
-
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return "", errServerNotInitialized
 	}
 
@@ -1011,8 +1025,7 @@ func (sys *IAMSys) GetServiceAccountParent(ctx context.Context, accessKey string
 
 // DeleteServiceAccount - delete a service account
 func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string) error {
-
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -1040,8 +1053,7 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string) e
 
 // SetUser - set user credentials and policy.
 func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
-
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -1078,8 +1090,7 @@ func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
 
 // SetUserSecretKey - sets user secret key
 func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
-
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -1107,7 +1118,7 @@ func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
 
 // GetUser - get user credentials
 func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return cred, false
 	}
 
@@ -1173,7 +1184,7 @@ func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
 // AddUsersToGroup - adds users to a group, creating the group if
 // needed. No error if user(s) already are in the group.
 func (sys *IAMSys) AddUsersToGroup(group string, members []string) error {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -1233,7 +1244,7 @@ func (sys *IAMSys) AddUsersToGroup(group string, members []string) error {
 // RemoveUsersFromGroup - remove users from group. If no users are
 // given, and the group is empty, deletes the group as well.
 func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -1313,7 +1324,7 @@ func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
 
 // SetGroupStatus - enable/disabled a group
 func (sys *IAMSys) SetGroupStatus(group string, enabled bool) error {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -1348,7 +1359,7 @@ func (sys *IAMSys) SetGroupStatus(group string, enabled bool) error {
 
 // GetGroupDescription - builds up group description
 func (sys *IAMSys) GetGroupDescription(group string) (gd madmin.GroupDesc, err error) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return gd, errServerNotInitialized
 	}
 
@@ -1388,7 +1399,7 @@ func (sys *IAMSys) GetGroupDescription(group string) (gd madmin.GroupDesc, err e
 
 // ListGroups - lists groups.
 func (sys *IAMSys) ListGroups() (r []string, err error) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return r, errServerNotInitialized
 	}
 
@@ -1411,7 +1422,7 @@ func (sys *IAMSys) ListGroups() (r []string, err error) {
 
 // PolicyDBSet - sets a policy for a user or group in the PolicyDB.
 func (sys *IAMSys) PolicyDBSet(name, policy string, isGroup bool) error {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -1477,7 +1488,7 @@ func (sys *IAMSys) policyDBSet(name, policyName string, userType IAMUserType, is
 // be a member of multiple groups, this function returns an array of
 // applicable policies (each group is mapped to at most one policy).
 func (sys *IAMSys) PolicyDBGet(name string, isGroup bool) ([]string, error) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return nil, errServerNotInitialized
 	}
 

cmd/listen-notification-handlers.go

@@ -30,7 +30,7 @@ import (
 func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListenNotification")
 
-	defer logger.AuditLog(w, r, "ListenNotification", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Validate if bucket exists.
 	objAPI := api.ObjectAPI()
@@ -139,6 +139,9 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 		return rulesMap.MatchSimple(ev.EventName, ev.S3.Object.Key)
 	})
 
+	if bucketName != "" {
+		values.Set(peerRESTListenBucket, bucketName)
+	}
 	for _, peer := range peers {
 		if peer == nil {
 			continue

cmd/local-locker.go

@@ -27,11 +27,11 @@ import (
 
 // lockRequesterInfo stores various info from the client for each lock that is requested.
 type lockRequesterInfo struct {
-	Writer        bool      // Bool whether write or read lock.
-	UID           string    // UID to uniquely identify request of client.
-	Timestamp     time.Time // Timestamp set at the time of initialization.
-	TimeLastCheck time.Time // Timestamp for last check of validity of lock.
-	Source        string    // Contains line, function and filename reqesting the lock.
+	Writer          bool      // Bool whether write or read lock.
+	UID             string    // UID to uniquely identify request of client.
+	Timestamp       time.Time // Timestamp set at the time of initialization.
+	TimeLastRefresh time.Time // Timestamp for last lock refresh.
+	Source          string    // Contains line, function and filename reqesting the lock.
 	// Owner represents the UUID of the owner who originally requested the lock
 	// useful in expiry.
 	Owner string
@@ -92,20 +92,20 @@ func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool
 	for _, resource := range args.Resources {
 		l.lockMap[resource] = []lockRequesterInfo{
 			{
-				Writer:        true,
-				Source:        args.Source,
-				Owner:         args.Owner,
-				UID:           args.UID,
-				Timestamp:     UTCNow(),
-				TimeLastCheck: UTCNow(),
-				Quorum:        args.Quorum,
+				Writer:          true,
+				Source:          args.Source,
+				Owner:           args.Owner,
+				UID:             args.UID,
+				Timestamp:       UTCNow(),
+				TimeLastRefresh: UTCNow(),
+				Quorum:          args.Quorum,
 			},
 		}
 	}
 	return true, nil
 }
 
-func (l *localLocker) Unlock(args dsync.LockArgs) (reply bool, err error) {
+func (l *localLocker) Unlock(_ context.Context, args dsync.LockArgs) (reply bool, err error) {
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
 
@@ -150,13 +150,13 @@ func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply boo
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
 	lrInfo := lockRequesterInfo{
-		Writer:        false,
-		Source:        args.Source,
-		Owner:         args.Owner,
-		UID:           args.UID,
-		Timestamp:     UTCNow(),
-		TimeLastCheck: UTCNow(),
-		Quorum:        args.Quorum,
+		Writer:          false,
+		Source:          args.Source,
+		Owner:           args.Owner,
+		UID:             args.UID,
+		Timestamp:       UTCNow(),
+		TimeLastRefresh: UTCNow(),
+		Quorum:          args.Quorum,
 	}
 	resource := args.Resources[0]
 	if lri, ok := l.lockMap[resource]; ok {
@@ -172,7 +172,7 @@ func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply boo
 	return reply, nil
 }
 
-func (l *localLocker) RUnlock(args dsync.LockArgs) (reply bool, err error) {
+func (l *localLocker) RUnlock(_ context.Context, args dsync.LockArgs) (reply bool, err error) {
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
 	var lri []lockRequesterInfo
@@ -215,7 +215,35 @@ func (l *localLocker) IsLocal() bool {
 	return true
 }
 
-func (l *localLocker) Expired(ctx context.Context, args dsync.LockArgs) (expired bool, err error) {
+func (l *localLocker) ForceUnlock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
+	select {
+	case <-ctx.Done():
+		return false, ctx.Err()
+	default:
+		l.mutex.Lock()
+		defer l.mutex.Unlock()
+		if len(args.UID) == 0 {
+			for _, resource := range args.Resources {
+				delete(l.lockMap, resource) // Remove the lock (irrespective of write or read lock)
+			}
+			return true, nil
+		}
+
+		lockUIDFound := false
+
+		for resource, lris := range l.lockMap {
+			for _, lri := range lris {
+				if lri.UID == args.UID {
+					l.removeEntry(resource, dsync.LockArgs{Owner: lri.Owner, UID: lri.UID}, &lris)
+					lockUIDFound = true
+				}
+			}
+		}
+		return lockUIDFound, nil
+	}
+}
+
+func (l *localLocker) Refresh(ctx context.Context, args dsync.LockArgs) (refreshed bool, err error) {
 	select {
 	case <-ctx.Done():
 		return false, ctx.Err()
@@ -223,33 +251,39 @@ func (l *localLocker) Expired(ctx context.Context, args dsync.LockArgs) (expired
 		l.mutex.Lock()
 		defer l.mutex.Unlock()
 
+		resource := args.Resources[0] // refresh check is always per resource.
+
 		// Lock found, proceed to verify if belongs to given uid.
-		for _, resource := range args.Resources {
-			if lri, ok := l.lockMap[resource]; ok {
-				// Check whether uid is still active
-				for _, entry := range lri {
-					if entry.UID == args.UID && entry.Owner == args.Owner {
-						return false, nil
-					}
-				}
+		lri, ok := l.lockMap[resource]
+		if !ok {
+			// lock doesn't exist yet, return false
+			return false, nil
+		}
+
+		// Check whether uid is still active
+		for i := range lri {
+			if lri[i].UID == args.UID && lri[i].Owner == args.Owner {
+				lri[i].TimeLastRefresh = UTCNow()
+				return true, nil
 			}
 		}
-		return true, nil
+
+		return false, nil
 	}
 }
 
 // Similar to removeEntry but only removes an entry only if the lock entry exists in map.
 // Caller must hold 'l.mutex' lock.
-func (l *localLocker) removeEntryIfExists(nlrip nameLockRequesterInfoPair) {
+func (l *localLocker) expireOldLocks(interval time.Duration) {
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
 
-	// Check if entry is still in map (could have been removed altogether by 'concurrent' (R)Unlock of last entry)
-	if lri, ok := l.lockMap[nlrip.name]; ok {
-		// Even if the entry exists, it may not be the same entry which was
-		// considered as expired, so we simply an attempt to remove it if its
-		// not possible there is nothing we need to do.
-		l.removeEntry(nlrip.name, dsync.LockArgs{Owner: nlrip.lri.Owner, UID: nlrip.lri.UID}, &lri)
+	for resource, lris := range l.lockMap {
+		for _, lri := range lris {
+			if time.Since(lri.TimeLastRefresh) > interval {
+				l.removeEntry(resource, dsync.LockArgs{Owner: lri.Owner, UID: lri.UID}, &lris)
+			}
+		}
 	}
 }
 

cmd/lock-rest-client.go

@@ -45,8 +45,8 @@ func toLockError(err error) error {
 	switch err.Error() {
 	case errLockConflict.Error():
 		return errLockConflict
-	case errLockNotExpired.Error():
-		return errLockNotExpired
+	case errLockNotFound.Error():
+		return errLockNotFound
 	}
 	return err
 }
@@ -105,7 +105,7 @@ func (client *lockRESTClient) restCall(ctx context.Context, call string, args ds
 	switch err {
 	case nil:
 		return true, nil
-	case errLockConflict, errLockNotExpired:
+	case errLockConflict, errLockNotFound:
 		return false, nil
 	default:
 		return false, err
@@ -123,18 +123,23 @@ func (client *lockRESTClient) Lock(ctx context.Context, args dsync.LockArgs) (re
 }
 
 // RUnlock calls read unlock REST API.
-func (client *lockRESTClient) RUnlock(args dsync.LockArgs) (reply bool, err error) {
-	return client.restCall(context.Background(), lockRESTMethodRUnlock, args)
+func (client *lockRESTClient) RUnlock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
+	return client.restCall(ctx, lockRESTMethodRUnlock, args)
 }
 
 // Unlock calls write unlock RPC.
-func (client *lockRESTClient) Unlock(args dsync.LockArgs) (reply bool, err error) {
-	return client.restCall(context.Background(), lockRESTMethodUnlock, args)
+func (client *lockRESTClient) Unlock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
+	return client.restCall(ctx, lockRESTMethodUnlock, args)
 }
 
-// Expired calls expired handler to check if lock args have expired.
-func (client *lockRESTClient) Expired(ctx context.Context, args dsync.LockArgs) (expired bool, err error) {
-	return client.restCall(ctx, lockRESTMethodExpired, args)
+// RUnlock calls read unlock REST API.
+func (client *lockRESTClient) Refresh(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
+	return client.restCall(ctx, lockRESTMethodRefresh, args)
+}
+
+// ForceUnlock calls force unlock handler to forcibly unlock an active lock.
+func (client *lockRESTClient) ForceUnlock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
+	return client.restCall(ctx, lockRESTMethodForceUnlock, args)
 }
 
 func newLockAPI(endpoint Endpoint) dsync.NetLocker {
@@ -162,11 +167,15 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient {
 
 	trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultTimeout)
 	restClient := rest.NewClient(serverURL, trFn, newAuthToken)
+	restClient.ExpectTimeouts = true
 	restClient.HealthCheckFn = func() bool {
-		ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
 		// Instantiate a new rest client for healthcheck
 		// to avoid recursive healthCheckFn()
-		respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).Call(ctx, lockRESTMethodHealth, nil, nil, -1)
+		healthCheckClient := rest.NewClient(serverURL, trFn, newAuthToken)
+		healthCheckClient.ExpectTimeouts = true
+		healthCheckClient.NoMetrics = true
+		ctx, cancel := context.WithTimeout(GlobalContext, healthCheckClient.HealthCheckTimeout)
+		respBody, err := healthCheckClient.Call(ctx, lockRESTMethodHealth, nil, nil, -1)
 		xhttp.DrainBody(respBody)
 		cancel()
 		var ne *rest.NetworkError

cmd/lock-rest-server-common.go

@@ -21,18 +21,19 @@ import (
 )
 
 const (
-	lockRESTVersion       = "v4" // Add Quorum query param
+	lockRESTVersion       = "v5" // Add Force unlock
 	lockRESTVersionPrefix = SlashSeparator + lockRESTVersion
 	lockRESTPrefix        = minioReservedBucketPath + "/lock"
 )
 
 const (
-	lockRESTMethodHealth  = "/health"
-	lockRESTMethodLock    = "/lock"
-	lockRESTMethodRLock   = "/rlock"
-	lockRESTMethodUnlock  = "/unlock"
-	lockRESTMethodRUnlock = "/runlock"
-	lockRESTMethodExpired = "/expired"
+	lockRESTMethodHealth      = "/health"
+	lockRESTMethodRefresh     = "/refresh"
+	lockRESTMethodLock        = "/lock"
+	lockRESTMethodRLock       = "/rlock"
+	lockRESTMethodUnlock      = "/unlock"
+	lockRESTMethodRUnlock     = "/runlock"
+	lockRESTMethodForceUnlock = "/force-unlock"
 
 	// lockRESTOwner represents owner UUID
 	lockRESTOwner = "owner"
@@ -51,6 +52,6 @@ const (
 
 var (
 	errLockConflict       = errors.New("lock conflict")
-	errLockNotExpired     = errors.New("lock not expired")
 	errLockNotInitialized = errors.New("lock not initialized")
+	errLockNotFound       = errors.New("lock not found")
 )

cmd/lock-rest-server-common_test.go

@@ -55,18 +55,18 @@ func TestLockRpcServerRemoveEntry(t *testing.T) {
 	defer os.RemoveAll(testPath)
 
 	lockRequesterInfo1 := lockRequesterInfo{
-		Owner:         "owner",
-		Writer:        true,
-		UID:           "0123-4567",
-		Timestamp:     UTCNow(),
-		TimeLastCheck: UTCNow(),
+		Owner:           "owner",
+		Writer:          true,
+		UID:             "0123-4567",
+		Timestamp:       UTCNow(),
+		TimeLastRefresh: UTCNow(),
 	}
 	lockRequesterInfo2 := lockRequesterInfo{
-		Owner:         "owner",
-		Writer:        true,
-		UID:           "89ab-cdef",
-		Timestamp:     UTCNow(),
-		TimeLastCheck: UTCNow(),
+		Owner:           "owner",
+		Writer:          true,
+		UID:             "89ab-cdef",
+		Timestamp:       UTCNow(),
+		TimeLastRefresh: UTCNow(),
 	}
 
 	locker.ll.lockMap["name"] = []lockRequesterInfo{

cmd/lock-rest-server.go

@@ -20,7 +20,6 @@ import (
 	"bufio"
 	"context"
 	"errors"
-	"math/rand"
 	"net/http"
 	"path"
 	"sort"
@@ -35,8 +34,8 @@ const (
 	// Lock maintenance interval.
 	lockMaintenanceInterval = 30 * time.Second
 
-	// Lock validity check interval.
-	lockValidityCheckInterval = 5 * time.Second
+	// Lock validity duration
+	lockValidityDuration = 20 * time.Second
 )
 
 // To abstract a node over network.
@@ -96,6 +95,31 @@ func (l *lockRESTServer) HealthHandler(w http.ResponseWriter, r *http.Request) {
 	l.IsValid(w, r)
 }
 
+// RefreshHandler - refresh the current lock
+func (l *lockRESTServer) RefreshHandler(w http.ResponseWriter, r *http.Request) {
+	if !l.IsValid(w, r) {
+		l.writeErrorResponse(w, errors.New("invalid request"))
+		return
+	}
+
+	args, err := getLockArgs(r)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+
+	refreshed, err := l.ll.Refresh(r.Context(), args)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+
+	if !refreshed {
+		l.writeErrorResponse(w, errLockNotFound)
+		return
+	}
+}
+
 // LockHandler - Acquires a lock.
 func (l *lockRESTServer) LockHandler(w http.ResponseWriter, r *http.Request) {
 	if !l.IsValid(w, r) {
@@ -132,7 +156,7 @@ func (l *lockRESTServer) UnlockHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	_, err = l.ll.Unlock(args)
+	_, err = l.ll.Unlock(context.Background(), args)
 	// Ignore the Unlock() "reply" return value because if err == nil, "reply" is always true
 	// Consequently, if err != nil, reply is always false
 	if err != nil {
@@ -179,16 +203,16 @@ func (l *lockRESTServer) RUnlockHandler(w http.ResponseWriter, r *http.Request)
 
 	// Ignore the RUnlock() "reply" return value because if err == nil, "reply" is always true.
 	// Consequently, if err != nil, reply is always false
-	if _, err = l.ll.RUnlock(args); err != nil {
+	if _, err = l.ll.RUnlock(context.Background(), args); err != nil {
 		l.writeErrorResponse(w, err)
 		return
 	}
 }
 
-// ExpiredHandler - query expired lock status.
-func (l *lockRESTServer) ExpiredHandler(w http.ResponseWriter, r *http.Request) {
+// ForceUnlockHandler - query expired lock status.
+func (l *lockRESTServer) ForceUnlockHandler(w http.ResponseWriter, r *http.Request) {
 	if !l.IsValid(w, r) {
-		l.writeErrorResponse(w, errors.New("Invalid request"))
+		l.writeErrorResponse(w, errors.New("invalid request"))
 		return
 	}
 
@@ -198,136 +222,23 @@ func (l *lockRESTServer) ExpiredHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	expired, err := l.ll.Expired(r.Context(), args)
-	if err != nil {
+	if _, err = l.ll.ForceUnlock(r.Context(), args); err != nil {
 		l.writeErrorResponse(w, err)
 		return
 	}
-	if !expired {
-		l.writeErrorResponse(w, errLockNotExpired)
-		return
-	}
 }
 
-// nameLockRequesterInfoPair is a helper type for lock maintenance
-type nameLockRequesterInfoPair struct {
-	name string
-	lri  lockRequesterInfo
-}
-
-// getLongLivedLocks returns locks that are older than a certain time and
-// have not been 'checked' for validity too soon enough
-func getLongLivedLocks(interval time.Duration) map[Endpoint][]nameLockRequesterInfoPair {
-	nlripMap := make(map[Endpoint][]nameLockRequesterInfoPair)
-	for endpoint, locker := range globalLockServers {
-		rslt := []nameLockRequesterInfoPair{}
-		locker.mutex.Lock()
-		for name, lriArray := range locker.lockMap {
-			for idx := range lriArray {
-				// Check whether enough time has gone by since last check
-				if time.Since(lriArray[idx].TimeLastCheck) >= interval {
-					rslt = append(rslt, nameLockRequesterInfoPair{
-						name: name,
-						lri:  lriArray[idx],
-					})
-					lriArray[idx].TimeLastCheck = UTCNow()
-				}
-			}
-		}
-		nlripMap[endpoint] = rslt
-		locker.mutex.Unlock()
-	}
-	return nlripMap
-}
-
-// lockMaintenance loops over locks that have been active for some time and checks back
-// with the original server whether it is still alive or not
-//
-// Following logic inside ignores the errors generated for Dsync.Active operation.
-// - server at client down
-// - some network error (and server is up normally)
-//
-// We will ignore the error, and we will retry later to get a resolve on this lock
-func lockMaintenance(ctx context.Context, interval time.Duration) error {
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		return nil
-	}
-
-	z, ok := objAPI.(*erasureServerSets)
-	if !ok {
-		return nil
-	}
-
-	type nlock struct {
-		locks  int
-		writer bool
-	}
-
-	updateNlocks := func(nlripsMap map[string]nlock, name string, writer bool) {
-		nlk, ok := nlripsMap[name]
-		if !ok {
-			nlripsMap[name] = nlock{
-				locks:  1,
-				writer: writer,
-			}
-		} else {
-			nlk.locks++
-			nlripsMap[name] = nlk
-		}
-	}
-
-	allLockersFn := z.GetAllLockers
-
-	// Validate if long lived locks are indeed clean.
-	// Get list of long lived locks to check for staleness.
-	for lendpoint, nlrips := range getLongLivedLocks(interval) {
-		nlripsMap := make(map[string]nlock, len(nlrips))
-		for _, nlrip := range nlrips {
-			for _, c := range allLockersFn() {
-				if !c.IsOnline() || c == nil {
-					continue
-				}
-
-				ctx, cancel := context.WithTimeout(GlobalContext, 5*time.Second)
-
-				// Call back to original server verify whether the lock is
-				// still active (based on name & uid)
-				expired, err := c.Expired(ctx, dsync.LockArgs{
-					Owner:     nlrip.lri.Owner,
-					UID:       nlrip.lri.UID,
-					Resources: []string{nlrip.name},
-				})
-				cancel()
-				if err != nil {
-					updateNlocks(nlripsMap, nlrip.name, nlrip.lri.Writer)
-					continue
-				}
-
-				if !expired {
-					updateNlocks(nlripsMap, nlrip.name, nlrip.lri.Writer)
-				}
-			}
-
-			// less than the quorum, we have locks expired.
-			if nlripsMap[nlrip.name].locks < nlrip.lri.Quorum {
-				// Purge the stale entry if it exists.
-				globalLockServers[lendpoint].removeEntryIfExists(nlrip)
-			}
-
-		}
-	}
-
-	return nil
-}
-
-// Start lock maintenance from all lock servers.
-func startLockMaintenance(ctx context.Context) {
+// lockMaintenance loops over all locks and discards locks
+// that have not been refreshed for some time.
+func lockMaintenance(ctx context.Context) {
 	// Wait until the object API is ready
 	// no need to start the lock maintenance
 	// if ObjectAPI is not initialized.
+
+	var objAPI ObjectLayer
+
 	for {
-		objAPI := newObjectLayerFn()
+		objAPI = newObjectLayerFn()
 		if objAPI == nil {
 			time.Sleep(time.Second)
 			continue
@@ -335,26 +246,26 @@ func startLockMaintenance(ctx context.Context) {
 		break
 	}
 
-	// Initialize a new ticker with a minute between each ticks.
-	ticker := time.NewTicker(lockMaintenanceInterval)
-	// Stop the timer upon service closure and cleanup the go-routine.
-	defer ticker.Stop()
+	if _, ok := objAPI.(*erasureServerSets); !ok {
+		return
+	}
+
+	// Initialize a new ticker with 1 minute between each ticks.
+	lkTimer := time.NewTimer(lockMaintenanceInterval)
+	// Stop the timer upon returning.
+	defer lkTimer.Stop()
 
-	r := rand.New(rand.NewSource(UTCNow().UnixNano()))
 	for {
 		// Verifies every minute for locks held more than 2 minutes.
 		select {
 		case <-ctx.Done():
 			return
-		case <-ticker.C:
-			// Start with random sleep time, so as to avoid
-			// "synchronous checks" between servers
-			duration := time.Duration(r.Float64() * float64(lockMaintenanceInterval))
-			time.Sleep(duration)
-			if err := lockMaintenance(ctx, lockValidityCheckInterval); err != nil {
-				// Sleep right after an error.
-				duration := time.Duration(r.Float64() * float64(lockMaintenanceInterval))
-				time.Sleep(duration)
+		case <-lkTimer.C:
+			// Reset the timer for next cycle.
+			lkTimer.Reset(lockMaintenanceInterval)
+
+			for _, lockServer := range globalLockServers {
+				lockServer.expireOldLocks(lockValidityDuration)
 			}
 		}
 	}
@@ -374,15 +285,16 @@ func registerLockRESTHandlers(router *mux.Router, endpointServerSets EndpointSer
 
 			subrouter := router.PathPrefix(path.Join(lockRESTPrefix, endpoint.Path)).Subrouter()
 			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodHealth).HandlerFunc(httpTraceHdrs(lockServer.HealthHandler))
+			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRefresh).HandlerFunc(httpTraceHdrs(lockServer.RefreshHandler))
 			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodLock).HandlerFunc(httpTraceHdrs(lockServer.LockHandler))
 			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRLock).HandlerFunc(httpTraceHdrs(lockServer.RLockHandler))
 			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodUnlock).HandlerFunc(httpTraceHdrs(lockServer.UnlockHandler))
 			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRUnlock).HandlerFunc(httpTraceHdrs(lockServer.RUnlockHandler))
-			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodExpired).HandlerFunc(httpTraceAll(lockServer.ExpiredHandler))
+			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodForceUnlock).HandlerFunc(httpTraceHdrs(lockServer.ForceUnlockHandler))
 
 			globalLockServers[endpoint] = lockServer.ll
 		}
 	}
 
-	go startLockMaintenance(GlobalContext)
+	go lockMaintenance(GlobalContext)
 }

cmd/logger/audit.go

@@ -18,14 +18,13 @@ package logger
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"net/http"
-	"net/url"
 	"strconv"
 	"time"
 
-	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger/message/audit"
 )
 
@@ -124,49 +123,85 @@ func (lrw *ResponseWriter) Size() int {
 	return lrw.bytesWritten
 }
 
+const contextAuditKey = contextKeyType("audit-entry")
+
+// SetAuditEntry sets Audit info in the context.
+func SetAuditEntry(ctx context.Context, audit *audit.Entry) context.Context {
+	if ctx == nil {
+		LogIf(context.Background(), fmt.Errorf("context is nil"))
+		return nil
+	}
+	return context.WithValue(ctx, contextAuditKey, audit)
+}
+
+// GetAuditEntry returns Audit entry if set.
+func GetAuditEntry(ctx context.Context) *audit.Entry {
+	if ctx != nil {
+		r, ok := ctx.Value(contextAuditKey).(*audit.Entry)
+		if ok {
+			return r
+		}
+		r = &audit.Entry{}
+		SetAuditEntry(ctx, r)
+		return r
+	}
+	return nil
+}
+
 // AuditLog - logs audit logs to all audit targets.
-func AuditLog(w http.ResponseWriter, r *http.Request, api string, reqClaims map[string]interface{}, filterKeys ...string) {
+func AuditLog(ctx context.Context, w http.ResponseWriter, r *http.Request, reqClaims map[string]interface{}, filterKeys ...string) {
 	// Fast exit if there is not audit target configured
 	if len(AuditTargets) == 0 {
 		return
 	}
 
-	var (
-		statusCode      int
-		timeToResponse  time.Duration
-		timeToFirstByte time.Duration
-	)
+	var entry audit.Entry
 
-	st, ok := w.(*ResponseWriter)
-	if ok {
-		statusCode = st.StatusCode
-		timeToResponse = time.Now().UTC().Sub(st.StartTime)
-		timeToFirstByte = st.TimeToFirstByte
-	}
+	if r != nil && w != nil {
+		reqInfo := GetReqInfo(ctx)
+		if reqInfo == nil {
+			return
+		}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-	object, err := url.PathUnescape(vars["object"])
-	if err != nil {
-		object = vars["object"]
-	}
+		entry = audit.ToEntry(w, r, reqClaims, globalDeploymentID)
+		entry.Trigger = "external-request"
+		for _, filterKey := range filterKeys {
+			delete(entry.ReqClaims, filterKey)
+			delete(entry.ReqQuery, filterKey)
+			delete(entry.ReqHeader, filterKey)
+			delete(entry.RespHeader, filterKey)
+		}
 
-	entry := audit.ToEntry(w, r, reqClaims, globalDeploymentID)
-	for _, filterKey := range filterKeys {
-		delete(entry.ReqClaims, filterKey)
-		delete(entry.ReqQuery, filterKey)
-		delete(entry.ReqHeader, filterKey)
-		delete(entry.RespHeader, filterKey)
-	}
-	entry.API.Name = api
-	entry.API.Bucket = bucket
-	entry.API.Object = object
-	entry.API.Status = http.StatusText(statusCode)
-	entry.API.StatusCode = statusCode
-	entry.API.TimeToResponse = strconv.FormatInt(timeToResponse.Nanoseconds(), 10) + "ns"
-	// ttfb will be recorded only for GET requests, Ignore such cases where ttfb will be empty.
-	if timeToFirstByte != 0 {
-		entry.API.TimeToFirstByte = strconv.FormatInt(timeToFirstByte.Nanoseconds(), 10) + "ns"
+		var (
+			statusCode      int
+			timeToResponse  time.Duration
+			timeToFirstByte time.Duration
+		)
+
+		st, ok := w.(*ResponseWriter)
+		if ok {
+			statusCode = st.StatusCode
+			timeToResponse = time.Now().UTC().Sub(st.StartTime)
+			timeToFirstByte = st.TimeToFirstByte
+		}
+
+		entry.API.Status = http.StatusText(statusCode)
+		entry.API.StatusCode = statusCode
+		entry.API.TimeToResponse = strconv.FormatInt(timeToResponse.Nanoseconds(), 10) + "ns"
+		// ttfb will be recorded only for GET requests, Ignore such cases where ttfb will be empty.
+		if timeToFirstByte != 0 {
+			entry.API.TimeToFirstByte = strconv.FormatInt(timeToFirstByte.Nanoseconds(), 10) + "ns"
+		}
+
+		entry.API.Name = reqInfo.API
+		entry.API.Bucket = reqInfo.BucketName
+		entry.API.Object = reqInfo.ObjectName
+		entry.API.Objects = reqInfo.ObjectNames
+	} else {
+		auditEntry := GetAuditEntry(ctx)
+		if auditEntry != nil {
+			entry = *auditEntry
+		}
 	}
 
 	// Send audit logs only to http targets.

cmd/logger/message/audit/entry.go

@@ -34,13 +34,14 @@ type Entry struct {
 	DeploymentID string `json:"deploymentid,omitempty"`
 	Time         string `json:"time"`
 	API          struct {
-		Name            string `json:"name,omitempty"`
-		Bucket          string `json:"bucket,omitempty"`
-		Object          string `json:"object,omitempty"`
-		Status          string `json:"status,omitempty"`
-		StatusCode      int    `json:"statusCode,omitempty"`
-		TimeToFirstByte string `json:"timeToFirstByte,omitempty"`
-		TimeToResponse  string `json:"timeToResponse,omitempty"`
+		Name            string   `json:"name,omitempty"`
+		Bucket          string   `json:"bucket,omitempty"`
+		Object          string   `json:"object,omitempty"`
+		Objects         []string `json:"objects,omitempty"`
+		Status          string   `json:"status,omitempty"`
+		StatusCode      int      `json:"statusCode,omitempty"`
+		TimeToFirstByte string   `json:"timeToFirstByte,omitempty"`
+		TimeToResponse  string   `json:"timeToResponse,omitempty"`
 	} `json:"api"`
 	RemoteHost string                 `json:"remotehost,omitempty"`
 	RequestID  string                 `json:"requestID,omitempty"`
@@ -49,10 +50,24 @@ type Entry struct {
 	ReqQuery   map[string]string      `json:"requestQuery,omitempty"`
 	ReqHeader  map[string]string      `json:"requestHeader,omitempty"`
 	RespHeader map[string]string      `json:"responseHeader,omitempty"`
+
+	Trigger string `json:"trigger,omitempty"`
+}
+
+// NewEntry initializes a new audit entry
+func NewEntry(deploymentID string) Entry {
+	return Entry{
+		Version:      Version,
+		DeploymentID: deploymentID,
+		Time:         time.Now().UTC().Format(time.RFC3339Nano),
+	}
+
 }
 
 // ToEntry - constructs an audit entry object.
 func ToEntry(w http.ResponseWriter, r *http.Request, reqClaims map[string]interface{}, deploymentID string) Entry {
+	entry := NewEntry(deploymentID)
+
 	q := r.URL.Query()
 	reqQuery := make(map[string]string, len(q))
 	for k, v := range q {
@@ -69,18 +84,13 @@ func ToEntry(w http.ResponseWriter, r *http.Request, reqClaims map[string]interf
 	}
 	respHeader[xhttp.ETag] = strings.Trim(respHeader[xhttp.ETag], `"`)
 
-	entry := Entry{
-		Version:      Version,
-		DeploymentID: deploymentID,
-		RemoteHost:   handlers.GetSourceIP(r),
-		RequestID:    wh.Get(xhttp.AmzRequestID),
-		UserAgent:    r.UserAgent(),
-		Time:         time.Now().UTC().Format(time.RFC3339Nano),
-		ReqQuery:     reqQuery,
-		ReqHeader:    reqHeader,
-		ReqClaims:    reqClaims,
-		RespHeader:   respHeader,
-	}
+	entry.RemoteHost = handlers.GetSourceIP(r)
+	entry.RequestID = wh.Get(xhttp.AmzRequestID)
+	entry.UserAgent = r.UserAgent()
+	entry.ReqQuery = reqQuery
+	entry.ReqHeader = reqHeader
+	entry.ReqClaims = reqClaims
+	entry.RespHeader = respHeader
 
 	return entry
 }

cmd/logger/reqinfo.go

@@ -43,6 +43,7 @@ type ReqInfo struct {
 	API          string   // API name - GetObject PutObject NewMultipartUpload etc.
 	BucketName   string   // Bucket name
 	ObjectName   string   // Object name
+	ObjectNames  []string // Object names for Multi delete API
 	tags         []KeyVal // Any additional info not accommodated by above fields
 	sync.RWMutex
 }

cmd/metrics.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/cmd/rest"
 	"github.com/minio/minio/pkg/env"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -288,6 +289,15 @@ func cacheMetricsPrometheus(ch chan<- prometheus.Metric) {
 func httpMetricsPrometheus(ch chan<- prometheus.Metric) {
 	httpStats := globalHTTPStats.toServerHTTPStats()
 
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName("s3", "requests", "queue"),
+			"Total number of s3 requests waiting in the queue in the current MinIO server instance",
+			nil, nil),
+		prometheus.CounterValue,
+		float64(httpStats.TotalClientsInQueue),
+	)
+
 	for api, value := range httpStats.CurrentS3Requests.APIStats {
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(
@@ -325,6 +335,12 @@ func httpMetricsPrometheus(ch chan<- prometheus.Metric) {
 	}
 }
 
+// This is used by metrics to show the number of failed RPC calls between internodes
+func loadAndResetRPCNetworkErrsCounter() uint64 {
+	defer rest.ResetNetworkErrsCounter()
+	return rest.GetNetworkErrsCounter()
+}
+
 // collects network metrics for MinIO server in Prometheus specific format
 // and sends to given channel
 func networkMetricsPrometheus(ch chan<- prometheus.Metric) {
@@ -349,6 +365,15 @@ func networkMetricsPrometheus(ch chan<- prometheus.Metric) {
 		float64(connStats.TotalInputBytes),
 	)
 
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName("internode", "failed", "requests"),
+			"Total number of internode failed http requests",
+			nil, nil),
+		prometheus.CounterValue,
+		float64(loadAndResetRPCNetworkErrsCounter()),
+	)
+
 	// Network Sent/Received Bytes (Outbound)
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
@@ -367,6 +392,7 @@ func networkMetricsPrometheus(ch chan<- prometheus.Metric) {
 		prometheus.CounterValue,
 		float64(connStats.S3InputBytes),
 	)
+
 }
 
 // Populates prometheus with bucket usage metrics, this metrics

cmd/namespace-lock.go

@@ -1,18 +1,19 @@
-/*
- * MinIO Cloud Storage, (C) 2016, 2017, 2018, 2019 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package cmd
 
@@ -38,10 +39,28 @@ var globalLockServers = make(map[Endpoint]*localLocker)
 
 // RWLocker - locker interface to introduce GetRLock, RUnlock.
 type RWLocker interface {
-	GetLock(timeout *dynamicTimeout) (timedOutErr error)
-	Unlock()
-	GetRLock(timeout *dynamicTimeout) (timedOutErr error)
-	RUnlock()
+	GetLock(ctx context.Context, timeout *dynamicTimeout) (lkCtx LockContext, timedOutErr error)
+	Unlock(cancel context.CancelFunc)
+	GetRLock(ctx context.Context, timeout *dynamicTimeout) (lkCtx LockContext, timedOutErr error)
+	RUnlock(cancel context.CancelFunc)
+}
+
+// LockContext lock context holds the lock backed context and canceler for the context.
+type LockContext struct {
+	ctx    context.Context
+	cancel context.CancelFunc
+}
+
+// Context returns lock context
+func (l LockContext) Context() context.Context {
+	return l.ctx
+}
+
+// Cancel function calls cancel() function
+func (l LockContext) Cancel() {
+	if l.cancel != nil {
+		l.cancel()
+	}
 }
 
 // newNSLock - return a new name space lock map.
@@ -139,52 +158,60 @@ func (n *nsLockMap) unlock(volume string, path string, readLock bool) {
 type distLockInstance struct {
 	rwMutex *dsync.DRWMutex
 	opsID   string
-	ctx     context.Context
 }
 
 // Lock - block until write lock is taken or timeout has occurred.
-func (di *distLockInstance) GetLock(timeout *dynamicTimeout) (timedOutErr error) {
+func (di *distLockInstance) GetLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) {
 	lockSource := getSource(2)
 	start := UTCNow()
 
-	if !di.rwMutex.GetLock(di.ctx, di.opsID, lockSource, dsync.Options{
+	newCtx, cancel := context.WithCancel(ctx)
+	if !di.rwMutex.GetLock(newCtx, cancel, di.opsID, lockSource, dsync.Options{
 		Timeout: timeout.Timeout(),
 	}) {
 		timeout.LogFailure()
-		return OperationTimedOut{}
+		cancel()
+		return LockContext{ctx: ctx, cancel: func() {}}, OperationTimedOut{}
 	}
 	timeout.LogSuccess(UTCNow().Sub(start))
-	return nil
+	return LockContext{ctx: newCtx, cancel: cancel}, nil
 }
 
 // Unlock - block until write lock is released.
-func (di *distLockInstance) Unlock() {
+func (di *distLockInstance) Unlock(cancel context.CancelFunc) {
+	if cancel != nil {
+		cancel()
+	}
 	di.rwMutex.Unlock()
 }
 
 // RLock - block until read lock is taken or timeout has occurred.
-func (di *distLockInstance) GetRLock(timeout *dynamicTimeout) (timedOutErr error) {
+func (di *distLockInstance) GetRLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) {
 	lockSource := getSource(2)
 	start := UTCNow()
 
-	if !di.rwMutex.GetRLock(di.ctx, di.opsID, lockSource, dsync.Options{
+	newCtx, cancel := context.WithCancel(ctx)
+	if !di.rwMutex.GetRLock(ctx, cancel, di.opsID, lockSource, dsync.Options{
 		Timeout: timeout.Timeout(),
 	}) {
 		timeout.LogFailure()
-		return OperationTimedOut{}
+		cancel()
+		return LockContext{ctx: ctx, cancel: func() {}}, OperationTimedOut{}
 	}
 	timeout.LogSuccess(UTCNow().Sub(start))
-	return nil
+	return LockContext{ctx: newCtx, cancel: cancel}, nil
 }
 
 // RUnlock - block until read lock is released.
-func (di *distLockInstance) RUnlock() {
+func (di *distLockInstance) RUnlock(cancel context.CancelFunc) {
+	if cancel != nil {
+		cancel()
+	}
 	di.rwMutex.RUnlock()
 }
 
 // localLockInstance - frontend/top-level interface for namespace locks.
 type localLockInstance struct {
-	ctx    context.Context
 	ns     *nsLockMap
 	volume string
 	paths  []string
@@ -194,69 +221,79 @@ type localLockInstance struct {
 // NewNSLock - returns a lock instance for a given volume and
 // path. The returned lockInstance object encapsulates the nsLockMap,
 // volume, path and operation ID.
-func (n *nsLockMap) NewNSLock(ctx context.Context, lockers func() ([]dsync.NetLocker, string), volume string, paths ...string) RWLocker {
+func (n *nsLockMap) NewNSLock(lockers func() ([]dsync.NetLocker, string), volume string, paths ...string) RWLocker {
 	opsID := mustGetUUID()
 	if n.isDistErasure {
 		drwmutex := dsync.NewDRWMutex(&dsync.Dsync{
 			GetLockers: lockers,
 		}, pathsJoinPrefix(volume, paths...)...)
-		return &distLockInstance{drwmutex, opsID, ctx}
+		return &distLockInstance{drwmutex, opsID}
 	}
 	sort.Strings(paths)
-	return &localLockInstance{ctx, n, volume, paths, opsID}
+	return &localLockInstance{n, volume, paths, opsID}
 }
 
 // Lock - block until write lock is taken or timeout has occurred.
-func (li *localLockInstance) GetLock(timeout *dynamicTimeout) (timedOutErr error) {
+func (li *localLockInstance) GetLock(ctx context.Context, timeout *dynamicTimeout) (_ LockContext, timedOutErr error) {
 	lockSource := getSource(2)
 	start := UTCNow()
-	readLock := false
-	var success []int
+	const readLock = false
+	success := make([]int, len(li.paths))
 	for i, path := range li.paths {
-		if !li.ns.lock(li.ctx, li.volume, path, lockSource, li.opsID, readLock, timeout.Timeout()) {
+		if !li.ns.lock(ctx, li.volume, path, lockSource, li.opsID, readLock, timeout.Timeout()) {
 			timeout.LogFailure()
-			for _, sint := range success {
-				li.ns.unlock(li.volume, li.paths[sint], readLock)
+			for si, sint := range success {
+				if sint == 1 {
+					li.ns.unlock(li.volume, li.paths[si], readLock)
+				}
 			}
-			return OperationTimedOut{}
+			return LockContext{}, OperationTimedOut{}
 		}
-		success = append(success, i)
+		success[i] = 1
 	}
 	timeout.LogSuccess(UTCNow().Sub(start))
-	return
+	return LockContext{ctx: ctx, cancel: func() {}}, nil
 }
 
 // Unlock - block until write lock is released.
-func (li *localLockInstance) Unlock() {
-	readLock := false
+func (li *localLockInstance) Unlock(cancel context.CancelFunc) {
+	if cancel != nil {
+		cancel()
+	}
+	const readLock = false
 	for _, path := range li.paths {
 		li.ns.unlock(li.volume, path, readLock)
 	}
 }
 
 // RLock - block until read lock is taken or timeout has occurred.
-func (li *localLockInstance) GetRLock(timeout *dynamicTimeout) (timedOutErr error) {
+func (li *localLockInstance) GetRLock(ctx context.Context, timeout *dynamicTimeout) (_ LockContext, timedOutErr error) {
 	lockSource := getSource(2)
 	start := UTCNow()
-	readLock := true
-	var success []int
+	const readLock = true
+	success := make([]int, len(li.paths))
 	for i, path := range li.paths {
-		if !li.ns.lock(li.ctx, li.volume, path, lockSource, li.opsID, readLock, timeout.Timeout()) {
+		if !li.ns.lock(ctx, li.volume, path, lockSource, li.opsID, readLock, timeout.Timeout()) {
 			timeout.LogFailure()
-			for _, sint := range success {
-				li.ns.unlock(li.volume, li.paths[sint], readLock)
+			for si, sint := range success {
+				if sint == 1 {
+					li.ns.unlock(li.volume, li.paths[si], readLock)
+				}
 			}
-			return OperationTimedOut{}
+			return LockContext{}, OperationTimedOut{}
 		}
-		success = append(success, i)
+		success[i] = 1
 	}
 	timeout.LogSuccess(UTCNow().Sub(start))
-	return
+	return LockContext{ctx: ctx, cancel: func() {}}, nil
 }
 
 // RUnlock - block until read lock is released.
-func (li *localLockInstance) RUnlock() {
-	readLock := true
+func (li *localLockInstance) RUnlock(cancel context.CancelFunc) {
+	if cancel != nil {
+		cancel()
+	}
+	const readLock = true
 	for _, path := range li.paths {
 		li.ns.unlock(li.volume, path, readLock)
 	}

cmd/naughty-disk_test.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"io"
 	"sync"
+	"time"
 )
 
 // naughtyDisk wraps a POSIX disk and returns programmed errors
@@ -54,6 +55,10 @@ func (d *naughtyDisk) IsOnline() bool {
 	return d.disk.IsOnline()
 }
 
+func (d *naughtyDisk) LastConn() time.Time {
+	return d.disk.LastConn()
+}
+
 func (d *naughtyDisk) IsLocal() bool {
 	return d.disk.IsLocal()
 }

cmd/notification.go

@@ -138,21 +138,6 @@ func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, a
 	}()
 }
 
-// ReloadFormat - calls ReloadFormat REST call on all peers.
-func (sys *NotificationSys) ReloadFormat(dryRun bool) []NotificationPeerErr {
-	ng := WithNPeers(len(sys.peerClients))
-	for idx, client := range sys.peerClients {
-		if client == nil {
-			continue
-		}
-		client := client
-		ng.Go(GlobalContext, func() error {
-			return client.ReloadFormat(dryRun)
-		}, idx, *client.host)
-	}
-	return ng.Wait()
-}
-
 // DeletePolicy - deletes policy across all peers.
 func (sys *NotificationSys) DeletePolicy(policyName string) []NotificationPeerErr {
 	ng := WithNPeers(len(sys.peerClients))

cmd/object-api-common.go

@@ -89,7 +89,7 @@ func cleanupDir(ctx context.Context, storage StorageAPI, volume, dirPath string)
 	delFunc = func(entryPath string) error {
 		if !HasSuffix(entryPath, SlashSeparator) {
 			// Delete the file entry.
-			err := storage.DeleteFile(ctx, volume, entryPath)
+			err := storage.DeleteFile(ctx, volume, entryPath, false)
 			if !IsErrIgnored(err, []error{
 				errDiskNotFound,
 				errUnformattedDisk,
@@ -118,7 +118,7 @@ func cleanupDir(ctx context.Context, storage StorageAPI, volume, dirPath string)
 
 		// Entry path is empty, just delete it.
 		if len(entries) == 0 {
-			err = storage.DeleteFile(ctx, volume, entryPath)
+			err = storage.DeleteFile(ctx, volume, entryPath, false)
 			if !IsErrIgnored(err, []error{
 				errDiskNotFound,
 				errUnformattedDisk,

cmd/object-api-interface.go

@@ -45,6 +45,9 @@ type ObjectOptions struct {
 	UserDefined          map[string]string   // only set in case of POST/PUT operations
 	PartNumber           int                 // only useful in case of GetObject/HeadObject
 	CheckPrecondFn       CheckPreconditionFn // only set during GetObject/HeadObject/CopyObjectPart preconditional valuation
+
+	// Force performing action
+	DeletePrefix bool
 }
 
 // BucketOptions represents bucket options for ObjectLayer bucket operations
@@ -68,7 +71,7 @@ type ObjectLayer interface {
 	SetDriveCount() int // Only implemented by erasure layer
 
 	// Locking operations on object.
-	NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker
+	NewNSLock(bucket string, objects ...string) RWLocker
 
 	// Storage operations.
 	Shutdown(context.Context) error
@@ -114,7 +117,6 @@ type ObjectLayer interface {
 	CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error)
 
 	// Healing operations.
-	ReloadFormat(ctx context.Context, dryRun bool) error
 	HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error)
 	HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (madmin.HealResultItem, error)
 	HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (madmin.HealResultItem, error)

cmd/object-api-options.go

@@ -116,11 +116,21 @@ func getOpts(ctx context.Context, r *http.Request, bucket, object string) (Objec
 		}, nil
 	}
 
+	deletePrefix := false
+	if d := r.Header.Get("x-minio-force-delete"); d != "" {
+		if b, err := strconv.ParseBool(d); err != nil {
+			return opts, err
+		} else {
+			deletePrefix = b
+		}
+	}
+
 	// default case of passing encryption headers to backend
 	opts, err = getDefaultOpts(r.Header, false, nil)
 	if err != nil {
 		return opts, err
 	}
+	opts.DeletePrefix = deletePrefix
 	opts.PartNumber = partNumber
 	opts.VersionID = vid
 	return opts, nil

cmd/object-handlers.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"encoding/hex"
 	"encoding/xml"
+	"errors"
 	"io"
 	"net/http"
 	"net/url"
@@ -91,7 +92,7 @@ func setHeadGetRespHeaders(w http.ResponseWriter, reqParams url.Values) {
 func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SelectObject")
 
-	defer logger.AuditLog(w, r, "SelectObject", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Fetch object stat info.
 	objectAPI := api.ObjectAPI()
@@ -299,7 +300,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetObject")
 
-	defer logger.AuditLog(w, r, "GetObject", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -504,7 +505,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HeadObject")
 
-	defer logger.AuditLog(w, r, "HeadObject", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -788,7 +789,7 @@ func isRemoteCallRequired(ctx context.Context, bucket string, objAPI ObjectLayer
 func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "CopyObject")
 
-	defer logger.AuditLog(w, r, "CopyObject", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -1299,7 +1300,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 //   - X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key
 func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutObject")
-	defer logger.AuditLog(w, r, "PutObject", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -1601,7 +1602,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "NewMultipartUpload")
 
-	defer logger.AuditLog(w, r, "NewMultipartUpload", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -1729,7 +1730,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "CopyObjectPart")
 
-	defer logger.AuditLog(w, r, "CopyObjectPart", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -2045,7 +2046,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutObjectPart")
 
-	defer logger.AuditLog(w, r, "PutObjectPart", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -2292,7 +2293,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AbortMultipartUpload")
 
-	defer logger.AuditLog(w, r, "AbortMultipartUpload", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -2332,7 +2333,7 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectParts")
 
-	defer logger.AuditLog(w, r, "ListObjectParts", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -2471,7 +2472,7 @@ func sendWhiteSpace(w http.ResponseWriter) <-chan bool {
 func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "CompleteMultipartUpload")
 
-	defer logger.AuditLog(w, r, "CompleteMultipartUpload", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -2666,7 +2667,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteObject")
 
-	defer logger.AuditLog(w, r, "DeleteObject", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -2708,6 +2709,10 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 
 	apiErr := ErrNone
 	if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
+		if opts.DeletePrefix {
+			writeErrorResponse(ctx, w, toAPIError(ctx, errors.New("force-delete is not allowed on object locked buckets")), r.URL, guessIsBrowserReq(r))
+			return
+		}
 		if opts.VersionID != "" {
 			apiErr = enforceRetentionBypassForDelete(ctx, r, bucket, ObjectToDelete{
 				ObjectName: object,
@@ -2745,7 +2750,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutObjectLegalHold")
 
-	defer logger.AuditLog(w, r, "PutObjectLegalHold", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -2835,7 +2840,7 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetObjectLegalHold")
 
-	defer logger.AuditLog(w, r, "GetObjectLegalHold", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -2900,7 +2905,7 @@ func (api objectAPIHandlers) GetObjectLegalHoldHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutObjectRetention")
 
-	defer logger.AuditLog(w, r, "PutObjectRetention", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -2998,7 +3003,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 // GetObjectRetentionHandler - get object retention configuration of object,
 func (api objectAPIHandlers) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetObjectRetention")
-	defer logger.AuditLog(w, r, "GetObjectRetention", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -3058,7 +3063,7 @@ func (api objectAPIHandlers) GetObjectRetentionHandler(w http.ResponseWriter, r
 // GetObjectTaggingHandler - GET object tagging
 func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetObjectTagging")
-	defer logger.AuditLog(w, r, "GetObjectTagging", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -3108,7 +3113,7 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
 // PutObjectTaggingHandler - PUT object tagging
 func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutObjectTagging")
-	defer logger.AuditLog(w, r, "PutObjectTagging", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -3163,7 +3168,7 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
 // DeleteObjectTaggingHandler - DELETE object tagging
 func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteObjectTagging")
-	defer logger.AuditLog(w, r, "DeleteObjectTagging", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {

cmd/peer-rest-client.go

@@ -454,19 +454,6 @@ func (client *peerRESTClient) DeleteBucketMetadata(bucket string) error {
 	return nil
 }
 
-// ReloadFormat - reload format on the peer node.
-func (client *peerRESTClient) ReloadFormat(dryRun bool) error {
-	values := make(url.Values)
-	values.Set(peerRESTDryRun, strconv.FormatBool(dryRun))
-
-	respBody, err := client.call(peerRESTMethodReloadFormat, values, nil, -1)
-	if err != nil {
-		return err
-	}
-	defer http.DrainBody(respBody)
-	return nil
-}
-
 // cycleServerBloomFilter will cycle the bloom filter to start recording to index y if not already.
 // The response will contain a bloom filter starting at index x up to, but not including index y.
 // If y is 0, the response will not update y, but return the currently recorded information
@@ -866,7 +853,10 @@ func newPeerRESTClient(peer *xnet.Host) *peerRESTClient {
 		ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
 		// Instantiate a new rest client for healthcheck
 		// to avoid recursive healthCheckFn()
-		respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).Call(ctx, peerRESTMethodHealth, nil, nil, -1)
+		healthCheckClient := rest.NewClient(serverURL, trFn, newAuthToken)
+		healthCheckClient.ExpectTimeouts = true
+		healthCheckClient.NoMetrics = true
+		respBody, err := healthCheckClient.Call(ctx, peerRESTMethodHealth, nil, nil, -1)
 		xhttp.DrainBody(respBody)
 		cancel()
 		var ne *rest.NetworkError

cmd/peer-rest-common.go

@@ -50,7 +50,6 @@ const (
 	peerRESTMethodLoadGroup             = "/loadgroup"
 	peerRESTMethodStartProfiling        = "/startprofiling"
 	peerRESTMethodDownloadProfilingData = "/downloadprofilingdata"
-	peerRESTMethodReloadFormat          = "/reloadformat"
 	peerRESTMethodCycleBloom            = "/cyclebloom"
 	peerRESTMethodTrace                 = "/trace"
 	peerRESTMethodListen                = "/listen"
@@ -70,7 +69,6 @@ const (
 	peerRESTIsGroup     = "is-group"
 	peerRESTSignal      = "signal"
 	peerRESTProfiler    = "profiler"
-	peerRESTDryRun      = "dry-run"
 	peerRESTTraceAll    = "all"
 	peerRESTTraceErr    = "err"
 

cmd/peer-rest-server.go

@@ -577,46 +577,7 @@ func (s *peerRESTServer) LoadBucketMetadataHandler(w http.ResponseWriter, r *htt
 	}
 }
 
-// ReloadFormatHandler - Reload Format.
-func (s *peerRESTServer) ReloadFormatHandler(w http.ResponseWriter, r *http.Request) {
-	if !s.IsValid(w, r) {
-		s.writeErrorResponse(w, errors.New("Invalid request"))
-		return
-	}
-
-	vars := mux.Vars(r)
-	dryRunString := vars[peerRESTDryRun]
-	if dryRunString == "" {
-		s.writeErrorResponse(w, errors.New("dry-run parameter is missing"))
-		return
-	}
-
-	var dryRun bool
-	switch strings.ToLower(dryRunString) {
-	case "true":
-		dryRun = true
-	case "false":
-		dryRun = false
-	default:
-		s.writeErrorResponse(w, errInvalidArgument)
-		return
-	}
-
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		s.writeErrorResponse(w, errServerNotInitialized)
-		return
-	}
-
-	err := objAPI.ReloadFormat(GlobalContext, dryRun)
-	if err != nil {
-		s.writeErrorResponse(w, err)
-		return
-	}
-	w.(http.Flusher).Flush()
-}
-
-// CycleServerBloomFilterHandler cycles bllom filter on server.
+// CycleServerBloomFilterHandler cycles bloom filter on server.
 func (s *peerRESTServer) CycleServerBloomFilterHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
 		s.writeErrorResponse(w, errors.New("Invalid request"))
@@ -1047,7 +1008,6 @@ func registerPeerRESTHandlers(router *mux.Router) {
 
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodStartProfiling).HandlerFunc(httpTraceAll(server.StartProfilingHandler)).Queries(restQueries(peerRESTProfiler)...)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDownloadProfilingData).HandlerFunc(httpTraceHdrs(server.DownloadProfilingDataHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadFormat).HandlerFunc(httpTraceHdrs(server.ReloadFormatHandler)).Queries(restQueries(peerRESTDryRun)...)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodTrace).HandlerFunc(server.TraceHandler)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodListen).HandlerFunc(httpTraceHdrs(server.ListenHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodBackgroundHealStatus).HandlerFunc(server.BackgroundHealStatusHandler)

cmd/prepare-storage.go

@@ -23,14 +23,12 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"path"
 	"sync"
 	"time"
 
 	"github.com/dustin/go-humanize"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
 var printEndpointError = func() func(Endpoint, error, bool) {
@@ -71,94 +69,50 @@ var printEndpointError = func() func(Endpoint, error, bool) {
 	}
 }()
 
-// Migrates backend format of local disks.
-func formatErasureMigrateLocalEndpoints(endpoints Endpoints) error {
-	g := errgroup.WithNErrs(len(endpoints))
-	for index, endpoint := range endpoints {
-		if !endpoint.IsLocal {
-			continue
-		}
-		index := index
-		g.Go(func() error {
-			epPath := endpoints[index].Path
-			formatPath := pathJoin(epPath, minioMetaBucket, formatConfigFile)
-			if _, err := os.Stat(formatPath); err != nil {
-				if os.IsNotExist(err) {
-					return nil
-				}
-				return fmt.Errorf("unable to access (%s) %w", formatPath, err)
-			}
-			return formatErasureMigrate(epPath)
-		}, index)
-	}
-	for _, err := range g.Wait() {
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// Cleans up tmp directory of local disks.
-func formatErasureCleanupTmpLocalEndpoints(endpoints Endpoints) error {
-	g := errgroup.WithNErrs(len(endpoints))
-	for index, endpoint := range endpoints {
-		if !endpoint.IsLocal {
-			continue
-		}
-		index := index
-		g.Go(func() error {
-			epPath := endpoints[index].Path
-			// If disk is not formatted there is nothing to be cleaned up.
-			formatPath := pathJoin(epPath, minioMetaBucket, formatConfigFile)
-			if _, err := os.Stat(formatPath); err != nil {
-				if os.IsNotExist(err) {
-					return nil
-				}
-				return fmt.Errorf("unable to access (%s) %w", formatPath, err)
-			}
-			if _, err := os.Stat(pathJoin(epPath, minioMetaTmpBucket+"-old")); err != nil {
-				if !os.IsNotExist(err) {
-					return fmt.Errorf("unable to access (%s) %w",
-						pathJoin(epPath, minioMetaTmpBucket+"-old"),
-						err)
-				}
-			}
-
-			// Need to move temporary objects left behind from previous run of minio
-			// server to a unique directory under `minioMetaTmpBucket-old` to clean
-			// up `minioMetaTmpBucket` for the current run.
-			//
-			// /disk1/.minio.sys/tmp-old/
-			//  |__ 33a58b40-aecc-4c9f-a22f-ff17bfa33b62
-			//  |__ e870a2c1-d09c-450c-a69c-6eaa54a89b3e
-			//
-			// In this example, `33a58b40-aecc-4c9f-a22f-ff17bfa33b62` directory contains
-			// temporary objects from one of the previous runs of minio server.
-			tmpOld := pathJoin(epPath, minioMetaTmpBucket+"-old", mustGetUUID())
-			if err := renameAll(pathJoin(epPath, minioMetaTmpBucket),
-				tmpOld); err != nil && err != errFileNotFound {
-				return fmt.Errorf("unable to rename (%s -> %s) %w",
-					pathJoin(epPath, minioMetaTmpBucket),
-					tmpOld,
-					osErrToFileErr(err))
-			}
-
-			// Removal of tmp-old folder is backgrounded completely.
-			go removeAll(pathJoin(epPath, minioMetaTmpBucket+"-old"))
-
-			if err := mkdirAll(pathJoin(epPath, minioMetaTmpBucket), 0777); err != nil {
-				return fmt.Errorf("unable to create (%s) %w",
-					pathJoin(epPath, minioMetaTmpBucket),
-					err)
-			}
+// Cleans up tmp directory of local disk.
+func formatErasureCleanupLocalTmp(diskPath string) error {
+	// If disk is not formatted there is nothing to be cleaned up.
+	formatPath := pathJoin(diskPath, minioMetaBucket, formatConfigFile)
+	if _, err := os.Stat(formatPath); err != nil {
+		if os.IsNotExist(err) {
 			return nil
-		}, index)
-	}
-	for _, err := range g.Wait() {
-		if err != nil {
-			return err
 		}
+		return fmt.Errorf("unable to access (%s) %w", formatPath, err)
+	}
+	if _, err := os.Stat(pathJoin(diskPath, minioMetaTmpBucket+"-old")); err != nil {
+		if !os.IsNotExist(err) {
+			return fmt.Errorf("unable to access (%s) %w",
+				pathJoin(diskPath, minioMetaTmpBucket+"-old"),
+				err)
+		}
+	}
+
+	// Need to move temporary objects left behind from previous run of minio
+	// server to a unique directory under `minioMetaTmpBucket-old` to clean
+	// up `minioMetaTmpBucket` for the current run.
+	//
+	// /disk1/.minio.sys/tmp-old/
+	//  |__ 33a58b40-aecc-4c9f-a22f-ff17bfa33b62
+	//  |__ e870a2c1-d09c-450c-a69c-6eaa54a89b3e
+	//
+	// In this example, `33a58b40-aecc-4c9f-a22f-ff17bfa33b62` directory contains
+	// temporary objects from one of the previous runs of minio server.
+	tmpOld := pathJoin(diskPath, minioMetaTmpBucket+"-old", mustGetUUID())
+	if err := renameAll(pathJoin(diskPath, minioMetaTmpBucket),
+		tmpOld); err != nil && err != errFileNotFound {
+		return fmt.Errorf("unable to rename (%s -> %s) %w",
+			pathJoin(diskPath, minioMetaTmpBucket),
+			tmpOld,
+			osErrToFileErr(err))
+	}
+
+	// Removal of tmp-old folder is backgrounded completely.
+	go removeAll(pathJoin(diskPath, minioMetaTmpBucket+"-old"))
+
+	if err := mkdirAll(pathJoin(diskPath, minioMetaTmpBucket), 0777); err != nil {
+		return fmt.Errorf("unable to create (%s) %w",
+			pathJoin(diskPath, minioMetaTmpBucket),
+			err)
 	}
 	return nil
 }
@@ -177,7 +131,7 @@ func IsServerResolvable(endpoint Endpoint) error {
 	serverURL := &url.URL{
 		Scheme: endpoint.Scheme,
 		Host:   endpoint.Host,
-		Path:   path.Join(healthCheckPathPrefix, healthCheckLivenessPath),
+		Path:   pathJoin(healthCheckPathPrefix, healthCheckLivenessPath),
 	}
 
 	var tlsConfig *tls.Config
@@ -195,9 +149,9 @@ func IsServerResolvable(endpoint Endpoint) error {
 		&http.Transport{
 			Proxy:                 http.ProxyFromEnvironment,
 			DialContext:           xhttp.NewCustomDialContext(3 * time.Second),
-			ResponseHeaderTimeout: 5 * time.Second,
-			TLSHandshakeTimeout:   5 * time.Second,
-			ExpectContinueTimeout: 5 * time.Second,
+			ResponseHeaderTimeout: 3 * time.Second,
+			TLSHandshakeTimeout:   3 * time.Second,
+			ExpectContinueTimeout: 3 * time.Second,
 			TLSClientConfig:       tlsConfig,
 			// Go net/http automatically unzip if content-type is
 			// gzip disable this feature, as we are always interested
@@ -207,23 +161,29 @@ func IsServerResolvable(endpoint Endpoint) error {
 	}
 	defer httpClient.CloseIdleConnections()
 
-	ctx, cancel := context.WithTimeout(GlobalContext, 5*time.Second)
-	defer cancel()
+	ctx, cancel := context.WithTimeout(GlobalContext, 3*time.Second)
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, serverURL.String(), nil)
 	if err != nil {
+		cancel()
 		return err
 	}
 
 	resp, err := httpClient.Do(req)
+	cancel()
 	if err != nil {
 		return err
 	}
-	defer xhttp.DrainBody(resp.Body)
+	xhttp.DrainBody(resp.Body)
 
 	if resp.StatusCode != http.StatusOK {
 		return StorageErr(resp.Status)
 	}
+
+	if resp.Header.Get(xhttp.MinIOServerStatus) == unavailable {
+		return StorageErr(unavailable)
+	}
+
 	return nil
 }
 
@@ -242,13 +202,13 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 
 	for i, err := range errs {
 		if err != nil {
-			if err != errDiskNotFound {
-				return nil, nil, fmt.Errorf("Disk %s: %w", endpoints[i], err)
-			}
-			if retryCount >= 5 {
-				logger.Info("Unable to connect to %s: %v\n", endpoints[i], IsServerResolvable(endpoints[i]))
+			if err == errDiskNotFound && retryCount >= 5 {
+				logger.Info("Unable to connect to %s: %v", endpoints[i], IsServerResolvable(endpoints[i]))
+			} else {
+				logger.Info("Unable to use the drive %s: %v", endpoints[i], err)
 			}
 		}
+
 	}
 
 	// Attempt to load all `format.json` from all disks.
@@ -352,14 +312,6 @@ func waitForFormatErasure(firstDisk bool, endpoints Endpoints, zoneCount, setCou
 		return nil, nil, errInvalidArgument
 	}
 
-	if err := formatErasureMigrateLocalEndpoints(endpoints); err != nil {
-		return nil, nil, err
-	}
-
-	if err := formatErasureCleanupTmpLocalEndpoints(endpoints); err != nil {
-		return nil, nil, err
-	}
-
 	// prepare getElapsedTime() to calculate elapsed time since we started trying formatting disks.
 	// All times are rounded to avoid showing milli, micro and nano seconds
 	formatStartTime := time.Now().Round(time.Second)

cmd/rest/client.go

@@ -40,6 +40,19 @@ const (
 	closed
 )
 
+// Hold the number of failed RPC calls due to networking errors
+var networkErrsCounter uint64
+
+// GetNetworkErrsCounter returns the number of failed RPC requests
+func GetNetworkErrsCounter() uint64 {
+	return atomic.LoadUint64(&networkErrsCounter)
+}
+
+// ResetNetworkErrsCounter resets the number of failed RPC requests
+func ResetNetworkErrsCounter() {
+	atomic.StoreUint64(&networkErrsCounter, 0)
+}
+
 // NetworkError - error type in case of errors related to http/transport
 // for ex. connection refused, connection reset, dns resolution failure etc.
 // All errors returned by storage-rest-server (ex errFileNotFound, errDiskNotFound) are not considered to be network errors.
@@ -75,10 +88,18 @@ type Client struct {
 	// Should only be modified before any calls are made.
 	MaxErrResponseSize int64
 
+	// ExpectTimeouts indicates if context timeouts are expected.
+	// This will not mark the client offline in these cases.
+	ExpectTimeouts bool
+
+	// NoMetrics to avoid updating networking metrics if set to true
+	NoMetrics bool
+
 	httpClient   *http.Client
 	url          *url.URL
 	newAuthToken func(audience string) string
 	connected    int32
+	lastConn     int64
 }
 
 // URL query separator constants
@@ -112,7 +133,10 @@ func (c *Client) Call(ctx context.Context, method string, values url.Values, bod
 	}
 	resp, err := c.httpClient.Do(req)
 	if err != nil {
-		if xnet.IsNetworkOrHostDown(err) {
+		if xnet.IsNetworkOrHostDown(err, c.ExpectTimeouts) {
+			if !c.NoMetrics {
+				atomic.AddUint64(&networkErrsCounter, 1)
+			}
 			c.MarkOffline()
 		}
 		return nil, &NetworkError{err}
@@ -141,7 +165,10 @@ func (c *Client) Call(ctx context.Context, method string, values url.Values, bod
 		// Limit the ReadAll(), just in case, because of a bug, the server responds with large data.
 		b, err := ioutil.ReadAll(io.LimitReader(resp.Body, c.MaxErrResponseSize))
 		if err != nil {
-			if xnet.IsNetworkOrHostDown(err) {
+			if xnet.IsNetworkOrHostDown(err, c.ExpectTimeouts) {
+				if !c.NoMetrics {
+					atomic.AddUint64(&networkErrsCounter, 1)
+				}
 				c.MarkOffline()
 			}
 			return nil, err
@@ -169,6 +196,7 @@ func NewClient(url *url.URL, newCustomTransport func() *http.Transport, newAuthT
 		url:                 url,
 		newAuthToken:        newAuthToken,
 		connected:           online,
+		lastConn:            time.Now().UnixNano(),
 		MaxErrResponseSize:  4096,
 		HealthCheckInterval: 200 * time.Millisecond,
 		HealthCheckTimeout:  time.Second,
@@ -180,6 +208,11 @@ func (c *Client) IsOnline() bool {
 	return atomic.LoadInt32(&c.connected) == online
 }
 
+// LastConn returns the last date/time when the disk is connected/reconnected
+func (c *Client) LastConn() time.Time {
+	return time.Unix(0, atomic.LoadInt64(&c.lastConn))
+}
+
 // MarkOffline - will mark a client as being offline and spawns
 // a goroutine that will attempt to reconnect if HealthCheckFn is set.
 func (c *Client) MarkOffline() {
@@ -194,6 +227,7 @@ func (c *Client) MarkOffline() {
 				}
 				if c.HealthCheckFn() {
 					atomic.CompareAndSwapInt32(&c.connected, offline, online)
+					atomic.StoreInt64(&c.lastConn, time.Now().UnixNano())
 					return
 				}
 				time.Sleep(time.Duration(r.Float64() * float64(c.HealthCheckInterval)))

cmd/routers.go

@@ -39,6 +39,10 @@ func registerDistErasureRouters(router *mux.Router, endpointServerSets EndpointS
 
 // List of some generic handlers which are applied for all incoming requests.
 var globalHandlers = []MiddlewareFunc{
+	// add redirect handler to redirect
+	// requests when object layer is not
+	// initialized.
+	setRedirectHandler,
 	// set x-amz-request-id header.
 	addCustomHeaders,
 	// set HTTP security headers such as Content-Security-Policy.

cmd/server-main.go

@@ -199,9 +199,6 @@ func initServer(ctx context.Context, newObject ObjectLayer) error {
 	globalObjectAPI = newObject
 	globalObjLayerMutex.Unlock()
 
-	// Initialize IAM store
-	globalIAMSys.InitStore(newObject)
-
 	// Create cancel context to control 'newRetryTimer' go routine.
 	retryCtx, cancel := context.WithCancel(ctx)
 
@@ -213,7 +210,7 @@ func initServer(ctx context.Context, newObject ObjectLayer) error {
 	// at a given time, this big transaction lock ensures this
 	// appropriately. This is also true for rotation of encrypted
 	// content.
-	txnLk := newObject.NewNSLock(retryCtx, minioMetaBucket, minioConfigPrefix+"/transaction.lock")
+	txnLk := newObject.NewNSLock(minioMetaBucket, minioConfigPrefix+"/transaction.lock")
 
 	// allocate dynamic timeout once before the loop
 	configLockTimeout := newDynamicTimeout(5*time.Second, 3*time.Second)
@@ -235,7 +232,8 @@ func initServer(ctx context.Context, newObject ObjectLayer) error {
 	for range retry.NewTimerWithJitter(retryCtx, 500*time.Millisecond, time.Second, retry.MaxJitter) {
 		// let one of the server acquire the lock, if not let them timeout.
 		// which shall be retried again by this loop.
-		if err = txnLk.GetLock(configLockTimeout); err != nil {
+		lkctx, err := txnLk.GetLock(retryCtx, configLockTimeout)
+		if err != nil {
 			logger.Info("Waiting for all MinIO sub-systems to be initialized.. trying to acquire lock")
 			continue
 		}
@@ -252,7 +250,7 @@ func initServer(ctx context.Context, newObject ObjectLayer) error {
 			// Upon success migrating the config, initialize all sub-systems
 			// if all sub-systems initialized successfully return right away
 			if err = initAllSubsystems(ctx, newObject); err == nil {
-				txnLk.Unlock()
+				txnLk.Unlock(lkctx.Cancel)
 				// All successful return.
 				if globalIsDistErasure {
 					// These messages only meant primarily for distributed setup, so only log during distributed setup.
@@ -262,7 +260,7 @@ func initServer(ctx context.Context, newObject ObjectLayer) error {
 			}
 		}
 
-		txnLk.Unlock() // Unlock the transaction lock and allow other nodes to acquire the lock if possible.
+		txnLk.Unlock(lkctx.Cancel) // Unlock the transaction lock and allow other nodes to acquire the lock if possible.
 
 		// One of these retriable errors shall be retried.
 		if errors.Is(err, errDiskNotFound) ||
@@ -338,6 +336,9 @@ func initAllSubsystems(ctx context.Context, newObject ObjectLayer) (err error) {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize config, some features may be missing %w", err))
 	}
 
+	// Initialize IAM store
+	globalIAMSys.InitStore(newObject)
+
 	// Populate existing buckets to the etcd backend
 	if globalDNSConfig != nil {
 		// Background this operation.

cmd/storage-interface.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"io"
+	"time"
 )
 
 // StorageAPI interface.
@@ -28,6 +29,8 @@ type StorageAPI interface {
 
 	// Storage operations.
 	IsOnline() bool // Returns true if disk is online.
+	LastConn() time.Time
+
 	IsLocal() bool
 
 	Hostname() string   // Returns host name if remote host.
@@ -71,7 +74,7 @@ type StorageAPI interface {
 	RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error
 	CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error
 	CheckFile(ctx context.Context, volume string, path string) (err error)
-	DeleteFile(ctx context.Context, volume string, path string) (err error)
+	DeleteFile(ctx context.Context, volume string, path string, recursive bool) (err error)
 	VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error
 
 	// Write all data, syncs the data to disk.

cmd/storage-rest-client.go

@@ -29,6 +29,7 @@ import (
 	"path"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/minio/minio/cmd/http"
 	xhttp "github.com/minio/minio/cmd/http"
@@ -42,7 +43,7 @@ func isNetworkError(err error) bool {
 		return false
 	}
 	if nerr, ok := err.(*rest.NetworkError); ok {
-		return xnet.IsNetworkOrHostDown(nerr.Err)
+		return xnet.IsNetworkOrHostDown(nerr.Err, false)
 	}
 	return false
 }
@@ -144,6 +145,11 @@ func (client *storageRESTClient) IsOnline() bool {
 	return client.restClient.IsOnline()
 }
 
+// LastConn - returns when the disk is seen to be connected the last time
+func (client *storageRESTClient) LastConn() time.Time {
+	return client.restClient.LastConn()
+}
+
 func (client *storageRESTClient) IsLocal() bool {
 	return false
 }
@@ -512,6 +518,7 @@ func (client *storageRESTClient) Walk(ctx context.Context, volume, dirPath, mark
 	values.Set(storageRESTRecursive, strconv.FormatBool(recursive))
 	respBody, err := client.call(ctx, storageRESTMethodWalk, values, nil, -1)
 	if err != nil {
+		logger.LogIf(ctx, err)
 		return nil, err
 	}
 
@@ -525,6 +532,9 @@ func (client *storageRESTClient) Walk(ctx context.Context, volume, dirPath, mark
 			var fi FileInfo
 			if gerr := decoder.Decode(&fi); gerr != nil {
 				// Upon error return
+				if gerr != io.EOF {
+					logger.LogIf(ctx, gerr)
+				}
 				return
 			}
 			select {
@@ -555,10 +565,11 @@ func (client *storageRESTClient) ListDir(ctx context.Context, volume, dirPath st
 }
 
 // DeleteFile - deletes a file.
-func (client *storageRESTClient) DeleteFile(ctx context.Context, volume string, path string) error {
+func (client *storageRESTClient) DeleteFile(ctx context.Context, volume string, path string, recursive bool) error {
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
 	values.Set(storageRESTFilePath, path)
+	values.Set(storageRESTRecursive, strconv.FormatBool(recursive))
 	respBody, err := client.call(ctx, storageRESTMethodDeleteFile, values, nil, -1)
 	defer http.DrainBody(respBody)
 	return err
@@ -684,7 +695,10 @@ func newStorageRESTClient(endpoint Endpoint, healthcheck bool) *storageRESTClien
 			ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
 			// Instantiate a new rest client for healthcheck
 			// to avoid recursive healthCheckFn()
-			respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).Call(ctx, storageRESTMethodHealth, nil, nil, -1)
+			healthCheckClient := rest.NewClient(serverURL, trFn, newAuthToken)
+			healthCheckClient.ExpectTimeouts = true
+			healthCheckClient.NoMetrics = true
+			respBody, err := healthCheckClient.Call(ctx, storageRESTMethodHealth, nil, nil, -1)
 			xhttp.DrainBody(respBody)
 			cancel()
 			return !errors.Is(err, context.DeadlineExceeded) && toStorageErr(err) != errDiskNotFound

cmd/storage-rest-common.go

@@ -17,7 +17,7 @@
 package cmd
 
 const (
-	storageRESTVersion       = "v21" // Add checkDataDir in ReadVersion API
+	storageRESTVersion       = "v22" // Add recursive flag to DeleteFile call
 	storageRESTVersionPrefix = SlashSeparator + storageRESTVersion
 	storageRESTPrefix        = minioReservedBucketPath + "/storage"
 )

cmd/storage-rest-server.go

@@ -633,8 +633,15 @@ func (s *storageRESTServer) DeleteFileHandler(w http.ResponseWriter, r *http.Req
 	vars := mux.Vars(r)
 	volume := vars[storageRESTVolume]
 	filePath := vars[storageRESTFilePath]
+	recursive := false
+	if b, err := strconv.ParseBool(vars[storageRESTRecursive]); err == nil {
+		recursive = b
+	} else {
+		s.writeErrorResponse(w, err)
+		return
+	}
 
-	err := s.storage.DeleteFile(r.Context(), volume, filePath)
+	err := s.storage.DeleteFile(r.Context(), volume, filePath, recursive)
 	if err != nil {
 		s.writeErrorResponse(w, err)
 	}
@@ -960,7 +967,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerSets Endpoint
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteVersions).HandlerFunc(httpTraceHdrs(server.DeleteVersionsHandler)).
 				Queries(restQueries(storageRESTVolume, storageRESTTotalVersions)...)
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteFile).HandlerFunc(httpTraceHdrs(server.DeleteFileHandler)).
-				Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
+				Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTRecursive)...)
 
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodRenameFile).HandlerFunc(httpTraceHdrs(server.RenameFileHandler)).
 				Queries(restQueries(storageRESTSrcVolume, storageRESTSrcPath, storageRESTDstVolume, storageRESTDstPath)...)

cmd/sts-handlers.go

@@ -173,7 +173,7 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 	}
 
 	ctx = newContext(r, w, action)
-	defer logger.AuditLog(w, r, action, nil)
+	defer logger.AuditLog(ctx, w, r, nil)
 
 	sessionPolicyStr := r.Form.Get(stsPolicy)
 	// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
@@ -284,7 +284,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 	}
 
 	ctx = newContext(r, w, action)
-	defer logger.AuditLog(w, r, action, nil)
+	defer logger.AuditLog(ctx, w, r, nil)
 
 	if globalOpenIDValidators == nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSNotInitialized, errServerNotInitialized)
@@ -328,11 +328,12 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 	var policyName string
 	policySet, ok := iampolicy.GetPoliciesFromClaims(m, iamPolicyClaimNameOpenID())
 	if ok {
-		policyName = globalIAMSys.currentPolicies(strings.Join(policySet.ToSlice(), ","))
+		policyName = globalIAMSys.CurrentPolicies(strings.Join(policySet.ToSlice(), ","))
 	}
 
 	if policyName == "" && globalPolicyOPA == nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, fmt.Errorf("%s claim missing from the JWT token, credentials will not be generated", iamPolicyClaimNameOpenID()))
+		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue,
+			fmt.Errorf("%s claim missing from the JWT token, credentials will not be generated", iamPolicyClaimNameOpenID()))
 		return
 	}
 	m[iamPolicyClaimNameOpenID()] = policyName
@@ -436,7 +437,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithClientGrants(w http.ResponseWriter, r *
 func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AssumeRoleWithLDAPIdentity")
 
-	defer logger.AuditLog(w, r, "AssumeRoleWithLDAPIdentity", nil, stsLDAPPassword)
+	defer logger.AuditLog(ctx, w, r, nil, stsLDAPPassword)
 
 	// Parse the incoming form data.
 	if err := r.ParseForm(); err != nil {

cmd/tree-walk.go

@@ -24,8 +24,9 @@ import (
 
 // TreeWalkResult - Tree walk result carries results of tree walking.
 type TreeWalkResult struct {
-	entry string
-	end   bool
+	entry    string
+	emptyDir bool
+	end      bool
 }
 
 // Return entries that have prefix prefixEntry.
@@ -254,7 +255,7 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker
 		select {
 		case <-endWalkCh:
 			return false, errWalkAbort
-		case resultCh <- TreeWalkResult{entry: pathJoin(prefixDir, entry), end: isEOF}:
+		case resultCh <- TreeWalkResult{entry: pathJoin(prefixDir, entry), emptyDir: leafDir, end: isEOF}:
 		}
 	}
 

cmd/update.go

@@ -309,7 +309,7 @@ func downloadReleaseURL(u *url.URL, timeout time.Duration, mode string) (content
 		client := &http.Client{Transport: getUpdateTransport(timeout)}
 		resp, err := client.Do(req)
 		if err != nil {
-			if xnet.IsNetworkOrHostDown(err) {
+			if xnet.IsNetworkOrHostDown(err, false) {
 				return content, AdminError{
 					Code:       AdminUpdateURLNotReachable,
 					Message:    err.Error(),
@@ -501,7 +501,7 @@ func getUpdateReaderFromURL(u *url.URL, transport http.RoundTripper, mode string
 
 	resp, err := clnt.Do(req)
 	if err != nil {
-		if xnet.IsNetworkOrHostDown(err) {
+		if xnet.IsNetworkOrHostDown(err, false) {
 			return nil, AdminError{
 				Code:       AdminUpdateURLNotReachable,
 				Message:    err.Error(),

cmd/utils.go

@@ -27,6 +27,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -45,7 +46,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/madmin"
-	"golang.org/x/net/http2"
+	http2 "golang.org/x/net/http2"
 )
 
 const (
@@ -481,10 +482,6 @@ func newInternodeHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration)
 		DisableCompression: true,
 	}
 
-	if tlsConfig != nil {
-		http2.ConfigureTransport(tr)
-	}
-
 	return func() *http.Transport {
 		return tr
 	}
@@ -514,7 +511,7 @@ func newCustomHTTPProxyTransport(tlsConfig *tls.Config, dialTimeout time.Duratio
 	}
 }
 
-func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) func() *http.Transport {
+func newCustomHTTPTransportWithHTTP2(tlsConfig *tls.Config, dialTimeout time.Duration) func() *http.Transport {
 	// For more details about various values used here refer
 	// https://golang.org/pkg/net/http/#Transport documentation
 	tr := &http.Transport{
@@ -533,7 +530,33 @@ func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) fu
 	}
 
 	if tlsConfig != nil {
-		http2.ConfigureTransport(tr)
+		trhttp2, _ := http2.ConfigureTransports(tr)
+		if trhttp2 != nil {
+			trhttp2.DisableCompression = true
+		}
+	}
+
+	return func() *http.Transport {
+		return tr
+	}
+}
+
+func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) func() *http.Transport {
+	// For more details about various values used here refer
+	// https://golang.org/pkg/net/http/#Transport documentation
+	tr := &http.Transport{
+		Proxy:                 http.ProxyFromEnvironment,
+		DialContext:           xhttp.DialContextWithDNSCache(globalDNSCache, xhttp.NewInternodeDialContext(dialTimeout)),
+		MaxIdleConnsPerHost:   1024,
+		IdleConnTimeout:       15 * time.Second,
+		ResponseHeaderTimeout: 3 * time.Minute, // Set conservative timeouts for MinIO internode.
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 10 * time.Second,
+		TLSClientConfig:       tlsConfig,
+		// Go net/http automatically unzip if content-type is
+		// gzip disable this feature, as we are always interested
+		// in raw stream.
+		DisableCompression: true,
 	}
 
 	return func() *http.Transport {
@@ -559,6 +582,34 @@ func newGatewayHTTPTransport(timeout time.Duration) *http.Transport {
 	return tr
 }
 
+// NewRemoteTargetHTTPTransport returns a new http configuration
+// used while communicating with the remote replication targets.
+func NewRemoteTargetHTTPTransport() *http.Transport {
+	// For more details about various values used here refer
+	// https://golang.org/pkg/net/http/#Transport documentation
+	tr := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   15 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).DialContext,
+		MaxIdleConnsPerHost:   1024,
+		WriteBufferSize:       16 << 10, // 16KiB moving up from 4KiB default
+		ReadBufferSize:        16 << 10, // 16KiB moving up from 4KiB default
+		IdleConnTimeout:       15 * time.Second,
+		TLSHandshakeTimeout:   5 * time.Second,
+		ExpectContinueTimeout: 5 * time.Second,
+		TLSClientConfig: &tls.Config{
+			RootCAs: globalRootCAs,
+		},
+		// Go net/http automatically unzip if content-type is
+		// gzip disable this feature, as we are always interested
+		// in raw stream.
+		DisableCompression: true,
+	}
+	return tr
+}
+
 // Load the json (typically from disk file).
 func jsonLoad(r io.ReadSeeker, data interface{}) error {
 	if _, err := r.Seek(0, io.SeekStart); err != nil {

cmd/web-handlers.go

@@ -967,7 +967,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 	// obtain the claims here if possible, for audit logging.
 	claims, owner, authErr := webRequestAuthenticate(r)
 
-	defer logger.AuditLog(w, r, "WebUpload", claims.Map())
+	defer logger.AuditLog(ctx, w, r, claims.Map())
 
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
@@ -1210,7 +1210,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 
 	claims, owner, authErr := webTokenAuthenticate(r.URL.Query().Get("token"))
-	defer logger.AuditLog(w, r, "WebDownload", claims.Map())
+	defer logger.AuditLog(ctx, w, r, claims.Map())
 
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
@@ -1407,7 +1407,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 	claims, owner, authErr := webTokenAuthenticate(r.URL.Query().Get("token"))
 
 	ctx := newContext(r, w, "WebDownloadZip")
-	defer logger.AuditLog(w, r, "WebDownloadZip", claims.Map())
+	defer logger.AuditLog(ctx, w, r, claims.Map())
 
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {

cmd/xl-storage-disk-id-check.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"io"
+	"time"
 )
 
 // Detects change in underlying disk.
@@ -39,6 +40,10 @@ func (p *xlStorageDiskIDCheck) IsOnline() bool {
 	return storedDiskID == p.diskID
 }
 
+func (p *xlStorageDiskIDCheck) LastConn() time.Time {
+	return p.storage.LastConn()
+}
+
 func (p *xlStorageDiskIDCheck) IsLocal() bool {
 	return p.storage.IsLocal()
 }
@@ -56,6 +61,12 @@ func (p *xlStorageDiskIDCheck) Healing() bool {
 }
 
 func (p *xlStorageDiskIDCheck) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (dataUsageCache, error) {
+	select {
+	case <-ctx.Done():
+		return dataUsageCache{}, ctx.Err()
+	default:
+	}
+
 	if err := p.checkDiskStale(); err != nil {
 		return dataUsageCache{}, err
 	}
@@ -93,6 +104,12 @@ func (p *xlStorageDiskIDCheck) checkDiskStale() error {
 }
 
 func (p *xlStorageDiskIDCheck) DiskInfo(ctx context.Context) (info DiskInfo, err error) {
+	select {
+	case <-ctx.Done():
+		return DiskInfo{}, ctx.Err()
+	default:
+	}
+
 	info, err = p.storage.DiskInfo(ctx)
 	if err != nil {
 		return info, err
@@ -108,6 +125,12 @@ func (p *xlStorageDiskIDCheck) DiskInfo(ctx context.Context) (info DiskInfo, err
 }
 
 func (p *xlStorageDiskIDCheck) MakeVolBulk(ctx context.Context, volumes ...string) (err error) {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+
 	if err = p.checkDiskStale(); err != nil {
 		return err
 	}
@@ -115,6 +138,12 @@ func (p *xlStorageDiskIDCheck) MakeVolBulk(ctx context.Context, volumes ...strin
 }
 
 func (p *xlStorageDiskIDCheck) MakeVol(ctx context.Context, volume string) (err error) {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+
 	if err = p.checkDiskStale(); err != nil {
 		return err
 	}
@@ -122,6 +151,12 @@ func (p *xlStorageDiskIDCheck) MakeVol(ctx context.Context, volume string) (err
 }
 
 func (p *xlStorageDiskIDCheck) ListVols(ctx context.Context) ([]VolInfo, error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	default:
+	}
+
 	if err := p.checkDiskStale(); err != nil {
 		return nil, err
 	}
@@ -129,6 +164,12 @@ func (p *xlStorageDiskIDCheck) ListVols(ctx context.Context) ([]VolInfo, error)
 }
 
 func (p *xlStorageDiskIDCheck) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) {
+	select {
+	case <-ctx.Done():
+		return VolInfo{}, ctx.Err()
+	default:
+	}
+
 	if err = p.checkDiskStale(); err != nil {
 		return vol, err
 	}
@@ -136,6 +177,12 @@ func (p *xlStorageDiskIDCheck) StatVol(ctx context.Context, volume string) (vol
 }
 
 func (p *xlStorageDiskIDCheck) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+
 	if err = p.checkDiskStale(); err != nil {
 		return err
 	}
@@ -143,6 +190,12 @@ func (p *xlStorageDiskIDCheck) DeleteVol(ctx context.Context, volume string, for
 }
 
 func (p *xlStorageDiskIDCheck) WalkVersions(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	default:
+	}
+
 	if err := p.checkDiskStale(); err != nil {
 		return nil, err
 	}
@@ -166,6 +219,12 @@ func (p *xlStorageDiskIDCheck) WalkSplunk(ctx context.Context, volume, dirPath,
 }
 
 func (p *xlStorageDiskIDCheck) ListDir(ctx context.Context, volume, dirPath string, count int) ([]string, error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	default:
+	}
+
 	if err := p.checkDiskStale(); err != nil {
 		return nil, err
 	}
@@ -174,6 +233,12 @@ func (p *xlStorageDiskIDCheck) ListDir(ctx context.Context, volume, dirPath stri
 }
 
 func (p *xlStorageDiskIDCheck) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
+	select {
+	case <-ctx.Done():
+		return 0, ctx.Err()
+	default:
+	}
+
 	if err := p.checkDiskStale(); err != nil {
 		return 0, err
 	}
@@ -182,6 +247,12 @@ func (p *xlStorageDiskIDCheck) ReadFile(ctx context.Context, volume string, path
 }
 
 func (p *xlStorageDiskIDCheck) AppendFile(ctx context.Context, volume string, path string, buf []byte) (err error) {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+
 	if err = p.checkDiskStale(); err != nil {
 		return err
 	}
@@ -190,6 +261,12 @@ func (p *xlStorageDiskIDCheck) AppendFile(ctx context.Context, volume string, pa
 }
 
 func (p *xlStorageDiskIDCheck) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+
 	if err := p.checkDiskStale(); err != nil {
 		return err
 	}
@@ -198,6 +275,12 @@ func (p *xlStorageDiskIDCheck) CreateFile(ctx context.Context, volume, path stri
 }
 
 func (p *xlStorageDiskIDCheck) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	default:
+	}
+
 	if err := p.checkDiskStale(); err != nil {
 		return nil, err
 	}
@@ -206,6 +289,12 @@ func (p *xlStorageDiskIDCheck) ReadFileStream(ctx context.Context, volume, path
 }
 
 func (p *xlStorageDiskIDCheck) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+
 	if err := p.checkDiskStale(); err != nil {
 		return err
 	}
@@ -214,6 +303,12 @@ func (p *xlStorageDiskIDCheck) RenameFile(ctx context.Context, srcVolume, srcPat
 }
 
 func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, dstVolume, dstPath string) error {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+
 	if err := p.checkDiskStale(); err != nil {
 		return err
 	}
@@ -222,6 +317,12 @@ func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPat
 }
 
 func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (err error) {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+
 	if err = p.checkDiskStale(); err != nil {
 		return err
 	}
@@ -230,6 +331,12 @@ func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, pa
 }
 
 func (p *xlStorageDiskIDCheck) CheckFile(ctx context.Context, volume string, path string) (err error) {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+
 	if err = p.checkDiskStale(); err != nil {
 		return err
 	}
@@ -237,12 +344,18 @@ func (p *xlStorageDiskIDCheck) CheckFile(ctx context.Context, volume string, pat
 	return p.storage.CheckFile(ctx, volume, path)
 }
 
-func (p *xlStorageDiskIDCheck) DeleteFile(ctx context.Context, volume string, path string) (err error) {
+func (p *xlStorageDiskIDCheck) DeleteFile(ctx context.Context, volume string, path string, recursive bool) (err error) {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+
 	if err = p.checkDiskStale(); err != nil {
 		return err
 	}
 
-	return p.storage.DeleteFile(ctx, volume, path)
+	return p.storage.DeleteFile(ctx, volume, path, recursive)
 }
 
 func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) (errs []error) {
@@ -257,6 +370,12 @@ func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string
 }
 
 func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+
 	if err := p.checkDiskStale(); err != nil {
 		return err
 	}
@@ -265,6 +384,12 @@ func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path stri
 }
 
 func (p *xlStorageDiskIDCheck) WriteAll(ctx context.Context, volume string, path string, reader io.Reader) (err error) {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+
 	if err = p.checkDiskStale(); err != nil {
 		return err
 	}
@@ -273,6 +398,12 @@ func (p *xlStorageDiskIDCheck) WriteAll(ctx context.Context, volume string, path
 }
 
 func (p *xlStorageDiskIDCheck) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) (err error) {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+
 	if err = p.checkDiskStale(); err != nil {
 		return err
 	}
@@ -281,6 +412,12 @@ func (p *xlStorageDiskIDCheck) DeleteVersion(ctx context.Context, volume, path s
 }
 
 func (p *xlStorageDiskIDCheck) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) (err error) {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+
 	if err = p.checkDiskStale(); err != nil {
 		return err
 	}
@@ -289,6 +426,12 @@ func (p *xlStorageDiskIDCheck) WriteMetadata(ctx context.Context, volume, path s
 }
 
 func (p *xlStorageDiskIDCheck) ReadVersion(ctx context.Context, volume, path, versionID string, checkDataDir bool) (fi FileInfo, err error) {
+	select {
+	case <-ctx.Done():
+		return fi, ctx.Err()
+	default:
+	}
+
 	if err = p.checkDiskStale(); err != nil {
 		return fi, err
 	}
@@ -297,6 +440,12 @@ func (p *xlStorageDiskIDCheck) ReadVersion(ctx context.Context, volume, path, ve
 }
 
 func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	default:
+	}
+
 	if err = p.checkDiskStale(); err != nil {
 		return nil, err
 	}

cmd/xl-storage.go

@@ -279,6 +279,14 @@ func newXLStorage(ep Endpoint) (*xlStorage, error) {
 		rootDisk:         rootDisk,
 	}
 
+	if err := formatErasureMigrate(path); err != nil {
+		return nil, err
+	}
+
+	if err := formatErasureCleanupLocalTmp(path); err != nil {
+		return nil, err
+	}
+
 	// Success.
 	return p, nil
 }
@@ -333,6 +341,10 @@ func (s *xlStorage) IsOnline() bool {
 	return true
 }
 
+func (s *xlStorage) LastConn() time.Time {
+	return time.Time{}
+}
+
 func (s *xlStorage) IsLocal() bool {
 	return true
 }
@@ -974,9 +986,21 @@ func (s *xlStorage) WalkVersions(ctx context.Context, volume, dirPath, marker st
 		}
 
 		walkResultCh := startTreeWalk(GlobalContext, volume, dirPath, marker, recursive, listDir, s.isLeaf, s.isLeafDir, endWalkCh)
+		dirObjects := make(map[string]struct{})
 		for walkResult := range walkResultCh {
 			var fiv FileInfoVersions
-			if HasSuffix(walkResult.entry, SlashSeparator) {
+			if HasSuffix(walkResult.entry, SlashSeparator) && walkResult.emptyDir {
+				// Avoid listing empty directory, they are not supposed to exist,
+				// if they are found delete them.
+				deleteFile(volumeDir, walkResult.entry, false)
+				continue
+			} else if HasSuffix(walkResult.entry, SlashSeparator) && !walkResult.emptyDir {
+				_, dirObj := dirObjects[walkResult.entry]
+				if dirObj {
+					continue
+				}
+				dirObjects[walkResult.entry] = struct{}{}
+
 				fiv = FileInfoVersions{
 					Volume: volume,
 					Name:   walkResult.entry,
@@ -998,6 +1022,18 @@ func (s *xlStorage) WalkVersions(ctx context.Context, volume, dirPath, marker st
 				if err != nil {
 					continue
 				}
+				if HasSuffix(fiv.Name, globalDirSuffix) {
+					entry := strings.TrimSuffix(fiv.Name, globalDirSuffix) + slashSeparator
+					_, dirObj := dirObjects[entry]
+					if dirObj {
+						continue
+					}
+					if !recursive && len(fiv.Versions) >= 2 {
+						// Remove multiple versions for directory objects, in non-recursive
+						fiv.Versions = fiv.Versions[:1]
+					}
+					dirObjects[entry] = struct{}{}
+				}
 			}
 			select {
 			case ch <- fiv:
@@ -1059,14 +1095,25 @@ func (s *xlStorage) Walk(ctx context.Context, volume, dirPath, marker string, re
 		}
 
 		walkResultCh := startTreeWalk(GlobalContext, volume, dirPath, marker, recursive, listDir, s.isLeaf, s.isLeafDir, endWalkCh)
+		dirObjects := make(map[string]struct{})
 		for walkResult := range walkResultCh {
 			var fi FileInfo
-			if HasSuffix(walkResult.entry, SlashSeparator) {
+			if HasSuffix(walkResult.entry, SlashSeparator) && walkResult.emptyDir {
+				// Avoid listing empty directory, they are not supposed to exist,
+				// if they are found delete them.
+				deleteFile(volumeDir, walkResult.entry, false)
+				continue
+			} else if HasSuffix(walkResult.entry, SlashSeparator) && !walkResult.emptyDir {
+				_, dirObj := dirObjects[walkResult.entry]
+				if dirObj {
+					continue
+				}
 				fi = FileInfo{
 					Volume: volume,
 					Name:   walkResult.entry,
 					Mode:   os.ModeDir,
 				}
+				dirObjects[walkResult.entry] = struct{}{}
 			} else {
 				var err error
 				var xlMetaBuf []byte
@@ -1082,6 +1129,13 @@ func (s *xlStorage) Walk(ctx context.Context, volume, dirPath, marker string, re
 					// Ignore delete markers.
 					continue
 				}
+				if HasSuffix(fi.Name, globalDirSuffix) {
+					entry := strings.TrimSuffix(fi.Name, globalDirSuffix) + slashSeparator
+					if _, dirObj := dirObjects[entry]; dirObj {
+						continue
+					}
+					dirObjects[entry] = struct{}{}
+				}
 			}
 			select {
 			case ch <- fi:
@@ -1146,7 +1200,7 @@ func (s *xlStorage) DeleteVersions(ctx context.Context, volume string, versions
 // DeleteVersion - deletes FileInfo metadata for path at `xl.meta`
 func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) error {
 	if HasSuffix(path, SlashSeparator) {
-		return s.DeleteFile(ctx, volume, path)
+		return s.DeleteFile(ctx, volume, path, false)
 	}
 
 	buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile))
@@ -1980,7 +2034,7 @@ func deleteFile(basePath, deletePath string, recursive bool) error {
 }
 
 // DeleteFile - delete a file at path.
-func (s *xlStorage) DeleteFile(ctx context.Context, volume string, path string) (err error) {
+func (s *xlStorage) DeleteFile(ctx context.Context, volume string, path string, recursive bool) (err error) {
 	atomic.AddInt32(&s.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt32(&s.activeIOCount, -1)
@@ -2012,7 +2066,7 @@ func (s *xlStorage) DeleteFile(ctx context.Context, volume string, path string)
 	}
 
 	// Delete file and delete parent directory as well if its empty.
-	return deleteFile(volumeDir, filePath, false)
+	return deleteFile(volumeDir, filePath, recursive)
 }
 
 func (s *xlStorage) DeleteFileBulk(volume string, paths []string) (errs []error, err error) {
@@ -2256,16 +2310,8 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir,
 	}
 
 	// Remove parent dir of the source file if empty
-	if parentDir := slashpath.Dir(srcFilePath); isDirEmpty(parentDir) {
-		deleteFile(srcVolumeDir, parentDir, false)
-	}
-
-	if srcDataPath != "" {
-		if parentDir := slashpath.Dir(srcDataPath); isDirEmpty(parentDir) {
-			deleteFile(srcVolumeDir, parentDir, false)
-		}
-	}
-
+	parentDir := slashpath.Dir(srcFilePath)
+	deleteFile(srcVolumeDir, parentDir, false)
 	return nil
 }
 
@@ -2322,18 +2368,18 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum
 		// If source is a directory, we expect the destination to be non-existent but we
 		// we still need to allow overwriting an empty directory since it represents
 		// an object empty directory.
-		_, err = os.Stat(dstFilePath)
-		if isSysErrIO(err) {
-			return errFaultyDisk
-		}
-		if err == nil && !isDirEmpty(dstFilePath) {
-			return errFileAccessDenied
-		}
-		if err != nil && !os.IsNotExist(err) {
-			return err
-		}
-		// Empty destination remove it before rename.
-		if isDirEmpty(dstFilePath) {
+		dirInfo, err := os.Stat(dstFilePath)
+		if err != nil {
+			if isSysErrIO(err) {
+				return errFaultyDisk
+			}
+			if !os.IsNotExist(err) {
+				return err
+			}
+		} else {
+			if !dirInfo.IsDir() {
+				return errFileAccessDenied
+			}
 			if err = os.Remove(dstFilePath); err != nil {
 				if isSysErrNotEmpty(err) {
 					return errFileAccessDenied
@@ -2348,10 +2394,8 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum
 	}
 
 	// Remove parent dir of the source file if empty
-	if parentDir := slashpath.Dir(srcFilePath); isDirEmpty(parentDir) {
-		deleteFile(srcVolumeDir, parentDir, false)
-	}
-
+	parentDir := slashpath.Dir(srcFilePath)
+	deleteFile(srcVolumeDir, parentDir, false)
 	return nil
 }
 

go.mod

@@ -52,7 +52,7 @@ require (
 	github.com/minio/selfupdate v0.3.1
 	github.com/minio/sha256-simd v0.1.1
 	github.com/minio/simdjson-go v0.1.5
-	github.com/minio/sio v0.2.0
+	github.com/minio/sio v0.2.1
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mmcloughlin/avo v0.0.0-20200803215136-443f81d77104 // indirect
 	github.com/montanaflynn/stats v0.5.0
@@ -81,8 +81,8 @@ require (
 	github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
 	go.etcd.io/etcd/v3 v3.3.0-rc.0.0.20200707003333-58bb8ae09f8e
 	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
-	golang.org/x/net v0.0.0-20200904194848-62affa334b73
-	golang.org/x/sys v0.0.0-20200915084602-288bc346aa39
+	golang.org/x/net v0.0.0-20210119194325-5f4716e94777
+	golang.org/x/sys v0.0.0-20201119102817-f84b799fce68
 	golang.org/x/tools v0.0.0-20200929223013-bf155c11ec6f // indirect
 	google.golang.org/api v0.5.0
 	gopkg.in/jcmturner/gokrb5.v7 v7.3.0

go.sum

@@ -318,6 +318,8 @@ github.com/minio/simdjson-go v0.1.5 h1:6T5mHh7r3kUvgwhmFWQAjoPV5Yt5oD/VPjAI9ViH1
 github.com/minio/simdjson-go v0.1.5/go.mod h1:oKURrZZEBtqObgJrSjN1Ln2n9MJj2icuBTkeJzZnvSI=
 github.com/minio/sio v0.2.0 h1:NCRCFLx0r5pRbXf65LVNjxbCGZgNQvNFQkgX3XF4BoA=
 github.com/minio/sio v0.2.0/go.mod h1:nKM5GIWSrqbOZp0uhyj6M1iA0X6xQzSGtYSaTKSCut0=
+github.com/minio/sio v0.2.1 h1:NjzKiIMSMcHediVQR0AFVx2tp7Wxh9tKPfDI3kH7aHQ=
+github.com/minio/sio v0.2.1/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
@@ -536,8 +538,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
-golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -572,8 +574,9 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20u
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200915084602-288bc346aa39 h1:356XA7ITklAU2//sYkjFeco+dH1bCRD8XCJ9FIEsvo4=
-golang.org/x/sys v0.0.0-20200915084602-288bc346aa39/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU=

pkg/dsync/drwmutex.go

@@ -41,17 +41,31 @@ func log(format string, data ...interface{}) {
 	}
 }
 
-// DRWMutexAcquireTimeout - tolerance limit to wait for lock acquisition before.
-const DRWMutexAcquireTimeout = 1 * time.Second // 1 second.
+// dRWMutexAcquireTimeout - tolerance limit to wait for lock acquisition before.
+const drwMutexAcquireTimeout = 1 * time.Second // 1 second.
+
+// dRWMutexUnlockTimeout - timeout for the unlock call
+const drwMutexUnlockCallTimeout = 30 * time.Second
+
+// dRWMutexRefreshTimeout - timeout for the refresh call
+const drwMutexRefreshTimeout = 5 * time.Second
+
+// dRWMutexForceUnlockTimeout - timeout for the unlock call
+const drwMutexForceUnlockCallTimeout = 30 * time.Second
+
+// dRWMutexRefreshInterval - the interval between two refresh calls
+const drwMutexRefreshInterval = 10 * time.Second
+
 const drwMutexInfinite = 1<<63 - 1
 
 // A DRWMutex is a distributed mutual exclusion lock.
 type DRWMutex struct {
-	Names        []string
-	writeLocks   []string   // Array of nodes that granted a write lock
-	readersLocks [][]string // Array of array of nodes that granted reader locks
-	m            sync.Mutex // Mutex to prevent multiple simultaneous locks from this node
-	clnt         *Dsync
+	Names         []string
+	writeLocks    []string   // Array of nodes that granted a write lock
+	readersLocks  [][]string // Array of array of nodes that granted reader locks
+	m             sync.Mutex // Mutex to prevent multiple simultaneous locks from this node
+	clnt          *Dsync
+	cancelRefresh context.CancelFunc
 }
 
 // Granted - represents a structure of a granted lock.
@@ -85,7 +99,7 @@ func NewDRWMutex(clnt *Dsync, names ...string) *DRWMutex {
 func (dm *DRWMutex) Lock(id, source string) {
 
 	isReadLock := false
-	dm.lockBlocking(context.Background(), id, source, isReadLock, Options{
+	dm.lockBlocking(context.Background(), nil, id, source, isReadLock, Options{
 		Timeout: drwMutexInfinite,
 	})
 }
@@ -101,10 +115,10 @@ type Options struct {
 // If the lock is already in use, the calling go routine
 // blocks until either the mutex becomes available and return success or
 // more time has passed than the timeout value and return false.
-func (dm *DRWMutex) GetLock(ctx context.Context, id, source string, opts Options) (locked bool) {
+func (dm *DRWMutex) GetLock(ctx context.Context, cancel context.CancelFunc, id, source string, opts Options) (locked bool) {
 
 	isReadLock := false
-	return dm.lockBlocking(ctx, id, source, isReadLock, opts)
+	return dm.lockBlocking(ctx, cancel, id, source, isReadLock, opts)
 }
 
 // RLock holds a read lock on dm.
@@ -114,7 +128,7 @@ func (dm *DRWMutex) GetLock(ctx context.Context, id, source string, opts Options
 func (dm *DRWMutex) RLock(id, source string) {
 
 	isReadLock := true
-	dm.lockBlocking(context.Background(), id, source, isReadLock, Options{
+	dm.lockBlocking(context.Background(), nil, id, source, isReadLock, Options{
 		Timeout: drwMutexInfinite,
 	})
 }
@@ -125,10 +139,10 @@ func (dm *DRWMutex) RLock(id, source string) {
 // Otherwise the calling go routine blocks until either the mutex becomes
 // available and return success or more time has passed than the timeout
 // value and return false.
-func (dm *DRWMutex) GetRLock(ctx context.Context, id, source string, opts Options) (locked bool) {
+func (dm *DRWMutex) GetRLock(ctx context.Context, cancel context.CancelFunc, id, source string, opts Options) (locked bool) {
 
 	isReadLock := true
-	return dm.lockBlocking(ctx, id, source, isReadLock, opts)
+	return dm.lockBlocking(ctx, cancel, id, source, isReadLock, opts)
 }
 
 const (
@@ -140,8 +154,8 @@ const (
 // The function will loop using a built-in timing randomized back-off
 // algorithm until either the lock is acquired successfully or more
 // time has elapsed than the timeout value.
-func (dm *DRWMutex) lockBlocking(ctx context.Context, id, source string, isReadLock bool, opts Options) (locked bool) {
-	restClnts, owner := dm.clnt.GetLockers()
+func (dm *DRWMutex) lockBlocking(ctx context.Context, lockLossCallback func(), id, source string, isReadLock bool, opts Options) (locked bool) {
+	restClnts, _ := dm.clnt.GetLockers()
 
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 
@@ -149,8 +163,9 @@ func (dm *DRWMutex) lockBlocking(ctx context.Context, id, source string, isReadL
 	locks := make([]string, len(restClnts))
 
 	log("lockBlocking %s/%s for %#v: lockType readLock(%t), additional opts: %#v\n", id, source, dm.Names, isReadLock, opts)
-	retryCtx, cancel := context.WithTimeout(ctx, opts.Timeout)
 
+	// Add total timeout
+	ctx, cancel := context.WithTimeout(ctx, opts.Timeout)
 	defer cancel()
 
 	// Tolerance is not set, defaults to half of the locker clients.
@@ -175,19 +190,11 @@ func (dm *DRWMutex) lockBlocking(ctx context.Context, id, source string, isReadL
 
 	for {
 		select {
-		case <-retryCtx.Done():
-			log("lockBlocking canceled %s/%s for %#v: lockType readLock(%t), additional opts: %#v\n", id, source, dm.Names, isReadLock, opts)
-
-			// Caller context canceled or we timedout,
-			// return false anyways for both situations.
-
-			// make sure to unlock any successful locks, since caller has timedout or canceled the request.
-			releaseAll(dm.clnt, tolerance, owner, &locks, isReadLock, restClnts, dm.Names...)
-
+		case <-ctx.Done():
 			return false
 		default:
 			// Try to acquire the lock.
-			if locked = lock(retryCtx, dm.clnt, &locks, id, source, isReadLock, tolerance, quorum, dm.Names...); locked {
+			if locked = lock(ctx, dm.clnt, &locks, id, source, isReadLock, tolerance, quorum, dm.Names...); locked {
 				dm.m.Lock()
 
 				// If success, copy array to object
@@ -201,6 +208,11 @@ func (dm *DRWMutex) lockBlocking(ctx context.Context, id, source string, isReadL
 				}
 
 				dm.m.Unlock()
+				log("lockBlocking %s/%s for %#v: granted\n", id, source, dm.Names)
+
+				// Refresh lock continuously and cancel if there is no quorum in the lock anymore
+				dm.startContinousLockRefresh(lockLossCallback, id, source, quorum)
+
 				return locked
 			}
 
@@ -209,6 +221,161 @@ func (dm *DRWMutex) lockBlocking(ctx context.Context, id, source string, isReadL
 	}
 }
 
+func (dm *DRWMutex) startContinousLockRefresh(lockLossCallback func(), id, source string, quorum int) {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	dm.m.Lock()
+	dm.cancelRefresh = cancel
+	dm.m.Unlock()
+
+	go func() {
+		defer cancel()
+
+		refreshTimer := time.NewTimer(drwMutexRefreshInterval)
+		defer refreshTimer.Stop()
+
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-refreshTimer.C:
+				refreshTimer.Reset(drwMutexRefreshInterval)
+				refreshed, err := refresh(ctx, dm.clnt, id, source, quorum, dm.Names...)
+				if err == nil && !refreshed {
+					// Clean the lock locally and in remote nodes
+					forceUnlock(ctx, dm.clnt, id)
+					// Execute the caller lock loss callback
+					if lockLossCallback != nil {
+						lockLossCallback()
+					}
+					return
+				}
+			}
+		}
+	}()
+}
+
+func forceUnlock(ctx context.Context, ds *Dsync, id string) {
+	ctx, cancel := context.WithTimeout(ctx, drwMutexForceUnlockCallTimeout)
+	defer cancel()
+
+	restClnts, _ := ds.GetLockers()
+
+	var wg sync.WaitGroup
+	for index, c := range restClnts {
+		wg.Add(1)
+		// Send refresh request to all nodes
+		go func(index int, c NetLocker) {
+			defer wg.Done()
+			args := LockArgs{
+				UID: id,
+			}
+			c.ForceUnlock(ctx, args)
+		}(index, c)
+	}
+	wg.Wait()
+}
+
+type refreshResult struct {
+	offline   bool
+	succeeded bool
+}
+
+func refresh(ctx context.Context, ds *Dsync, id, source string, quorum int, lockNames ...string) (bool, error) {
+	restClnts, owner := ds.GetLockers()
+
+	// Create buffered channel of size equal to total number of nodes.
+	ch := make(chan refreshResult, len(restClnts))
+	var wg sync.WaitGroup
+
+	for index, c := range restClnts {
+		wg.Add(1)
+		// Send refresh request to all nodes
+		go func(index int, c NetLocker) {
+			defer wg.Done()
+
+			if c == nil {
+				ch <- refreshResult{offline: true}
+				return
+			}
+
+			args := LockArgs{
+				Owner:     owner,
+				UID:       id,
+				Resources: lockNames,
+				Source:    source,
+				Quorum:    quorum,
+			}
+
+			ctx, cancel := context.WithTimeout(ctx, drwMutexRefreshTimeout)
+			defer cancel()
+
+			refreshed, err := c.Refresh(ctx, args)
+			if refreshed && err == nil {
+				ch <- refreshResult{succeeded: true}
+			} else {
+				if err != nil {
+					ch <- refreshResult{offline: true}
+					log("dsync: Unable to call Refresh failed with %s for %#v at %s\n", err, args, c)
+				} else {
+					ch <- refreshResult{succeeded: false}
+					log("dsync: Refresh returned false for %#v at %s\n", args, c)
+				}
+			}
+
+		}(index, c)
+	}
+
+	// Wait until we have either
+	//
+	// a) received all refresh responses
+	// b) received too many refreshed for quorum to be still possible
+	// c) timed out
+	//
+	i, refreshFailed, refreshSucceeded := 0, 0, 0
+	done := false
+
+	for ; i < len(restClnts); i++ {
+		select {
+		case refresh := <-ch:
+			if refresh.offline {
+				continue
+			}
+			if refresh.succeeded {
+				refreshSucceeded++
+			} else {
+				refreshFailed++
+			}
+			if refreshFailed > quorum {
+				// We know that we are not going to succeed with refresh
+				done = true
+			}
+		case <-ctx.Done():
+			// Refreshing is canceled
+			return false, ctx.Err()
+		}
+
+		if done {
+			break
+		}
+	}
+
+	refreshQuorum := refreshSucceeded >= quorum
+	if !refreshQuorum {
+		refreshQuorum = refreshFailed < quorum
+	}
+
+	// We may have some unused results in ch, release them async.
+	go func() {
+		wg.Wait()
+		close(ch)
+		for range ch {
+		}
+	}()
+
+	return refreshQuorum, nil
+}
+
 // lock tries to acquire the distributed lock, returning true or false.
 func lock(ctx context.Context, ds *Dsync, locks *[]string, id, source string, isReadLock bool, tolerance, quorum int, lockNames ...string) bool {
 	for i := range *locks {
@@ -219,11 +386,12 @@ func lock(ctx context.Context, ds *Dsync, locks *[]string, id, source string, is
 
 	// Create buffered channel of size equal to total number of nodes.
 	ch := make(chan Granted, len(restClnts))
-	defer close(ch)
-
 	var wg sync.WaitGroup
-	for index, c := range restClnts {
 
+	// Combined timeout for the lock attempt.
+	ctx, cancel := context.WithTimeout(ctx, drwMutexAcquireTimeout)
+	defer cancel()
+	for index, c := range restClnts {
 		wg.Add(1)
 		// broadcast lock request to all nodes
 		go func(index int, isReadLock bool, c NetLocker) {
@@ -231,7 +399,7 @@ func lock(ctx context.Context, ds *Dsync, locks *[]string, id, source string, is
 
 			g := Granted{index: index}
 			if c == nil {
-				log("dsync: nil locker")
+				log("dsync: nil locker\n")
 				ch <- g
 				return
 			}
@@ -247,93 +415,80 @@ func lock(ctx context.Context, ds *Dsync, locks *[]string, id, source string, is
 			var locked bool
 			var err error
 			if isReadLock {
-				if locked, err = c.RLock(ctx, args); err != nil {
+				if locked, err = c.RLock(context.Background(), args); err != nil {
 					log("dsync: Unable to call RLock failed with %s for %#v at %s\n", err, args, c)
 				}
 			} else {
-				if locked, err = c.Lock(ctx, args); err != nil {
+				if locked, err = c.Lock(context.Background(), args); err != nil {
 					log("dsync: Unable to call Lock failed with %s for %#v at %s\n", err, args, c)
 				}
 			}
-
 			if locked {
 				g.lockUID = args.UID
 			}
-
 			ch <- g
 
 		}(index, isReadLock, c)
 	}
 
-	quorumLocked := false
+	// Wait until we have either
+	//
+	// a) received all lock responses
+	// b) received too many 'non-'locks for quorum to be still possible
+	// c) timed out
+	//
+	i, locksFailed := 0, 0
+	done := false
 
-	wg.Add(1)
-	go func(isReadLock bool) {
-		defer wg.Done()
-
-		// Wait until we have either
-		//
-		// a) received all lock responses
-		// b) received too many 'non-'locks for quorum to be still possible
-		// c) timedout
-		//
-		i, locksFailed := 0, 0
-		done := false
-		timeout := time.After(DRWMutexAcquireTimeout)
-
-		for ; i < len(restClnts); i++ { // Loop until we acquired all locks
-
-			select {
-			case grant := <-ch:
-				if grant.isLocked() {
-					// Mark that this node has acquired the lock
-					(*locks)[grant.index] = grant.lockUID
-				} else {
-					locksFailed++
-					if locksFailed > tolerance {
-						// We know that we are not going to get the lock anymore,
-						// so exit out and release any locks that did get acquired
-						done = true
-						// Increment the number of grants received from the buffered channel.
-						i++
-						releaseAll(ds, tolerance, owner, locks, isReadLock, restClnts, lockNames...)
-					}
-				}
-			case <-timeout:
-				done = true
-				// timeout happened, maybe one of the nodes is slow, count
-				// number of locks to check whether we have quorum or not
-				if !checkQuorumLocked(locks, quorum) {
-					log("Quorum not met after timeout\n")
-					releaseAll(ds, tolerance, owner, locks, isReadLock, restClnts, lockNames...)
-				} else {
-					log("Quorum met after timeout\n")
+	for ; i < len(restClnts); i++ { // Loop until we acquired all locks
+		select {
+		case grant := <-ch:
+			if grant.isLocked() {
+				// Mark that this node has acquired the lock
+				(*locks)[grant.index] = grant.lockUID
+			} else {
+				locksFailed++
+				if locksFailed > tolerance {
+					// We know that we are not going to get the lock anymore,
+					// so exit out and release any locks that did get acquired
+					done = true
 				}
 			}
-
-			if done {
-				break
+		case <-ctx.Done():
+			// Capture timedout locks as failed or took too long
+			locksFailed++
+			if locksFailed > tolerance {
+				// We know that we are not going to get the lock anymore,
+				// so exit out and release any locks that did get acquired
+				done = true
 			}
 		}
 
-		// Count locks in order to determine whether we have quorum or not
-		quorumLocked = checkQuorumLocked(locks, quorum)
+		if done {
+			break
+		}
+	}
 
-		// Wait for the other responses and immediately release the locks
-		// (do not add them to the locks array because the DRWMutex could
-		//  already has been unlocked again by the original calling thread)
-		for ; i < len(restClnts); i++ {
-			grantToBeReleased := <-ch
+	quorumLocked := checkQuorumLocked(locks, quorum) && locksFailed <= tolerance
+	if !quorumLocked {
+		log("Releasing all acquired locks now abandoned after quorum was not met\n")
+		releaseAll(ds, tolerance, owner, locks, isReadLock, restClnts, lockNames...)
+	}
+
+	// We may have some unused results in ch, release them async.
+	go func() {
+		wg.Wait()
+		close(ch)
+		for grantToBeReleased := range ch {
 			if grantToBeReleased.isLocked() {
 				// release lock
+				log("Releasing abandoned lock\n")
 				sendRelease(ds, restClnts[grantToBeReleased.index],
 					owner,
 					grantToBeReleased.lockUID, isReadLock, lockNames...)
 			}
 		}
-	}(isReadLock)
-
-	wg.Wait()
+	}()
 
 	return quorumLocked
 }
@@ -404,6 +559,9 @@ func releaseAll(ds *Dsync, tolerance int, owner string, locks *[]string, isReadL
 //
 // It is a run-time error if dm is not locked on entry to Unlock.
 func (dm *DRWMutex) Unlock() {
+	dm.m.Lock()
+	dm.cancelRefresh()
+	dm.m.Unlock()
 
 	restClnts, owner := dm.clnt.GetLockers()
 	// create temp array on stack
@@ -443,6 +601,9 @@ func (dm *DRWMutex) Unlock() {
 //
 // It is a run-time error if dm is not locked on entry to RUnlock.
 func (dm *DRWMutex) RUnlock() {
+	dm.m.Lock()
+	dm.cancelRefresh()
+	dm.m.Unlock()
 
 	// create temp array on stack
 	restClnts, owner := dm.clnt.GetLockers()
@@ -483,13 +644,16 @@ func sendRelease(ds *Dsync, c NetLocker, owner string, uid string, isReadLock bo
 		Resources: names,
 	}
 
+	ctx, cancel := context.WithTimeout(context.Background(), drwMutexUnlockCallTimeout)
+	defer cancel()
+
 	if isReadLock {
-		if _, err := c.RUnlock(args); err != nil {
+		if _, err := c.RUnlock(ctx, args); err != nil {
 			log("dsync: Unable to call RUnlock failed with %s for %#v at %s\n", err, args, c)
 			return false
 		}
 	} else {
-		if _, err := c.Unlock(args); err != nil {
+		if _, err := c.Unlock(ctx, args); err != nil {
 			log("dsync: Unable to call Unlock failed with %s for %#v at %s\n", err, args, c)
 			return false
 		}

pkg/dsync/drwmutex_test.go

@@ -36,12 +36,14 @@ func testSimpleWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 
 	drwm := NewDRWMutex(ds, "simplelock")
 
-	if !drwm.GetRLock(context.Background(), id, source, Options{Timeout: time.Second}) {
+	ctx1, cancel1 := context.WithCancel(context.Background())
+	if !drwm.GetRLock(ctx1, cancel1, id, source, Options{Timeout: time.Second}) {
 		panic("Failed to acquire read lock")
 	}
 	// fmt.Println("1st read lock acquired, waiting...")
 
-	if !drwm.GetRLock(context.Background(), id, source, Options{Timeout: time.Second}) {
+	ctx2, cancel2 := context.WithCancel(context.Background())
+	if !drwm.GetRLock(ctx2, cancel2, id, source, Options{Timeout: time.Second}) {
 		panic("Failed to acquire read lock")
 	}
 	// fmt.Println("2nd read lock acquired, waiting...")
@@ -59,7 +61,8 @@ func testSimpleWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 	}()
 
 	// fmt.Println("Trying to acquire write lock, waiting...")
-	locked = drwm.GetLock(context.Background(), id, source, Options{Timeout: duration})
+	ctx3, cancel3 := context.WithCancel(context.Background())
+	locked = drwm.GetLock(ctx3, cancel3, id, source, Options{Timeout: duration})
 	if locked {
 		// fmt.Println("Write lock acquired, waiting...")
 		time.Sleep(time.Second)
@@ -93,7 +96,8 @@ func testDualWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 	drwm := NewDRWMutex(ds, "duallock")
 
 	// fmt.Println("Getting initial write lock")
-	if !drwm.GetLock(context.Background(), id, source, Options{Timeout: time.Second}) {
+	ctx1, cancel1 := context.WithCancel(context.Background())
+	if !drwm.GetLock(ctx1, cancel1, id, source, Options{Timeout: time.Second}) {
 		panic("Failed to acquire initial write lock")
 	}
 
@@ -104,7 +108,8 @@ func testDualWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 	}()
 
 	// fmt.Println("Trying to acquire 2nd write lock, waiting...")
-	locked = drwm.GetLock(context.Background(), id, source, Options{Timeout: duration})
+	ctx2, cancel2 := context.WithCancel(context.Background())
+	locked = drwm.GetLock(ctx2, cancel2, id, source, Options{Timeout: duration})
 	if locked {
 		// fmt.Println("2nd write lock acquired, waiting...")
 		time.Sleep(time.Second)
@@ -139,7 +144,7 @@ func TestDualWriteLockTimedOut(t *testing.T) {
 
 // Borrowed from rwmutex_test.go
 func parallelReader(ctx context.Context, m *DRWMutex, clocked, cunlock, cdone chan bool) {
-	if m.GetRLock(ctx, id, source, Options{Timeout: time.Second}) {
+	if m.GetRLock(ctx, nil, id, source, Options{Timeout: time.Second}) {
 		clocked <- true
 		<-cunlock
 		m.RUnlock()
@@ -182,7 +187,7 @@ func TestParallelReaders(t *testing.T) {
 // Borrowed from rwmutex_test.go
 func reader(rwm *DRWMutex, numIterations int, activity *int32, cdone chan bool) {
 	for i := 0; i < numIterations; i++ {
-		if rwm.GetRLock(context.Background(), id, source, Options{Timeout: time.Second}) {
+		if rwm.GetRLock(context.Background(), nil, id, source, Options{Timeout: time.Second}) {
 			n := atomic.AddInt32(activity, 1)
 			if n < 1 || n >= 10000 {
 				panic(fmt.Sprintf("wlock(%d)\n", n))
@@ -199,7 +204,7 @@ func reader(rwm *DRWMutex, numIterations int, activity *int32, cdone chan bool)
 // Borrowed from rwmutex_test.go
 func writer(rwm *DRWMutex, numIterations int, activity *int32, cdone chan bool) {
 	for i := 0; i < numIterations; i++ {
-		if rwm.GetLock(context.Background(), id, source, Options{Timeout: time.Second}) {
+		if rwm.GetLock(context.Background(), nil, id, source, Options{Timeout: time.Second}) {
 			n := atomic.AddInt32(activity, 10000)
 			if n != 10000 {
 				panic(fmt.Sprintf("wlock(%d)\n", n))

pkg/dsync/dsync-server_test.go

@@ -30,6 +30,15 @@ type lockServer struct {
 	// Map of locks, with negative value indicating (exclusive) write lock
 	// and positive values indicating number of read locks
 	lockMap map[string]int64
+
+	// Refresh returns lock not found if set to true
+	lockNotFound bool
+}
+
+func (l *lockServer) setRefreshReply(refreshed bool) {
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+	l.lockNotFound = !refreshed
 }
 
 func (l *lockServer) Lock(args *LockArgs, reply *bool) error {
@@ -91,6 +100,13 @@ func (l *lockServer) RUnlock(args *LockArgs, reply *bool) error {
 	return nil
 }
 
+func (l *lockServer) Refresh(args *LockArgs, reply *bool) error {
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+	*reply = !l.lockNotFound
+	return nil
+}
+
 func (l *lockServer) ForceUnlock(args *LockArgs, reply *bool) error {
 	l.mutex.Lock()
 	defer l.mutex.Unlock()

pkg/dsync/dsync_test.go

@@ -19,6 +19,7 @@
 package dsync_test
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"math/rand"
@@ -32,19 +33,26 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/minio/minio/pkg/dsync"
 	. "github.com/minio/minio/pkg/dsync"
 )
 
+const numberOfNodes = 5
+
 var ds *Dsync
 var rpcPaths []string // list of rpc paths where lock server is serving.
 
-func startRPCServers(nodes []string) {
+var nodes = make([]string, numberOfNodes) // list of node IP addrs or hostname with ports.
+var lockServers []*lockServer
+
+func startRPCServers() {
 	for i := range nodes {
 		server := rpc.NewServer()
-		server.RegisterName("Dsync", &lockServer{
+		ls := &lockServer{
 			mutex:   sync.Mutex{},
 			lockMap: make(map[string]int64),
-		})
+		}
+		server.RegisterName("Dsync", ls)
 		// For some reason the registration paths need to be different (even for different server objs)
 		server.HandleHTTP(rpcPaths[i], fmt.Sprintf("%s-debug", rpcPaths[i]))
 		l, e := net.Listen("tcp", ":"+strconv.Itoa(i+12345))
@@ -52,6 +60,8 @@ func startRPCServers(nodes []string) {
 			log.Fatal("listen error:", e)
 		}
 		go http.Serve(l, nil)
+
+		lockServers = append(lockServers, ls)
 	}
 
 	// Let servers start
@@ -64,7 +74,6 @@ func TestMain(m *testing.M) {
 
 	rand.Seed(time.Now().UTC().UnixNano())
 
-	nodes := make([]string, 5) // list of node IP addrs or hostname with ports.
 	for i := range nodes {
 		nodes[i] = fmt.Sprintf("127.0.0.1:%d", i+12345)
 	}
@@ -82,7 +91,7 @@ func TestMain(m *testing.M) {
 		GetLockers: func() ([]NetLocker, string) { return clnts, uuid.New().String() },
 	}
 
-	startRPCServers(nodes)
+	startRPCServers()
 
 	os.Exit(m.Run())
 }
@@ -231,6 +240,42 @@ func TestTwoSimultaneousLocksForDifferentResources(t *testing.T) {
 	time.Sleep(10 * time.Millisecond)
 }
 
+// Test refreshing lock
+func TestFailedRefreshLock(t *testing.T) {
+	// Simulate Refresh RPC response to return no locking found
+	for i := range lockServers {
+		lockServers[i].setRefreshReply(false)
+	}
+
+	dm := NewDRWMutex(ds, "aap")
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+
+	ctx, cl := context.WithCancel(context.Background())
+	cancel := func() {
+		cl()
+		wg.Done()
+	}
+
+	if !dm.GetLock(ctx, cancel, id, source, dsync.Options{Timeout: 5 * time.Minute}) {
+		t.Fatal("GetLock() should be successful")
+	}
+
+	// Wait until context is canceled
+	wg.Wait()
+	if ctx.Err() == nil {
+		t.Fatal("Unexpected error", ctx.Err())
+	}
+
+	// Should be safe operation in all cases
+	dm.Unlock()
+
+	// Revert Refresh RPC response to locking found
+	for i := range lockServers {
+		lockServers[i].setRefreshReply(false)
+	}
+}
+
 // Borrowed from mutex_test.go
 func HammerMutex(m *DRWMutex, loops int, cdone chan bool) {
 	for i := 0; i < loops; i++ {

pkg/dsync/rpc-client-impl_test.go

@@ -114,9 +114,14 @@ func (rpcClient *ReconnectRPCClient) Unlock(args LockArgs) (status bool, err err
 	return status, err
 }
 
-func (rpcClient *ReconnectRPCClient) Expired(ctx context.Context, args LockArgs) (expired bool, err error) {
-	err = rpcClient.Call("Dsync.Expired", &args, &expired)
-	return expired, err
+func (rpcClient *ReconnectRPCClient) Refresh(ctx context.Context, args LockArgs) (refreshed bool, err error) {
+	err = rpcClient.Call("Dsync.Refresh", &args, &refreshed)
+	return refreshed, err
+}
+
+func (rpcClient *ReconnectRPCClient) ForceUnlock(ctx context.Context, args LockArgs) (status bool, err error) {
+	err = rpcClient.Call("Dsync.ForceUnlock", &args, &status)
+	return status, err
 }
 
 func (rpcClient *ReconnectRPCClient) String() string {

pkg/dsync/rpc-client-interface.go

@@ -53,15 +53,18 @@ type NetLocker interface {
 	// Do read unlock for given LockArgs. It should return
 	// * a boolean to indicate success/failure of the operation
 	// * an error on failure of unlock request operation.
-	RUnlock(args LockArgs) (bool, error)
+	RUnlock(ctx context.Context, args LockArgs) (bool, error)
 
 	// Do write unlock for given LockArgs. It should return
 	// * a boolean to indicate success/failure of the operation
 	// * an error on failure of unlock request operation.
-	Unlock(args LockArgs) (bool, error)
+	Unlock(ctx context.Context, args LockArgs) (bool, error)
 
-	// Expired returns if current lock args has expired.
-	Expired(ctx context.Context, args LockArgs) (bool, error)
+	// Force unlock a resource
+	ForceUnlock(ctx context.Context, args LockArgs) (bool, error)
+
+	// Refresh the given lock to prevent it from becoming stale
+	Refresh(ctx context.Context, args LockArgs) (bool, error)
 
 	// Returns underlying endpoint of this lock client instance.
 	String() string

pkg/event/target/elasticsearch.go

@@ -124,7 +124,7 @@ func (target *ElasticsearchTarget) IsActive() (bool, error) {
 	}
 	_, code, err := target.client.Ping(target.args.URL.String()).HttpHeadOnly(true).Do(ctx)
 	if err != nil {
-		if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) {
+		if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) {
 			return false, errNotConnected
 		}
 		return false, err
@@ -138,7 +138,7 @@ func (target *ElasticsearchTarget) Save(eventData event.Event) error {
 		return target.store.Put(eventData)
 	}
 	err := target.send(eventData)
-	if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) {
+	if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) {
 		return errNotConnected
 	}
 	return err
@@ -214,7 +214,7 @@ func (target *ElasticsearchTarget) Send(eventKey string) error {
 	}
 
 	if err := target.send(eventData); err != nil {
-		if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) {
+		if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) {
 			return errNotConnected
 		}
 		return err
@@ -267,7 +267,7 @@ func newClient(args ElasticsearchArgs) (*elastic.Client, error) {
 	client, err := elastic.NewClient(options...)
 	if err != nil {
 		// https://github.com/olivere/elastic/wiki/Connection-Errors
-		if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) {
+		if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) {
 			return nil, errNotConnected
 		}
 		return nil, err

pkg/event/target/webhook.go

@@ -111,7 +111,7 @@ func (target *WebhookTarget) IsActive() (bool, error) {
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodHead, target.args.Endpoint.String(), nil)
 	if err != nil {
-		if xnet.IsNetworkOrHostDown(err) {
+		if xnet.IsNetworkOrHostDown(err, false) {
 			return false, errNotConnected
 		}
 		return false, err
@@ -119,7 +119,7 @@ func (target *WebhookTarget) IsActive() (bool, error) {
 
 	resp, err := target.httpClient.Do(req)
 	if err != nil {
-		if xnet.IsNetworkOrHostDown(err) || errors.Is(err, context.DeadlineExceeded) {
+		if xnet.IsNetworkOrHostDown(err, false) || errors.Is(err, context.DeadlineExceeded) {
 			return false, errNotConnected
 		}
 		return false, err
@@ -137,7 +137,7 @@ func (target *WebhookTarget) Save(eventData event.Event) error {
 	}
 	err := target.send(eventData)
 	if err != nil {
-		if xnet.IsNetworkOrHostDown(err) {
+		if xnet.IsNetworkOrHostDown(err, false) {
 			return errNotConnected
 		}
 	}
@@ -197,7 +197,7 @@ func (target *WebhookTarget) Send(eventKey string) error {
 	}
 
 	if err := target.send(eventData); err != nil {
-		if xnet.IsNetworkOrHostDown(err) {
+		if xnet.IsNetworkOrHostDown(err, false) {
 			return errNotConnected
 		}
 		return err