From a4f6705874bbfb2f4fef90ae5c3c6e93ab6afd18 Mon Sep 17 00:00:00 2001
From: Harshavardhana <harsha@minio.io>
Date: Thu, 7 Jan 2021 19:16:18 -0800
Subject: [PATCH] expire stale locks when owner is down (#11247)

fixes #11246
---
 cmd/admin-server-info.go |  2 +-
 cmd/globals.go           |  2 ++
 cmd/local-locker.go      |  5 +++++
 cmd/prepare-storage.go   |  6 +++---
 cmd/server-main.go       | 10 ++++++++++
 5 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/cmd/admin-server-info.go b/cmd/admin-server-info.go
index aef2b178b..a2f20f831 100644
--- a/cmd/admin-server-info.go
+++ b/cmd/admin-server-info.go
@@ -45,7 +45,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 			}
 			_, present := network[nodeName]
 			if !present {
-				if err := IsServerResolvable(endpoint); err == nil {
+				if err := isServerResolvable(endpoint); err == nil {
 					network[nodeName] = "online"
 				} else {
 					network[nodeName] = "offline"
diff --git a/cmd/globals.go b/cmd/globals.go
index 961a0bb3c..16a1ba043 100644
--- a/cmd/globals.go
+++ b/cmd/globals.go
@@ -192,6 +192,8 @@ var (
 
 	globalEndpoints EndpointServerPools
 
+	globalRemoteEndpoints map[string]Endpoint
+
 	// Global server's network statistics
 	globalConnStats = newConnStats()
 
diff --git a/cmd/local-locker.go b/cmd/local-locker.go
index 8bdc51c5e..ff70cd66a 100644
--- a/cmd/local-locker.go
+++ b/cmd/local-locker.go
@@ -228,6 +228,11 @@ func (l *localLocker) Expired(ctx context.Context, args dsync.LockArgs) (expired
 				// Check whether uid is still active
 				for _, entry := range lri {
 					if entry.UID == args.UID && entry.Owner == args.Owner {
+						if ep, ok := globalRemoteEndpoints[args.Owner]; ok {
+							if err = isServerResolvable(ep); err != nil {
+								return true, nil
+							}
+						}
 						return false, nil
 					}
 				}
diff --git a/cmd/prepare-storage.go b/cmd/prepare-storage.go
index f80c1da5b..532da45ee 100644
--- a/cmd/prepare-storage.go
+++ b/cmd/prepare-storage.go
@@ -163,9 +163,9 @@ func formatErasureCleanupTmpLocalEndpoints(endpoints Endpoints) error {
 // https://github.com/minio/minio/issues/5667
 var errErasureV3ThisEmpty = fmt.Errorf("Erasure format version 3 has This field empty")
 
-// IsServerResolvable - checks if the endpoint is resolvable
+// isServerResolvable - checks if the endpoint is resolvable
 // by sending a naked HTTP request with liveness checks.
-func IsServerResolvable(endpoint Endpoint) error {
+func isServerResolvable(endpoint Endpoint) error {
 	serverURL := &url.URL{
 		Scheme: endpoint.Scheme,
 		Host:   endpoint.Host,
@@ -244,7 +244,7 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 				return nil, nil, fmt.Errorf("Disk %s: %w", endpoints[i], err)
 			}
 			if retryCount >= 5 {
-				logger.Info("Unable to connect to %s: %v\n", endpoints[i], IsServerResolvable(endpoints[i]))
+				logger.Info("Unable to connect to %s: %v\n", endpoints[i], isServerResolvable(endpoints[i]))
 			}
 		}
 	}
diff --git a/cmd/server-main.go b/cmd/server-main.go
index eedf18b45..22c1715c6 100644
--- a/cmd/server-main.go
+++ b/cmd/server-main.go
@@ -139,6 +139,16 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 
 	globalMinioHost, globalMinioPort = mustSplitHostPort(globalMinioAddr)
 	globalEndpoints, setupType, err = createServerEndpoints(globalCLIContext.Addr, serverCmdArgs(ctx)...)
+
+	globalRemoteEndpoints = make(map[string]Endpoint)
+	for _, z := range globalEndpoints {
+		for _, ep := range z.Endpoints {
+			if ep.IsLocal {
+				continue
+			}
+			globalRemoteEndpoints[ep.Host] = ep
+		}
+	}
 	logger.FatalIf(err, "Invalid command line arguments")
 
 	// allow transport to be HTTP/1.1 for proxying.