2016-04-29 10:29:09 +02:00
|
|
|
/*
|
|
|
|
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2016-08-19 01:23:42 +02:00
|
|
|
package cmd
|
2016-04-29 10:29:09 +02:00
|
|
|
|
2016-04-29 20:39:20 +02:00
|
|
|
import (
|
2016-05-16 23:31:28 +02:00
|
|
|
"errors"
|
2016-08-24 04:19:24 +02:00
|
|
|
pathutil "path"
|
2016-08-31 20:39:08 +02:00
|
|
|
"runtime"
|
2016-08-10 22:08:11 +02:00
|
|
|
"strconv"
|
2016-04-29 20:39:20 +02:00
|
|
|
"sync"
|
2016-08-10 22:08:11 +02:00
|
|
|
|
|
|
|
"github.com/minio/dsync"
|
2016-04-29 20:39:20 +02:00
|
|
|
)
|
2016-04-29 10:29:09 +02:00
|
|
|
|
2016-08-10 22:08:11 +02:00
|
|
|
// Global name space lock.
|
|
|
|
var nsMutex *nsLockMap
|
|
|
|
|
|
|
|
// Initialize distributed locking only in case of distributed setup.
|
|
|
|
// Returns if the setup is distributed or not on success.
|
2016-10-18 21:49:24 +02:00
|
|
|
func initDsyncNodes(eps []storageEndPoint) error {
|
2016-08-24 04:19:24 +02:00
|
|
|
cred := serverConfig.GetCredential()
|
|
|
|
// Initialize rpc lock client information only if this instance is a distributed setup.
|
|
|
|
var clnts []dsync.RPC
|
2016-09-16 09:30:55 +02:00
|
|
|
myNode := -1
|
2016-10-18 21:49:24 +02:00
|
|
|
for _, ep := range eps {
|
|
|
|
if ep.host == "" || ep.port == 0 || ep.path == "" {
|
|
|
|
return errInvalidArgument
|
|
|
|
}
|
|
|
|
clnts = append(clnts, newAuthClient(&authConfig{
|
|
|
|
accessKey: cred.AccessKeyID,
|
|
|
|
secretKey: cred.SecretAccessKey,
|
|
|
|
// Construct a new dsync server addr.
|
|
|
|
secureConn: isSSL(),
|
|
|
|
address: ep.host + ":" + strconv.Itoa(ep.port),
|
|
|
|
// Construct a new rpc path for the endpoint.
|
|
|
|
path: pathutil.Join(lockRPCPath, ep.path),
|
|
|
|
loginMethod: "Dsync.LoginHandler",
|
|
|
|
}))
|
|
|
|
if isLocalStorage(ep) && myNode == -1 {
|
|
|
|
myNode = len(clnts) - 1
|
2016-08-10 22:08:11 +02:00
|
|
|
}
|
|
|
|
}
|
2016-09-16 09:30:55 +02:00
|
|
|
return dsync.SetNodesWithClients(clnts, myNode)
|
2016-08-10 22:08:11 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// initNSLock - initialize name space lock map.
|
|
|
|
func initNSLock(isDist bool) {
|
|
|
|
nsMutex = &nsLockMap{
|
|
|
|
isDist: isDist,
|
|
|
|
lockMap: make(map[nsParam]*nsLock),
|
|
|
|
}
|
2016-09-27 23:35:43 +02:00
|
|
|
|
2016-10-11 09:50:27 +02:00
|
|
|
// Initialize nsLockMap with entry for instrumentation information.
|
2016-09-27 23:35:43 +02:00
|
|
|
// Entries of <volume,path> -> stateInfo of locks
|
|
|
|
nsMutex.debugLockMap = make(map[nsParam]*debugLockInfoPerVolumePath)
|
2016-08-31 20:39:08 +02:00
|
|
|
}
|
|
|
|
|
2016-08-10 22:08:11 +02:00
|
|
|
// RWLocker - interface that any read-write locking library should implement.
|
|
|
|
type RWLocker interface {
|
|
|
|
sync.Locker
|
|
|
|
RLock()
|
|
|
|
RUnlock()
|
|
|
|
}
|
|
|
|
|
2016-04-29 10:29:09 +02:00
|
|
|
// nsParam - carries name space resource.
|
|
|
|
type nsParam struct {
|
|
|
|
volume string
|
|
|
|
path string
|
|
|
|
}
|
|
|
|
|
|
|
|
// nsLock - provides primitives for locking critical namespace regions.
|
|
|
|
type nsLock struct {
|
2016-09-16 09:30:55 +02:00
|
|
|
RWLocker
|
|
|
|
ref uint
|
2016-04-29 10:29:09 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// nsLockMap - namespace lock map, provides primitives to Lock,
|
|
|
|
// Unlock, RLock and RUnlock.
|
|
|
|
type nsLockMap struct {
|
2016-10-11 09:50:27 +02:00
|
|
|
// Lock counter used for lock debugging.
|
2016-09-19 22:14:55 +02:00
|
|
|
globalLockCounter int64 // Total locks held.
|
|
|
|
blockedCounter int64 // Total operations blocked waiting for locks.
|
|
|
|
runningLockCounter int64 // Total locks held but not released yet.
|
|
|
|
debugLockMap map[nsParam]*debugLockInfoPerVolumePath // Info for instrumentation on locks.
|
|
|
|
|
|
|
|
// Indicates whether the locking service is part
|
|
|
|
// of a distributed setup or not.
|
|
|
|
isDist bool
|
2016-08-10 22:08:11 +02:00
|
|
|
lockMap map[nsParam]*nsLock
|
|
|
|
lockMapMutex sync.Mutex
|
2016-04-29 10:29:09 +02:00
|
|
|
}
|
|
|
|
|
2016-04-29 20:39:20 +02:00
|
|
|
// Lock the namespace resource.
|
2016-08-31 20:39:08 +02:00
|
|
|
func (n *nsLockMap) lock(volume, path string, lockOrigin, opsID string, readLock bool) {
|
2016-08-10 22:08:11 +02:00
|
|
|
var nsLk *nsLock
|
|
|
|
n.lockMapMutex.Lock()
|
2016-04-29 10:29:09 +02:00
|
|
|
|
|
|
|
param := nsParam{volume, path}
|
|
|
|
nsLk, found := n.lockMap[param]
|
|
|
|
if !found {
|
|
|
|
nsLk = &nsLock{
|
2016-09-16 09:30:55 +02:00
|
|
|
RWLocker: func() RWLocker {
|
2016-08-10 22:08:11 +02:00
|
|
|
if n.isDist {
|
2016-08-24 04:19:24 +02:00
|
|
|
return dsync.NewDRWMutex(pathutil.Join(volume, path))
|
2016-08-10 22:08:11 +02:00
|
|
|
}
|
|
|
|
return &sync.RWMutex{}
|
|
|
|
}(),
|
|
|
|
ref: 0,
|
2016-04-29 10:29:09 +02:00
|
|
|
}
|
2016-04-29 20:39:20 +02:00
|
|
|
n.lockMap[param] = nsLk
|
2016-04-29 10:29:09 +02:00
|
|
|
}
|
2016-06-02 01:43:31 +02:00
|
|
|
nsLk.ref++ // Update ref count here to avoid multiple races.
|
2016-08-31 20:39:08 +02:00
|
|
|
|
2016-09-27 23:35:43 +02:00
|
|
|
// Change the state of the lock to be blocked for the given
|
|
|
|
// pair of <volume, path> and <OperationID> till the lock
|
|
|
|
// unblocks. The lock for accessing `nsMutex` is held inside
|
|
|
|
// the function itself.
|
|
|
|
if err := n.statusNoneToBlocked(param, lockOrigin, opsID, readLock); err != nil {
|
|
|
|
errorIf(err, "Failed to set lock state to blocked.")
|
2016-08-31 20:39:08 +02:00
|
|
|
}
|
2016-09-19 22:14:55 +02:00
|
|
|
|
2016-04-29 20:39:20 +02:00
|
|
|
// Unlock map before Locking NS which might block.
|
2016-08-10 22:08:11 +02:00
|
|
|
n.lockMapMutex.Unlock()
|
2016-04-29 20:39:20 +02:00
|
|
|
|
|
|
|
// Locking here can block.
|
|
|
|
if readLock {
|
2016-09-16 09:30:55 +02:00
|
|
|
nsLk.RLock()
|
2016-04-29 20:39:20 +02:00
|
|
|
} else {
|
2016-09-16 09:30:55 +02:00
|
|
|
nsLk.Lock()
|
2016-04-29 20:39:20 +02:00
|
|
|
}
|
2016-08-31 20:39:08 +02:00
|
|
|
|
2016-09-27 23:35:43 +02:00
|
|
|
// Changing the status of the operation from blocked to
|
|
|
|
// running. change the state of the lock to be running (from
|
2016-10-11 09:50:27 +02:00
|
|
|
// blocked) for the given pair of <volume, path> and <OperationID>.
|
2016-09-27 23:35:43 +02:00
|
|
|
if err := n.statusBlockedToRunning(param, lockOrigin, opsID, readLock); err != nil {
|
|
|
|
errorIf(err, "Failed to set the lock state to running.")
|
2016-08-31 20:39:08 +02:00
|
|
|
}
|
2016-04-29 10:29:09 +02:00
|
|
|
}
|
|
|
|
|
2016-04-29 20:39:20 +02:00
|
|
|
// Unlock the namespace resource.
|
2016-08-31 20:39:08 +02:00
|
|
|
func (n *nsLockMap) unlock(volume, path, opsID string, readLock bool) {
|
2016-09-27 23:35:43 +02:00
|
|
|
// nsLk.Unlock() will not block, hence locking the map for the
|
|
|
|
// entire function is fine.
|
2016-08-10 22:08:11 +02:00
|
|
|
n.lockMapMutex.Lock()
|
|
|
|
defer n.lockMapMutex.Unlock()
|
2016-04-29 10:29:09 +02:00
|
|
|
|
|
|
|
param := nsParam{volume, path}
|
|
|
|
if nsLk, found := n.lockMap[param]; found {
|
2016-04-29 20:39:20 +02:00
|
|
|
if readLock {
|
2016-09-16 09:30:55 +02:00
|
|
|
nsLk.RUnlock()
|
2016-04-29 20:39:20 +02:00
|
|
|
} else {
|
2016-09-16 09:30:55 +02:00
|
|
|
nsLk.Unlock()
|
2016-04-29 20:39:20 +02:00
|
|
|
}
|
|
|
|
if nsLk.ref == 0 {
|
2016-09-27 23:35:43 +02:00
|
|
|
errorIf(errors.New("Namespace reference count cannot be 0."),
|
|
|
|
"Invalid reference count detected.")
|
2016-04-29 20:39:20 +02:00
|
|
|
}
|
2016-04-29 10:29:09 +02:00
|
|
|
if nsLk.ref != 0 {
|
|
|
|
nsLk.ref--
|
2016-09-27 23:35:43 +02:00
|
|
|
|
|
|
|
// delete the lock state entry for given operation ID.
|
|
|
|
err := n.deleteLockInfoEntryForOps(param, opsID)
|
|
|
|
if err != nil {
|
|
|
|
errorIf(err, "Failed to delete lock info entry.")
|
2016-08-31 20:39:08 +02:00
|
|
|
}
|
2016-04-29 10:29:09 +02:00
|
|
|
}
|
2016-04-29 20:39:20 +02:00
|
|
|
if nsLk.ref == 0 {
|
|
|
|
// Remove from the map if there are no more references.
|
|
|
|
delete(n.lockMap, param)
|
2016-08-31 20:39:08 +02:00
|
|
|
|
2016-09-27 23:35:43 +02:00
|
|
|
// delete the lock state entry for given
|
|
|
|
// <volume, path> pair.
|
|
|
|
err := n.deleteLockInfoEntryForVolumePath(param)
|
|
|
|
if err != nil {
|
|
|
|
errorIf(err, "Failed to delete lock info entry.")
|
2016-08-31 20:39:08 +02:00
|
|
|
}
|
2016-04-29 10:29:09 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-04-29 20:39:20 +02:00
|
|
|
// Lock - locks the given resource for writes, using a previously
|
|
|
|
// allocated name space lock or initializing a new one.
|
2016-08-31 20:39:08 +02:00
|
|
|
func (n *nsLockMap) Lock(volume, path, opsID string) {
|
2016-10-11 09:50:27 +02:00
|
|
|
readLock := false // This is a write lock.
|
2016-09-27 23:35:43 +02:00
|
|
|
|
|
|
|
// The caller information of the lock held has been obtained
|
|
|
|
// here before calling any other function.
|
|
|
|
|
|
|
|
// Fetching the package, function name and the line number of
|
2016-10-11 09:50:27 +02:00
|
|
|
// the caller from the runtime.
|
|
|
|
pc, file, line, success := runtime.Caller(1)
|
2016-09-27 23:35:43 +02:00
|
|
|
if !success {
|
2016-10-11 09:50:27 +02:00
|
|
|
file = "???"
|
|
|
|
line = 0
|
2016-08-31 20:39:08 +02:00
|
|
|
}
|
2016-10-11 09:50:27 +02:00
|
|
|
shortFile := true // We are only interested in short file form.
|
|
|
|
lockLocation := funcFromPC(pc, file, line, shortFile)
|
2016-09-27 23:35:43 +02:00
|
|
|
|
2016-10-11 09:50:27 +02:00
|
|
|
n.lock(volume, path, lockLocation, opsID, readLock)
|
2016-04-29 20:39:20 +02:00
|
|
|
}
|
2016-04-29 10:29:09 +02:00
|
|
|
|
2016-04-29 20:39:20 +02:00
|
|
|
// Unlock - unlocks any previously acquired write locks.
|
2016-08-31 20:39:08 +02:00
|
|
|
func (n *nsLockMap) Unlock(volume, path, opsID string) {
|
2016-04-29 20:39:20 +02:00
|
|
|
readLock := false
|
2016-08-31 20:39:08 +02:00
|
|
|
n.unlock(volume, path, opsID, readLock)
|
2016-04-29 20:39:20 +02:00
|
|
|
}
|
2016-04-29 10:29:09 +02:00
|
|
|
|
2016-04-29 20:39:20 +02:00
|
|
|
// RLock - locks any previously acquired read locks.
|
2016-08-31 20:39:08 +02:00
|
|
|
func (n *nsLockMap) RLock(volume, path, opsID string) {
|
2016-04-29 20:39:20 +02:00
|
|
|
readLock := true
|
2016-09-27 23:35:43 +02:00
|
|
|
|
|
|
|
// The caller information of the lock held has been obtained
|
|
|
|
// here before calling any other function.
|
|
|
|
|
|
|
|
// Fetching the package, function name and the line number of
|
2016-10-11 09:50:27 +02:00
|
|
|
// the caller from the runtime.
|
|
|
|
pc, file, line, success := runtime.Caller(1)
|
2016-09-27 23:35:43 +02:00
|
|
|
if !success {
|
2016-10-11 09:50:27 +02:00
|
|
|
file = "???"
|
|
|
|
line = 0
|
2016-08-31 20:39:08 +02:00
|
|
|
}
|
2016-10-11 09:50:27 +02:00
|
|
|
shortFile := true // We are only interested in short file form.
|
|
|
|
lockLocation := funcFromPC(pc, file, line, shortFile)
|
2016-09-27 23:35:43 +02:00
|
|
|
|
2016-10-11 09:50:27 +02:00
|
|
|
n.lock(volume, path, lockLocation, opsID, readLock)
|
2016-04-29 10:29:09 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// RUnlock - unlocks any previously acquired read locks.
|
2016-08-31 20:39:08 +02:00
|
|
|
func (n *nsLockMap) RUnlock(volume, path, opsID string) {
|
2016-04-29 20:39:20 +02:00
|
|
|
readLock := true
|
2016-08-31 20:39:08 +02:00
|
|
|
n.unlock(volume, path, opsID, readLock)
|
2016-04-29 10:29:09 +02:00
|
|
|
}
|
2016-10-19 18:27:36 +02:00
|
|
|
|
|
|
|
// ForceUnlock - forcefully unlock a lock based on name.
|
|
|
|
func (n *nsLockMap) ForceUnlock(volume, path string) {
|
|
|
|
n.lockMapMutex.Lock()
|
|
|
|
defer n.lockMapMutex.Unlock()
|
|
|
|
|
2016-10-20 22:15:28 +02:00
|
|
|
// Clarification on operation:
|
|
|
|
// - In case of FS or XL we call ForceUnlock on the local nsMutex
|
|
|
|
// (since there is only a single server) which will cause the 'stuck'
|
|
|
|
// mutex to be removed from the map. Existing operations for this
|
|
|
|
// will continue to be blocked (and timeout). New operations on this
|
|
|
|
// resource will use a new mutex and proceed normally.
|
|
|
|
//
|
|
|
|
// - In case of Distributed setup (using dsync), there is no need to call
|
|
|
|
// ForceUnlock on the server where the lock was acquired and is presumably
|
|
|
|
// 'stuck'. Instead dsync.ForceUnlock() will release the underlying locks
|
|
|
|
// that participated in granting the lock. Any pending dsync locks that
|
|
|
|
// are blocking can now proceed as normal and any new locks will also
|
|
|
|
// participate normally.
|
|
|
|
|
2016-10-19 18:27:36 +02:00
|
|
|
if n.isDist { // For distributed mode, broadcast ForceUnlock message.
|
|
|
|
dsync.NewDRWMutex(pathutil.Join(volume, path)).ForceUnlock()
|
|
|
|
}
|
|
|
|
|
|
|
|
param := nsParam{volume, path}
|
|
|
|
if _, found := n.lockMap[param]; found {
|
|
|
|
// Remove lock from the map.
|
|
|
|
delete(n.lockMap, param)
|
|
|
|
|
|
|
|
// delete the lock state entry for given
|
|
|
|
// <volume, path> pair.
|
|
|
|
err := n.deleteLockInfoEntryForVolumePath(param)
|
|
|
|
if err != nil {
|
|
|
|
errorIf(err, "Failed to delete lock info entry.")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|