2016-07-31 23:11:14 +02:00
|
|
|
/*
|
|
|
|
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2016-08-18 23:50:50 +02:00
|
|
|
package cmd
|
2016-07-31 23:11:14 +02:00
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
2016-09-16 09:30:55 +02:00
|
|
|
"math/rand"
|
2016-07-31 23:11:14 +02:00
|
|
|
"net/rpc"
|
|
|
|
"path"
|
|
|
|
"strings"
|
|
|
|
"sync"
|
2016-08-22 20:01:21 +02:00
|
|
|
"time"
|
2016-07-31 23:11:14 +02:00
|
|
|
|
|
|
|
router "github.com/gorilla/mux"
|
2016-10-11 09:50:27 +02:00
|
|
|
"github.com/minio/minio-go/pkg/set"
|
2016-07-31 23:11:14 +02:00
|
|
|
)
|
|
|
|
|
2016-08-10 08:57:16 +02:00
|
|
|
const lockRPCPath = "/minio/lock"
|
2016-09-16 09:30:55 +02:00
|
|
|
const lockMaintenanceLoop = 1 * time.Minute
|
|
|
|
const lockCheckValidityInterval = 2 * time.Minute
|
2016-07-31 23:11:14 +02:00
|
|
|
|
2016-08-22 20:01:21 +02:00
|
|
|
// LockArgs besides lock name, holds Token and Timestamp for session
|
|
|
|
// authentication and validation server restart.
|
|
|
|
type LockArgs struct {
|
|
|
|
Name string
|
|
|
|
Token string
|
|
|
|
Timestamp time.Time
|
2016-09-16 09:30:55 +02:00
|
|
|
Node string
|
|
|
|
RPCPath string
|
|
|
|
UID string
|
2016-08-22 20:01:21 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// SetToken - sets the token to the supplied value.
|
|
|
|
func (l *LockArgs) SetToken(token string) {
|
|
|
|
l.Token = token
|
|
|
|
}
|
|
|
|
|
|
|
|
// SetTimestamp - sets the timestamp to the supplied value.
|
|
|
|
func (l *LockArgs) SetTimestamp(tstamp time.Time) {
|
|
|
|
l.Timestamp = tstamp
|
|
|
|
}
|
|
|
|
|
2016-09-16 09:30:55 +02:00
|
|
|
// lockRequesterInfo stores various info from the client for each lock that is requested
|
|
|
|
type lockRequesterInfo struct {
|
|
|
|
writer bool // Bool whether write or read lock
|
|
|
|
node string // Network address of client claiming lock
|
|
|
|
rpcPath string // RPC path of client claiming lock
|
|
|
|
uid string // Uid to uniquely identify request of client
|
|
|
|
timestamp time.Time // Timestamp set at the time of initialization
|
|
|
|
timeLastCheck time.Time // Timestamp for last check of validity of lock
|
|
|
|
}
|
|
|
|
|
|
|
|
// isWriteLock returns whether the lock is a write or read lock
|
|
|
|
func isWriteLock(lri []lockRequesterInfo) bool {
|
|
|
|
return len(lri) == 1 && lri[0].writer
|
|
|
|
}
|
|
|
|
|
|
|
|
// lockServer is type for RPC handlers
|
2016-07-31 23:11:14 +02:00
|
|
|
type lockServer struct {
|
2016-09-19 22:14:55 +02:00
|
|
|
rpcPath string
|
|
|
|
mutex sync.Mutex
|
|
|
|
lockMap map[string][]lockRequesterInfo
|
2016-08-22 20:01:21 +02:00
|
|
|
}
|
|
|
|
|
2016-10-05 21:48:07 +02:00
|
|
|
// Register distributed NS lock handlers.
|
2016-10-13 08:13:24 +02:00
|
|
|
func registerDistNSLockRouter(mux *router.Router, serverConfig serverCmdConfig) error {
|
2016-09-19 22:14:55 +02:00
|
|
|
lockServers := newLockServers(serverConfig)
|
2016-10-13 08:13:24 +02:00
|
|
|
return registerStorageLockers(mux, lockServers)
|
2016-09-19 22:14:55 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Create one lock server for every local storage rpc server.
|
|
|
|
func newLockServers(serverConfig serverCmdConfig) (lockServers []*lockServer) {
|
|
|
|
// Initialize posix storage API.
|
|
|
|
exports := serverConfig.disks
|
|
|
|
ignoredExports := serverConfig.ignoredDisks
|
|
|
|
|
|
|
|
// Save ignored disks in a map
|
2016-10-11 09:50:27 +02:00
|
|
|
// Initialize ignored disks in a new set.
|
|
|
|
ignoredSet := set.NewStringSet()
|
|
|
|
if len(ignoredExports) > 0 {
|
|
|
|
ignoredSet = set.CreateStringSet(ignoredExports...)
|
2016-08-24 04:19:24 +02:00
|
|
|
}
|
2016-09-19 22:14:55 +02:00
|
|
|
for _, export := range exports {
|
2016-10-11 09:50:27 +02:00
|
|
|
if ignoredSet.Contains(export) {
|
|
|
|
// Ignore initializing ignored export.
|
2016-09-19 22:14:55 +02:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
// Not local storage move to the next node.
|
|
|
|
if !isLocalStorage(export) {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if idx := strings.LastIndex(export, ":"); idx != -1 {
|
|
|
|
export = export[idx+1:]
|
|
|
|
}
|
|
|
|
// Create handler for lock RPCs
|
|
|
|
locker := &lockServer{
|
2016-10-11 09:50:27 +02:00
|
|
|
rpcPath: export,
|
|
|
|
mutex: sync.Mutex{},
|
|
|
|
lockMap: make(map[string][]lockRequesterInfo),
|
2016-09-19 22:14:55 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Start loop for stale lock maintenance
|
|
|
|
go func() {
|
|
|
|
// Start with random sleep time, so as to avoid "synchronous checks" between servers
|
|
|
|
time.Sleep(time.Duration(rand.Float64() * float64(lockMaintenanceLoop)))
|
|
|
|
for {
|
|
|
|
time.Sleep(lockMaintenanceLoop)
|
|
|
|
locker.lockMaintenance(lockCheckValidityInterval)
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
lockServers = append(lockServers, locker)
|
|
|
|
}
|
|
|
|
return lockServers
|
|
|
|
}
|
|
|
|
|
|
|
|
// registerStorageLockers - register locker rpc handlers for net/rpc library clients
|
2016-10-13 08:13:24 +02:00
|
|
|
func registerStorageLockers(mux *router.Router, lockServers []*lockServer) error {
|
2016-09-19 22:14:55 +02:00
|
|
|
for _, lockServer := range lockServers {
|
|
|
|
lockRPCServer := rpc.NewServer()
|
2016-10-13 08:13:24 +02:00
|
|
|
err := lockRPCServer.RegisterName("Dsync", lockServer)
|
|
|
|
if err != nil {
|
|
|
|
return traceError(err)
|
|
|
|
}
|
2016-09-19 22:14:55 +02:00
|
|
|
lockRouter := mux.PathPrefix(reservedBucket).Subrouter()
|
|
|
|
lockRouter.Path(path.Join("/lock", lockServer.rpcPath)).Handler(lockRPCServer)
|
2016-08-24 04:19:24 +02:00
|
|
|
}
|
2016-10-13 08:13:24 +02:00
|
|
|
return nil
|
2016-07-31 23:11:14 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/// Distributed lock handlers
|
|
|
|
|
2016-08-22 20:01:21 +02:00
|
|
|
// LoginHandler - handles LoginHandler RPC call.
|
|
|
|
func (l *lockServer) LoginHandler(args *RPCLoginArgs, reply *RPCLoginReply) error {
|
2016-10-05 19:18:55 +02:00
|
|
|
jwt, err := newJWT(defaultInterNodeJWTExpiry)
|
2016-08-22 20:01:21 +02:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if err = jwt.Authenticate(args.Username, args.Password); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
token, err := jwt.GenerateToken(args.Username)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
reply.Token = token
|
2016-10-11 09:50:27 +02:00
|
|
|
reply.Timestamp = time.Now().UTC()
|
2016-09-24 12:34:45 +02:00
|
|
|
reply.ServerVersion = Version
|
2016-08-22 20:01:21 +02:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-08-27 12:16:52 +02:00
|
|
|
// Lock - rpc handler for (single) write lock operation.
|
2016-08-22 20:01:21 +02:00
|
|
|
func (l *lockServer) Lock(args *LockArgs, reply *bool) error {
|
2016-07-31 23:11:14 +02:00
|
|
|
l.mutex.Lock()
|
|
|
|
defer l.mutex.Unlock()
|
2016-09-19 22:14:55 +02:00
|
|
|
if err := l.validateLockArgs(args); err != nil {
|
2016-08-24 04:19:24 +02:00
|
|
|
return err
|
2016-08-22 20:01:21 +02:00
|
|
|
}
|
2016-09-16 09:30:55 +02:00
|
|
|
_, *reply = l.lockMap[args.Name]
|
|
|
|
if !*reply { // No locks held on the given name, so claim write lock
|
2016-09-19 22:14:55 +02:00
|
|
|
l.lockMap[args.Name] = []lockRequesterInfo{
|
|
|
|
{
|
|
|
|
writer: true,
|
|
|
|
node: args.Node,
|
|
|
|
rpcPath: args.RPCPath,
|
|
|
|
uid: args.UID,
|
|
|
|
timestamp: time.Now().UTC(),
|
|
|
|
timeLastCheck: time.Now().UTC(),
|
|
|
|
},
|
|
|
|
}
|
2016-07-31 23:11:14 +02:00
|
|
|
}
|
2016-09-16 09:30:55 +02:00
|
|
|
*reply = !*reply // Negate *reply to return true when lock is granted or false otherwise
|
2016-07-31 23:11:14 +02:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-08-27 12:16:52 +02:00
|
|
|
// Unlock - rpc handler for (single) write unlock operation.
|
2016-08-22 20:01:21 +02:00
|
|
|
func (l *lockServer) Unlock(args *LockArgs, reply *bool) error {
|
2016-07-31 23:11:14 +02:00
|
|
|
l.mutex.Lock()
|
|
|
|
defer l.mutex.Unlock()
|
2016-09-19 22:14:55 +02:00
|
|
|
if err := l.validateLockArgs(args); err != nil {
|
2016-08-24 04:19:24 +02:00
|
|
|
return err
|
2016-08-22 20:01:21 +02:00
|
|
|
}
|
2016-09-16 09:30:55 +02:00
|
|
|
var lri []lockRequesterInfo
|
2016-09-19 22:14:55 +02:00
|
|
|
if lri, *reply = l.lockMap[args.Name]; !*reply { // No lock is held on the given name
|
2016-09-16 09:30:55 +02:00
|
|
|
return fmt.Errorf("Unlock attempted on an unlocked entity: %s", args.Name)
|
|
|
|
}
|
|
|
|
if *reply = isWriteLock(lri); !*reply { // Unless it is a write lock
|
|
|
|
return fmt.Errorf("Unlock attempted on a read locked entity: %s (%d read locks active)", args.Name, len(lri))
|
|
|
|
}
|
2016-09-19 22:14:55 +02:00
|
|
|
if !l.removeEntry(args.Name, args.UID, &lri) {
|
|
|
|
return fmt.Errorf("Unlock unable to find corresponding lock for uid: %s", args.UID)
|
2016-07-31 23:11:14 +02:00
|
|
|
}
|
2016-09-19 22:14:55 +02:00
|
|
|
return nil
|
2016-07-31 23:11:14 +02:00
|
|
|
}
|
|
|
|
|
2016-08-27 12:16:52 +02:00
|
|
|
// RLock - rpc handler for read lock operation.
|
2016-08-22 20:01:21 +02:00
|
|
|
func (l *lockServer) RLock(args *LockArgs, reply *bool) error {
|
2016-08-15 01:57:01 +02:00
|
|
|
l.mutex.Lock()
|
|
|
|
defer l.mutex.Unlock()
|
2016-09-19 22:14:55 +02:00
|
|
|
if err := l.validateLockArgs(args); err != nil {
|
2016-08-24 04:19:24 +02:00
|
|
|
return err
|
2016-08-22 20:01:21 +02:00
|
|
|
}
|
2016-09-19 22:14:55 +02:00
|
|
|
lrInfo := lockRequesterInfo{
|
|
|
|
writer: false,
|
|
|
|
node: args.Node,
|
|
|
|
rpcPath: args.RPCPath,
|
|
|
|
uid: args.UID,
|
|
|
|
timestamp: time.Now().UTC(),
|
|
|
|
timeLastCheck: time.Now().UTC(),
|
|
|
|
}
|
|
|
|
if lri, ok := l.lockMap[args.Name]; ok {
|
2016-09-16 09:30:55 +02:00
|
|
|
if *reply = !isWriteLock(lri); *reply { // Unless there is a write lock
|
2016-09-19 22:14:55 +02:00
|
|
|
l.lockMap[args.Name] = append(l.lockMap[args.Name], lrInfo)
|
2016-09-16 09:30:55 +02:00
|
|
|
}
|
2016-09-19 22:14:55 +02:00
|
|
|
} else { // No locks held on the given name, so claim (first) read lock
|
|
|
|
l.lockMap[args.Name] = []lockRequesterInfo{lrInfo}
|
|
|
|
*reply = true
|
2016-08-15 01:57:01 +02:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-08-27 12:16:52 +02:00
|
|
|
// RUnlock - rpc handler for read unlock operation.
|
2016-08-22 20:01:21 +02:00
|
|
|
func (l *lockServer) RUnlock(args *LockArgs, reply *bool) error {
|
2016-08-15 01:57:01 +02:00
|
|
|
l.mutex.Lock()
|
|
|
|
defer l.mutex.Unlock()
|
2016-09-19 22:14:55 +02:00
|
|
|
if err := l.validateLockArgs(args); err != nil {
|
2016-08-24 04:19:24 +02:00
|
|
|
return err
|
2016-08-22 20:01:21 +02:00
|
|
|
}
|
2016-09-16 09:30:55 +02:00
|
|
|
var lri []lockRequesterInfo
|
|
|
|
if lri, *reply = l.lockMap[args.Name]; !*reply { // No lock is held on the given name
|
|
|
|
return fmt.Errorf("RUnlock attempted on an unlocked entity: %s", args.Name)
|
|
|
|
}
|
|
|
|
if *reply = !isWriteLock(lri); !*reply { // A write-lock is held, cannot release a read lock
|
2016-08-27 12:16:52 +02:00
|
|
|
return fmt.Errorf("RUnlock attempted on a write locked entity: %s", args.Name)
|
2016-08-15 01:57:01 +02:00
|
|
|
}
|
2016-09-19 22:14:55 +02:00
|
|
|
if !l.removeEntry(args.Name, args.UID, &lri) {
|
|
|
|
return fmt.Errorf("RUnlock unable to find corresponding read lock for uid: %s", args.UID)
|
2016-09-16 09:30:55 +02:00
|
|
|
}
|
2016-09-19 22:14:55 +02:00
|
|
|
return nil
|
2016-09-16 09:30:55 +02:00
|
|
|
}
|
|
|
|
|
2016-10-17 10:53:29 +02:00
|
|
|
// ForceUnlock - rpc handler for force unlock operation.
|
|
|
|
func (l *lockServer) ForceUnlock(args *LockArgs, reply *bool) error {
|
|
|
|
l.mutex.Lock()
|
|
|
|
defer l.mutex.Unlock()
|
|
|
|
if err := l.validateLockArgs(args); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if len(args.UID) != 0 {
|
|
|
|
return fmt.Errorf("ForceUnlock called with non-empty UID: %s", args.UID)
|
|
|
|
}
|
|
|
|
if _, ok := l.lockMap[args.Name]; ok { // Only clear lock when set
|
|
|
|
delete(l.lockMap, args.Name) // Remove the lock (irrespective of write or read lock)
|
|
|
|
}
|
|
|
|
*reply = true
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-09-19 22:14:55 +02:00
|
|
|
// Expired - rpc handler for expired lock status.
|
|
|
|
func (l *lockServer) Expired(args *LockArgs, reply *bool) error {
|
2016-09-16 09:30:55 +02:00
|
|
|
l.mutex.Lock()
|
|
|
|
defer l.mutex.Unlock()
|
2016-09-19 22:14:55 +02:00
|
|
|
if err := l.validateLockArgs(args); err != nil {
|
2016-09-16 09:30:55 +02:00
|
|
|
return err
|
|
|
|
}
|
2016-09-19 22:14:55 +02:00
|
|
|
// Lock found, proceed to verify if belongs to given uid.
|
|
|
|
if lri, ok := l.lockMap[args.Name]; ok {
|
|
|
|
// Check whether uid is still active
|
|
|
|
for _, entry := range lri {
|
|
|
|
if entry.uid == args.UID {
|
|
|
|
*reply = false // When uid found, lock is still active so return not expired.
|
|
|
|
return nil // When uid found *reply is set to true.
|
2016-09-16 09:30:55 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2016-09-19 22:14:55 +02:00
|
|
|
// When we get here lock is no longer active due to either args.Name
|
|
|
|
// being absent from map or uid not found for given args.Name
|
|
|
|
*reply = true
|
|
|
|
return nil
|
2016-09-16 09:30:55 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// nameLockRequesterInfoPair is a helper type for lock maintenance
|
|
|
|
type nameLockRequesterInfoPair struct {
|
|
|
|
name string
|
|
|
|
lri lockRequesterInfo
|
|
|
|
}
|
|
|
|
|
|
|
|
// lockMaintenance loops over locks that have been active for some time and checks back
|
|
|
|
// with the original server whether it is still alive or not
|
2016-09-19 22:14:55 +02:00
|
|
|
//
|
|
|
|
// Following logic inside ignores the errors generated for Dsync.Active operation.
|
|
|
|
// - server at client down
|
|
|
|
// - some network error (and server is up normally)
|
|
|
|
//
|
|
|
|
// We will ignore the error, and we will retry later to get a resolve on this lock
|
2016-09-16 09:30:55 +02:00
|
|
|
func (l *lockServer) lockMaintenance(interval time.Duration) {
|
|
|
|
l.mutex.Lock()
|
2016-09-19 22:14:55 +02:00
|
|
|
// Get list of long lived locks to check for staleness.
|
2016-09-16 09:30:55 +02:00
|
|
|
nlripLongLived := getLongLivedLocks(l.lockMap, interval)
|
|
|
|
l.mutex.Unlock()
|
|
|
|
|
2016-09-19 22:14:55 +02:00
|
|
|
// Validate if long lived locks are indeed clean.
|
2016-09-16 09:30:55 +02:00
|
|
|
for _, nlrip := range nlripLongLived {
|
2016-09-19 22:14:55 +02:00
|
|
|
// Initialize client based on the long live locks.
|
2016-09-30 08:42:37 +02:00
|
|
|
c := newClient(nlrip.lri.node, nlrip.lri.rpcPath, isSSL())
|
2016-09-16 09:30:55 +02:00
|
|
|
|
2016-09-19 22:14:55 +02:00
|
|
|
var expired bool
|
2016-09-16 09:30:55 +02:00
|
|
|
|
|
|
|
// Call back to original server verify whether the lock is still active (based on name & uid)
|
2016-09-19 22:14:55 +02:00
|
|
|
c.Call("Dsync.Expired", &LockArgs{
|
|
|
|
Name: nlrip.name,
|
|
|
|
UID: nlrip.lri.uid,
|
|
|
|
}, &expired)
|
|
|
|
c.Close() // Close the connection regardless of the call response.
|
|
|
|
|
|
|
|
// For successful response, verify if lock is indeed active or stale.
|
|
|
|
if expired {
|
|
|
|
// The lock is no longer active at server that originated the lock
|
|
|
|
// So remove the lock from the map.
|
|
|
|
l.mutex.Lock()
|
|
|
|
l.removeEntryIfExists(nlrip) // Purge the stale entry if it exists.
|
|
|
|
l.mutex.Unlock()
|
2016-07-31 23:11:14 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|