2016-10-12 10:03:50 +02:00
|
|
|
/*
|
|
|
|
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
|
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
package cmd
|
|
|
|
|
|
|
|
import (
|
2016-10-15 07:49:51 +02:00
|
|
|
"encoding/json"
|
2016-10-12 10:03:50 +02:00
|
|
|
"fmt"
|
2016-10-19 06:26:58 +02:00
|
|
|
"net/rpc"
|
2016-10-12 10:03:50 +02:00
|
|
|
"path"
|
2016-10-13 10:33:50 +02:00
|
|
|
"sync"
|
2016-10-12 10:03:50 +02:00
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/minio/minio-go/pkg/set"
|
|
|
|
)
|
|
|
|
|
|
|
|
type s3Peers struct {
|
|
|
|
// A map of peer server address (in `host:port` format) to RPC
|
2016-10-13 10:33:50 +02:00
|
|
|
// client connections.
|
|
|
|
rpcClients map[string]*AuthRPCClient
|
2016-10-12 10:03:50 +02:00
|
|
|
|
2016-10-13 10:33:50 +02:00
|
|
|
mutex *sync.RWMutex
|
|
|
|
|
|
|
|
// Slice of all peer addresses (in `host:port` format).
|
2016-10-12 10:03:50 +02:00
|
|
|
peers []string
|
|
|
|
}
|
|
|
|
|
|
|
|
func initGlobalS3Peers(disks []string) {
|
2016-10-13 10:33:50 +02:00
|
|
|
// Get list of de-duplicated peers.
|
2016-10-12 10:03:50 +02:00
|
|
|
peers := getAllPeers(disks)
|
2016-10-13 10:33:50 +02:00
|
|
|
|
|
|
|
// Initialize global state.
|
|
|
|
globalS3Peers = s3Peers{
|
|
|
|
rpcClients: make(map[string]*AuthRPCClient),
|
|
|
|
mutex: &sync.RWMutex{},
|
|
|
|
}
|
|
|
|
// Initialize each peer connection.
|
2016-10-12 10:03:50 +02:00
|
|
|
for _, peer := range peers {
|
|
|
|
globalS3Peers.InitS3PeerClient(peer)
|
|
|
|
}
|
|
|
|
|
2016-10-13 10:33:50 +02:00
|
|
|
// Additionally setup a local peer if one does not exist.
|
2016-10-12 10:03:50 +02:00
|
|
|
if globalS3Peers.GetPeerClient(globalMinioAddr) == nil {
|
|
|
|
globalS3Peers.InitS3PeerClient(globalMinioAddr)
|
|
|
|
peers = append(peers, globalMinioAddr)
|
|
|
|
}
|
|
|
|
|
|
|
|
globalS3Peers.peers = peers
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s3p *s3Peers) GetPeers() []string {
|
|
|
|
return s3p.peers
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s3p *s3Peers) GetPeerClient(peer string) *AuthRPCClient {
|
2016-10-13 10:33:50 +02:00
|
|
|
// Take a read lock
|
|
|
|
s3p.mutex.RLock()
|
|
|
|
defer s3p.mutex.RUnlock()
|
|
|
|
return s3p.rpcClients[peer]
|
2016-10-12 10:03:50 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Initializes a new RPC connection (or closes and re-opens if it
|
|
|
|
// already exists) to a peer. Note that peer address is in `host:port`
|
|
|
|
// format.
|
|
|
|
func (s3p *s3Peers) InitS3PeerClient(peer string) {
|
2016-10-13 10:33:50 +02:00
|
|
|
// Take a write lock
|
|
|
|
s3p.mutex.Lock()
|
|
|
|
defer s3p.mutex.Unlock()
|
|
|
|
|
|
|
|
if s3p.rpcClients[peer] != nil {
|
|
|
|
_ = s3p.rpcClients[peer].Close()
|
|
|
|
delete(s3p.rpcClients, peer)
|
2016-10-12 10:03:50 +02:00
|
|
|
}
|
|
|
|
authCfg := &authConfig{
|
|
|
|
accessKey: serverConfig.GetCredential().AccessKeyID,
|
|
|
|
secretKey: serverConfig.GetCredential().SecretAccessKey,
|
|
|
|
address: peer,
|
2016-10-18 20:46:33 +02:00
|
|
|
secureConn: isSSL(),
|
2016-10-12 10:03:50 +02:00
|
|
|
path: path.Join(reservedBucket, s3Path),
|
|
|
|
loginMethod: "S3.LoginHandler",
|
|
|
|
}
|
2016-10-13 10:33:50 +02:00
|
|
|
s3p.rpcClients[peer] = newAuthClient(authCfg)
|
2016-10-12 10:03:50 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
func (s3p *s3Peers) Close() error {
|
2016-10-13 10:33:50 +02:00
|
|
|
// Take a write lock
|
|
|
|
s3p.mutex.Lock()
|
|
|
|
defer s3p.mutex.Unlock()
|
|
|
|
|
|
|
|
for _, v := range s3p.rpcClients {
|
2016-10-12 10:03:50 +02:00
|
|
|
if err := v.Close(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
2016-10-13 10:33:50 +02:00
|
|
|
s3p.rpcClients = nil
|
2016-10-12 10:03:50 +02:00
|
|
|
s3p.peers = nil
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// returns the network addresses of all Minio servers in the cluster
|
|
|
|
// in `host:port` format.
|
|
|
|
func getAllPeers(disks []string) []string {
|
|
|
|
res := []string{}
|
|
|
|
// use set to de-duplicate
|
|
|
|
sset := set.NewStringSet()
|
|
|
|
for _, disk := range disks {
|
|
|
|
netAddr, _, err := splitNetPath(disk)
|
|
|
|
if err != nil || netAddr == "" {
|
|
|
|
errorIf(err, "Unexpected error - most likely a bug.")
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if !sset.Contains(netAddr) {
|
|
|
|
res = append(
|
|
|
|
res,
|
|
|
|
fmt.Sprintf("%s:%d", netAddr, globalMinioPort),
|
|
|
|
)
|
|
|
|
sset.Add(netAddr)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return res
|
|
|
|
}
|
|
|
|
|
|
|
|
// Make RPC calls with the given method and arguments to all the given
|
|
|
|
// peers (in parallel), and collects the results. Since the methods
|
|
|
|
// intended for use here, have only a success or failure response, we
|
|
|
|
// do not return/inspect the `reply` parameter in the RPC call. The
|
|
|
|
// function attempts to connect to a peer only once, and returns a map
|
|
|
|
// of peer address to error response. If the error is nil, it means
|
|
|
|
// the RPC succeeded.
|
|
|
|
func (s3p *s3Peers) SendRPC(peers []string, method string, args interface {
|
|
|
|
SetToken(token string)
|
|
|
|
SetTimestamp(tstamp time.Time)
|
|
|
|
}) map[string]error {
|
2016-10-13 10:33:50 +02:00
|
|
|
|
2016-10-19 06:26:58 +02:00
|
|
|
// peer error responses array
|
|
|
|
errArr := make([]error, len(peers))
|
|
|
|
|
|
|
|
// Start a wait group and make RPC requests to peers.
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
for i, target := range peers {
|
|
|
|
wg.Add(1)
|
|
|
|
go func(ix int, target string) {
|
|
|
|
defer wg.Done()
|
|
|
|
reply := &GenericReply{}
|
|
|
|
// Get RPC client object safely.
|
|
|
|
client := s3p.GetPeerClient(target)
|
|
|
|
var err error
|
|
|
|
if client == nil {
|
|
|
|
err = fmt.Errorf("Requested client was not initialized - %v",
|
|
|
|
target)
|
|
|
|
} else {
|
|
|
|
err = client.Call(method, args, reply)
|
|
|
|
// Check for network errors and try
|
|
|
|
// again just once.
|
|
|
|
if err != nil {
|
|
|
|
if err.Error() == rpc.ErrShutdown.Error() {
|
|
|
|
err = client.Call(method, args, reply)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
errArr[ix] = err
|
|
|
|
}(i, target)
|
2016-10-12 10:03:50 +02:00
|
|
|
}
|
2016-10-13 10:33:50 +02:00
|
|
|
|
2016-10-19 06:26:58 +02:00
|
|
|
// Wait for requests to complete.
|
|
|
|
wg.Wait()
|
|
|
|
|
2016-10-13 10:33:50 +02:00
|
|
|
// Map of errors
|
2016-10-12 10:03:50 +02:00
|
|
|
errsMap := make(map[string]error)
|
2016-10-19 06:26:58 +02:00
|
|
|
for i, errVal := range errArr {
|
|
|
|
if errVal != nil {
|
|
|
|
errsMap[peers[i]] = errVal
|
2016-10-12 10:03:50 +02:00
|
|
|
}
|
|
|
|
}
|
2016-10-13 10:33:50 +02:00
|
|
|
|
2016-10-12 10:03:50 +02:00
|
|
|
return errsMap
|
|
|
|
}
|
|
|
|
|
|
|
|
// S3PeersUpdateBucketNotification - Sends Update Bucket notification
|
|
|
|
// request to all peers. Currently we log an error and continue.
|
|
|
|
func S3PeersUpdateBucketNotification(bucket string, ncfg *notificationConfig) {
|
|
|
|
setBNPArgs := &SetBNPArgs{Bucket: bucket, NCfg: ncfg}
|
|
|
|
peers := globalS3Peers.GetPeers()
|
|
|
|
errsMap := globalS3Peers.SendRPC(peers, "S3.SetBucketNotificationPeer",
|
|
|
|
setBNPArgs)
|
|
|
|
for peer, err := range errsMap {
|
|
|
|
errorIf(err, "Error sending peer update bucket notification to %s - %v", peer, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// S3PeersUpdateBucketListener - Sends Update Bucket listeners request
|
|
|
|
// to all peers. Currently we log an error and continue.
|
|
|
|
func S3PeersUpdateBucketListener(bucket string, lcfg []listenerConfig) {
|
|
|
|
setBLPArgs := &SetBLPArgs{Bucket: bucket, LCfg: lcfg}
|
|
|
|
peers := globalS3Peers.GetPeers()
|
|
|
|
errsMap := globalS3Peers.SendRPC(peers, "S3.SetBucketListenerPeer",
|
|
|
|
setBLPArgs)
|
|
|
|
for peer, err := range errsMap {
|
|
|
|
errorIf(err, "Error sending peer update bucket listener to %s - %v", peer, err)
|
|
|
|
}
|
|
|
|
}
|
2016-10-13 18:19:04 +02:00
|
|
|
|
|
|
|
// S3PeersUpdateBucketPolicy - Sends update bucket policy request to
|
|
|
|
// all peers. Currently we log an error and continue.
|
|
|
|
func S3PeersUpdateBucketPolicy(bucket string, pCh policyChange) {
|
2016-10-15 07:49:51 +02:00
|
|
|
byts, err := json.Marshal(pCh)
|
|
|
|
if err != nil {
|
|
|
|
errorIf(err, "Failed to marshal policyChange - this is a BUG!")
|
|
|
|
return
|
|
|
|
}
|
|
|
|
setBPPArgs := &SetBPPArgs{Bucket: bucket, PChBytes: byts}
|
2016-10-13 18:19:04 +02:00
|
|
|
peers := globalS3Peers.GetPeers()
|
|
|
|
errsMap := globalS3Peers.SendRPC(peers, "S3.SetBucketPolicyPeer", setBPPArgs)
|
|
|
|
for peer, err := range errsMap {
|
|
|
|
errorIf(err, "Error sending peer update bucket policy to %s - %v", peer, err)
|
|
|
|
}
|
|
|
|
}
|