2015-04-29 11:19:51 +02:00
|
|
|
/*
|
2015-07-25 02:51:40 +02:00
|
|
|
* Minio Cloud Storage, (C) 2015 Minio, Inc.
|
2015-04-29 11:19:51 +02:00
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2016-08-19 01:23:42 +02:00
|
|
|
package cmd
|
2015-04-23 01:28:13 +02:00
|
|
|
|
|
|
|
import (
|
|
|
|
"encoding/base64"
|
2016-04-21 02:35:38 +02:00
|
|
|
"encoding/xml"
|
2016-08-21 21:06:53 +02:00
|
|
|
"errors"
|
2016-08-26 09:11:53 +02:00
|
|
|
"fmt"
|
2016-04-21 02:35:38 +02:00
|
|
|
"io"
|
2016-08-15 23:33:48 +02:00
|
|
|
"net"
|
2016-09-09 18:38:07 +02:00
|
|
|
"net/http"
|
2016-08-05 22:48:31 +02:00
|
|
|
"os"
|
2016-08-21 21:06:53 +02:00
|
|
|
"os/exec"
|
2016-08-15 23:33:48 +02:00
|
|
|
"path/filepath"
|
|
|
|
"runtime"
|
2016-04-21 02:35:38 +02:00
|
|
|
"strings"
|
2016-08-15 08:55:48 +02:00
|
|
|
"sync"
|
2016-08-05 22:48:31 +02:00
|
|
|
"syscall"
|
2016-09-02 05:13:11 +02:00
|
|
|
|
2016-09-19 19:17:46 +02:00
|
|
|
"encoding/json"
|
|
|
|
|
2016-09-02 05:13:11 +02:00
|
|
|
"github.com/pkg/profile"
|
2015-04-23 01:28:13 +02:00
|
|
|
)
|
|
|
|
|
2016-09-09 18:38:07 +02:00
|
|
|
// make a copy of http.Header
|
|
|
|
func cloneHeader(h http.Header) http.Header {
|
|
|
|
h2 := make(http.Header, len(h))
|
|
|
|
for k, vv := range h {
|
|
|
|
vv2 := make([]string, len(vv))
|
|
|
|
copy(vv2, vv)
|
|
|
|
h2[k] = vv2
|
|
|
|
|
|
|
|
}
|
|
|
|
return h2
|
|
|
|
}
|
|
|
|
|
2016-08-26 09:11:53 +02:00
|
|
|
// checkDuplicates - function to validate if there are duplicates in a slice of strings.
|
|
|
|
func checkDuplicates(list []string) error {
|
|
|
|
// Empty lists are not allowed.
|
|
|
|
if len(list) == 0 {
|
|
|
|
return errInvalidArgument
|
|
|
|
}
|
|
|
|
// Empty keys are not allowed.
|
|
|
|
for _, key := range list {
|
|
|
|
if key == "" {
|
|
|
|
return errInvalidArgument
|
|
|
|
}
|
|
|
|
}
|
|
|
|
listMaps := make(map[string]int)
|
|
|
|
// Navigate through each configs and count the entries.
|
|
|
|
for _, key := range list {
|
|
|
|
listMaps[key]++
|
|
|
|
}
|
|
|
|
// Validate if there are any duplicate counts.
|
|
|
|
for key, count := range listMaps {
|
|
|
|
if count != 1 {
|
|
|
|
return fmt.Errorf("Duplicate key: \"%s\" found of count: \"%d\"", key, count)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// No duplicates.
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-08-15 23:33:48 +02:00
|
|
|
// splits network path into its components Address and Path.
|
|
|
|
func splitNetPath(networkPath string) (netAddr, netPath string, err error) {
|
|
|
|
if runtime.GOOS == "windows" {
|
|
|
|
if volumeName := filepath.VolumeName(networkPath); volumeName != "" {
|
|
|
|
return "", networkPath, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
networkParts := strings.SplitN(networkPath, ":", 2)
|
2016-08-26 09:11:53 +02:00
|
|
|
if len(networkParts) == 1 {
|
2016-08-15 23:33:48 +02:00
|
|
|
return "", networkPath, nil
|
2016-08-26 09:11:53 +02:00
|
|
|
}
|
|
|
|
if networkParts[1] == "" {
|
|
|
|
return "", "", &net.AddrError{Err: "Missing path in network path", Addr: networkPath}
|
|
|
|
} else if networkParts[0] == "" {
|
|
|
|
return "", "", &net.AddrError{Err: "Missing address in network path", Addr: networkPath}
|
|
|
|
} else if !filepath.IsAbs(networkParts[1]) {
|
|
|
|
return "", "", &net.AddrError{Err: "Network path should be absolute", Addr: networkPath}
|
2016-08-15 23:33:48 +02:00
|
|
|
}
|
|
|
|
return networkParts[0], networkParts[1], nil
|
|
|
|
}
|
|
|
|
|
2016-04-21 02:35:38 +02:00
|
|
|
// xmlDecoder provide decoded value in xml.
|
2016-07-19 06:20:17 +02:00
|
|
|
func xmlDecoder(body io.Reader, v interface{}, size int64) error {
|
|
|
|
var lbody io.Reader
|
|
|
|
if size > 0 {
|
|
|
|
lbody = io.LimitReader(body, size)
|
|
|
|
} else {
|
|
|
|
lbody = body
|
|
|
|
}
|
|
|
|
d := xml.NewDecoder(lbody)
|
2016-04-21 02:35:38 +02:00
|
|
|
return d.Decode(v)
|
|
|
|
}
|
|
|
|
|
2016-03-13 01:08:15 +01:00
|
|
|
// checkValidMD5 - verify if valid md5, returns md5 in bytes.
|
2016-04-29 23:24:10 +02:00
|
|
|
func checkValidMD5(md5 string) ([]byte, error) {
|
|
|
|
return base64.StdEncoding.DecodeString(strings.TrimSpace(md5))
|
2015-04-23 01:28:13 +02:00
|
|
|
}
|
2015-04-29 11:19:51 +02:00
|
|
|
|
2015-04-29 19:51:59 +02:00
|
|
|
/// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html
|
2015-04-29 11:19:51 +02:00
|
|
|
const (
|
2015-12-28 08:00:36 +01:00
|
|
|
// maximum object size per PUT request is 5GiB
|
2015-04-29 11:19:51 +02:00
|
|
|
maxObjectSize = 1024 * 1024 * 1024 * 5
|
2016-05-08 21:06:05 +02:00
|
|
|
// minimum Part size for multipart upload is 5MB
|
|
|
|
minPartSize = 1024 * 1024 * 5
|
2016-05-24 10:52:47 +02:00
|
|
|
// maximum Part ID for multipart upload is 10000 (Acceptable values range from 1 to 10000 inclusive)
|
|
|
|
maxPartID = 10000
|
2015-04-29 11:19:51 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
// isMaxObjectSize - verify if max object size
|
2015-12-28 08:00:36 +01:00
|
|
|
func isMaxObjectSize(size int64) bool {
|
2016-04-03 02:28:54 +02:00
|
|
|
return size > maxObjectSize
|
2015-04-29 11:19:51 +02:00
|
|
|
}
|
2016-02-05 12:09:31 +01:00
|
|
|
|
2016-05-08 21:06:05 +02:00
|
|
|
// Check if part size is more than or equal to minimum allowed size.
|
|
|
|
func isMinAllowedPartSize(size int64) bool {
|
|
|
|
return size >= minPartSize
|
|
|
|
}
|
|
|
|
|
2016-05-24 10:52:47 +02:00
|
|
|
// isMaxPartNumber - Check if part ID is greater than the maximum allowed ID.
|
|
|
|
func isMaxPartID(partID int) bool {
|
|
|
|
return partID > maxPartID
|
|
|
|
}
|
|
|
|
|
2016-02-05 12:09:31 +01:00
|
|
|
func contains(stringList []string, element string) bool {
|
|
|
|
for _, e := range stringList {
|
|
|
|
if e == element {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
2016-08-05 22:48:31 +02:00
|
|
|
|
2016-08-15 08:55:48 +02:00
|
|
|
// Represents a type of an exit func which will be invoked upon shutdown signal.
|
|
|
|
type onExitFunc func(code int)
|
2016-08-05 22:48:31 +02:00
|
|
|
|
2016-08-15 08:55:48 +02:00
|
|
|
// Represents a type for all the the callback functions invoked upon shutdown signal.
|
|
|
|
type cleanupOnExitFunc func() errCode
|
|
|
|
|
|
|
|
// Represents a collection of various callbacks executed upon exit signals.
|
|
|
|
type shutdownCallbacks struct {
|
|
|
|
// Protect callbacks list from a concurrent access
|
|
|
|
*sync.RWMutex
|
|
|
|
// genericCallbacks - is the list of function callbacks executed one by one
|
|
|
|
// when a shutdown starts. A callback returns 0 for success and 1 for failure.
|
|
|
|
// Failure is considered an emergency error that needs an immediate exit
|
|
|
|
genericCallbacks []cleanupOnExitFunc
|
|
|
|
// objectLayerCallbacks - contains the list of function callbacks that
|
|
|
|
// need to be invoked when a shutdown starts. These callbacks will be called before
|
|
|
|
// the general callback shutdowns
|
|
|
|
objectLayerCallbacks []cleanupOnExitFunc
|
|
|
|
}
|
2016-08-05 22:48:31 +02:00
|
|
|
|
2016-08-15 08:55:48 +02:00
|
|
|
// globalShutdownCBs stores regular and object storages callbacks
|
|
|
|
var globalShutdownCBs *shutdownCallbacks
|
2016-08-05 22:48:31 +02:00
|
|
|
|
2016-09-14 03:43:45 +02:00
|
|
|
func (s *shutdownCallbacks) RunObjectLayerCBs() errCode {
|
2016-08-15 08:55:48 +02:00
|
|
|
s.RLock()
|
|
|
|
defer s.RUnlock()
|
2016-09-14 03:43:45 +02:00
|
|
|
exitCode := exitSuccess
|
|
|
|
for _, callback := range s.objectLayerCallbacks {
|
|
|
|
exitCode = callback()
|
|
|
|
if exitCode != exitSuccess {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return exitCode
|
2016-08-05 22:48:31 +02:00
|
|
|
}
|
|
|
|
|
2016-09-14 03:43:45 +02:00
|
|
|
func (s *shutdownCallbacks) RunGenericCBs() errCode {
|
2016-08-15 08:55:48 +02:00
|
|
|
s.RLock()
|
|
|
|
defer s.RUnlock()
|
2016-09-14 03:43:45 +02:00
|
|
|
exitCode := exitSuccess
|
|
|
|
for _, callback := range s.genericCallbacks {
|
|
|
|
exitCode = callback()
|
|
|
|
if exitCode != exitSuccess {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return exitCode
|
2016-08-05 22:48:31 +02:00
|
|
|
}
|
|
|
|
|
2016-08-15 08:55:48 +02:00
|
|
|
func (s *shutdownCallbacks) AddObjectLayerCB(callback cleanupOnExitFunc) error {
|
|
|
|
s.Lock()
|
|
|
|
defer s.Unlock()
|
|
|
|
if callback == nil {
|
|
|
|
return errInvalidArgument
|
|
|
|
}
|
|
|
|
s.objectLayerCallbacks = append(s.objectLayerCallbacks, callback)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *shutdownCallbacks) AddGenericCB(callback cleanupOnExitFunc) error {
|
|
|
|
s.Lock()
|
|
|
|
defer s.Unlock()
|
|
|
|
if callback == nil {
|
|
|
|
return errInvalidArgument
|
|
|
|
}
|
|
|
|
s.genericCallbacks = append(s.genericCallbacks, callback)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Initialize graceful shutdown mechanism.
|
|
|
|
func initGracefulShutdown(onExitFn onExitFunc) error {
|
|
|
|
// Validate exit func.
|
|
|
|
if onExitFn == nil {
|
|
|
|
return errInvalidArgument
|
|
|
|
}
|
|
|
|
globalShutdownCBs = &shutdownCallbacks{
|
|
|
|
RWMutex: &sync.RWMutex{},
|
|
|
|
}
|
|
|
|
// Return start monitor shutdown signal.
|
|
|
|
return startMonitorShutdownSignal(onExitFn)
|
|
|
|
}
|
|
|
|
|
2016-08-21 21:06:53 +02:00
|
|
|
type shutdownSignal int
|
|
|
|
|
|
|
|
const (
|
|
|
|
shutdownHalt = iota
|
|
|
|
shutdownRestart
|
|
|
|
)
|
|
|
|
|
2016-09-02 05:13:11 +02:00
|
|
|
// Starts a profiler returns nil if profiler is not enabled, caller needs to handle this.
|
|
|
|
func startProfiler(profiler string) interface {
|
|
|
|
Stop()
|
|
|
|
} {
|
|
|
|
// Set ``MINIO_PROFILE_DIR`` to the directory where profiling information should be persisted
|
|
|
|
profileDir := os.Getenv("MINIO_PROFILE_DIR")
|
|
|
|
// Enable profiler if ``MINIO_PROFILER`` is set. Supported options are [cpu, mem, block].
|
|
|
|
switch profiler {
|
|
|
|
case "cpu":
|
|
|
|
return profile.Start(profile.CPUProfile, profile.NoShutdownHook, profile.ProfilePath(profileDir))
|
|
|
|
case "mem":
|
|
|
|
return profile.Start(profile.MemProfile, profile.NoShutdownHook, profile.ProfilePath(profileDir))
|
|
|
|
case "block":
|
|
|
|
return profile.Start(profile.BlockProfile, profile.NoShutdownHook, profile.ProfilePath(profileDir))
|
|
|
|
default:
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-08-15 08:55:48 +02:00
|
|
|
// Global shutdown signal channel.
|
2016-08-21 21:06:53 +02:00
|
|
|
var globalShutdownSignalCh = make(chan shutdownSignal, 1)
|
2016-08-07 08:53:10 +02:00
|
|
|
|
2016-09-02 05:13:11 +02:00
|
|
|
// Global profiler to be used by shutdown go-routine.
|
|
|
|
var globalProfiler interface {
|
|
|
|
Stop()
|
|
|
|
}
|
|
|
|
|
2016-08-05 22:48:31 +02:00
|
|
|
// Start to monitor shutdownSignal to execute shutdown callbacks
|
2016-08-15 08:55:48 +02:00
|
|
|
func startMonitorShutdownSignal(onExitFn onExitFunc) error {
|
|
|
|
// Validate exit func.
|
|
|
|
if onExitFn == nil {
|
|
|
|
return errInvalidArgument
|
|
|
|
}
|
2016-09-02 05:13:11 +02:00
|
|
|
|
2016-09-14 03:43:45 +02:00
|
|
|
// Custom exit function
|
|
|
|
runExitFn := func(exitCode errCode) {
|
|
|
|
// If global profiler is set stop before we exit.
|
|
|
|
if globalProfiler != nil {
|
|
|
|
globalProfiler.Stop()
|
|
|
|
}
|
|
|
|
// Call user supplied user exit function
|
|
|
|
onExitFn(int(exitCode))
|
|
|
|
}
|
|
|
|
|
2016-09-02 05:13:11 +02:00
|
|
|
// Start listening on shutdown signal.
|
2016-08-05 22:48:31 +02:00
|
|
|
go func() {
|
2016-08-15 08:55:48 +02:00
|
|
|
defer close(globalShutdownSignalCh)
|
2016-09-02 05:13:11 +02:00
|
|
|
|
2016-08-07 08:53:10 +02:00
|
|
|
// Monitor signals.
|
2016-08-05 22:48:31 +02:00
|
|
|
trapCh := signalTrap(os.Interrupt, syscall.SIGTERM)
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-trapCh:
|
2016-08-15 08:55:48 +02:00
|
|
|
// Initiate graceful shutdown.
|
2016-08-21 21:06:53 +02:00
|
|
|
globalShutdownSignalCh <- shutdownHalt
|
|
|
|
case signal := <-globalShutdownSignalCh:
|
2016-08-15 08:55:48 +02:00
|
|
|
// Call all object storage shutdown callbacks and exit for emergency
|
2016-09-14 03:43:45 +02:00
|
|
|
exitCode := globalShutdownCBs.RunObjectLayerCBs()
|
|
|
|
if exitCode != exitSuccess {
|
|
|
|
runExitFn(exitCode)
|
2016-08-05 22:48:31 +02:00
|
|
|
|
|
|
|
}
|
2016-09-14 03:43:45 +02:00
|
|
|
exitCode = globalShutdownCBs.RunGenericCBs()
|
|
|
|
if exitCode != exitSuccess {
|
|
|
|
runExitFn(exitCode)
|
2016-08-05 22:48:31 +02:00
|
|
|
}
|
2016-08-21 21:06:53 +02:00
|
|
|
// All shutdown callbacks ensure that the server is safely terminated
|
|
|
|
// and any concurrent process could be started again
|
|
|
|
if signal == shutdownRestart {
|
|
|
|
path := os.Args[0]
|
|
|
|
cmdArgs := os.Args[1:]
|
|
|
|
cmd := exec.Command(path, cmdArgs...)
|
|
|
|
cmd.Stdout = os.Stdout
|
|
|
|
cmd.Stderr = os.Stderr
|
|
|
|
|
|
|
|
err := cmd.Start()
|
|
|
|
if err != nil {
|
|
|
|
errorIf(errors.New("Unable to reboot."), err.Error())
|
|
|
|
}
|
2016-09-02 05:13:11 +02:00
|
|
|
|
2016-09-01 12:50:31 +02:00
|
|
|
// Successfully forked.
|
2016-09-14 03:43:45 +02:00
|
|
|
runExitFn(exitSuccess)
|
2016-09-01 12:50:31 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Exit as success if no errors.
|
2016-09-14 03:43:45 +02:00
|
|
|
runExitFn(exitSuccess)
|
2016-08-05 22:48:31 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
2016-08-15 08:55:48 +02:00
|
|
|
// Successfully started routine.
|
|
|
|
return nil
|
2016-08-05 22:48:31 +02:00
|
|
|
}
|
2016-09-19 19:17:46 +02:00
|
|
|
|
|
|
|
// dump the request into a string in JSON format.
|
|
|
|
func dumpRequest(r *http.Request) string {
|
|
|
|
header := cloneHeader(r.Header)
|
|
|
|
header.Set("Host", r.Host)
|
|
|
|
req := struct {
|
|
|
|
Method string `json:"method"`
|
|
|
|
Path string `json:"path"`
|
|
|
|
Query string `json:"query"`
|
|
|
|
Header http.Header `json:"header"`
|
|
|
|
}{r.Method, r.URL.Path, r.URL.RawQuery, header}
|
|
|
|
jsonBytes, err := json.Marshal(req)
|
|
|
|
if err != nil {
|
|
|
|
return "<error dumping request>"
|
|
|
|
}
|
|
|
|
return string(jsonBytes)
|
|
|
|
}
|