2015-04-29 11:19:51 +02:00
|
|
|
/*
|
2019-04-09 20:39:42 +02:00
|
|
|
* MinIO Cloud Storage, (C) 2015, 2016, 2017 MinIO, Inc.
|
2015-04-29 11:19:51 +02:00
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2016-08-19 01:23:42 +02:00
|
|
|
package cmd
|
2015-04-23 01:28:13 +02:00
|
|
|
|
|
|
|
import (
|
2017-05-31 09:11:06 +02:00
|
|
|
"bytes"
|
2018-03-14 20:01:47 +01:00
|
|
|
"context"
|
2017-12-06 02:58:09 +01:00
|
|
|
"crypto/tls"
|
2015-04-23 01:28:13 +02:00
|
|
|
"encoding/base64"
|
2017-04-07 23:37:32 +02:00
|
|
|
"encoding/json"
|
2016-04-21 02:35:38 +02:00
|
|
|
"encoding/xml"
|
2018-09-19 01:46:35 +02:00
|
|
|
"errors"
|
2016-08-26 09:11:53 +02:00
|
|
|
"fmt"
|
2016-04-21 02:35:38 +02:00
|
|
|
"io"
|
2018-09-06 01:47:14 +02:00
|
|
|
"io/ioutil"
|
2017-12-06 02:58:09 +01:00
|
|
|
"net"
|
2016-09-09 18:38:07 +02:00
|
|
|
"net/http"
|
2017-10-13 12:01:15 +02:00
|
|
|
"os"
|
2018-09-19 01:46:35 +02:00
|
|
|
"path/filepath"
|
2017-12-28 18:32:48 +01:00
|
|
|
"reflect"
|
2016-04-21 02:35:38 +02:00
|
|
|
"strings"
|
2017-03-18 19:28:41 +01:00
|
|
|
"time"
|
2016-09-02 05:13:11 +02:00
|
|
|
|
2019-11-21 13:52:35 +01:00
|
|
|
"github.com/beevik/ntp"
|
2019-07-03 07:34:32 +02:00
|
|
|
xhttp "github.com/minio/minio/cmd/http"
|
2018-03-15 02:36:54 +01:00
|
|
|
"github.com/minio/minio/cmd/logger"
|
2018-07-02 23:40:18 +02:00
|
|
|
"github.com/minio/minio/pkg/handlers"
|
2019-10-23 07:59:13 +02:00
|
|
|
iampolicy "github.com/minio/minio/pkg/iam/policy"
|
2018-03-15 02:36:54 +01:00
|
|
|
|
2016-11-23 03:18:22 +01:00
|
|
|
humanize "github.com/dustin/go-humanize"
|
2018-03-14 20:01:47 +01:00
|
|
|
"github.com/gorilla/mux"
|
2016-09-02 05:13:11 +02:00
|
|
|
"github.com/pkg/profile"
|
2015-04-23 01:28:13 +02:00
|
|
|
)
|
|
|
|
|
2018-04-10 18:36:37 +02:00
|
|
|
// IsErrIgnored returns whether given error is ignored or not.
|
|
|
|
func IsErrIgnored(err error, ignoredErrs ...error) bool {
|
|
|
|
return IsErr(err, ignoredErrs...)
|
|
|
|
}
|
|
|
|
|
|
|
|
// IsErr returns whether given error is exact error.
|
|
|
|
func IsErr(err error, errs ...error) bool {
|
|
|
|
for _, exactErr := range errs {
|
2019-09-11 19:21:43 +02:00
|
|
|
if errors.Is(err, exactErr) {
|
2018-04-10 18:36:37 +02:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2019-06-27 03:21:54 +02:00
|
|
|
func request2BucketObjectName(r *http.Request) (bucketName, objectName string) {
|
|
|
|
path, err := getResource(r.URL.Path, r.Host, globalDomainNames)
|
|
|
|
if err != nil {
|
|
|
|
logger.CriticalIf(context.Background(), err)
|
|
|
|
}
|
|
|
|
return urlPath2BucketObjectName(path)
|
|
|
|
}
|
|
|
|
|
2017-01-10 20:01:23 +01:00
|
|
|
// Convert url path into bucket and object name.
|
2018-02-28 04:14:49 +01:00
|
|
|
func urlPath2BucketObjectName(path string) (bucketName, objectName string) {
|
2019-08-06 21:08:58 +02:00
|
|
|
if path == "" || path == SlashSeparator {
|
2019-06-27 03:21:54 +02:00
|
|
|
return "", ""
|
|
|
|
}
|
|
|
|
|
2017-01-10 20:01:23 +01:00
|
|
|
// Trim any preceding slash separator.
|
2019-08-06 21:08:58 +02:00
|
|
|
urlPath := strings.TrimPrefix(path, SlashSeparator)
|
2017-01-10 20:01:23 +01:00
|
|
|
|
|
|
|
// Split urlpath using slash separator into a given number of
|
|
|
|
// expected tokens.
|
2019-08-06 21:08:58 +02:00
|
|
|
tokens := strings.SplitN(urlPath, SlashSeparator, 2)
|
2017-04-12 00:44:27 +02:00
|
|
|
bucketName = tokens[0]
|
|
|
|
if len(tokens) == 2 {
|
|
|
|
objectName = tokens[1]
|
|
|
|
}
|
2017-01-10 20:01:23 +01:00
|
|
|
|
|
|
|
// Success.
|
|
|
|
return bucketName, objectName
|
|
|
|
}
|
|
|
|
|
2017-01-18 21:24:34 +01:00
|
|
|
// URI scheme constants.
|
|
|
|
const (
|
|
|
|
httpScheme = "http"
|
|
|
|
httpsScheme = "https"
|
|
|
|
)
|
|
|
|
|
2018-12-13 21:09:50 +01:00
|
|
|
// nopCharsetConverter is a dummy charset convert which just copies input to output,
|
|
|
|
// it is used to ignore custom encoding charset in S3 XML body.
|
|
|
|
func nopCharsetConverter(label string, input io.Reader) (io.Reader, error) {
|
|
|
|
return input, nil
|
|
|
|
}
|
|
|
|
|
2016-04-21 02:35:38 +02:00
|
|
|
// xmlDecoder provide decoded value in xml.
|
2016-07-19 06:20:17 +02:00
|
|
|
func xmlDecoder(body io.Reader, v interface{}, size int64) error {
|
|
|
|
var lbody io.Reader
|
|
|
|
if size > 0 {
|
|
|
|
lbody = io.LimitReader(body, size)
|
|
|
|
} else {
|
|
|
|
lbody = body
|
|
|
|
}
|
|
|
|
d := xml.NewDecoder(lbody)
|
2018-12-13 21:09:50 +01:00
|
|
|
// Ignore any encoding set in the XML body
|
|
|
|
d.CharsetReader = nopCharsetConverter
|
2016-04-21 02:35:38 +02:00
|
|
|
return d.Decode(v)
|
|
|
|
}
|
|
|
|
|
2016-03-13 01:08:15 +01:00
|
|
|
// checkValidMD5 - verify if valid md5, returns md5 in bytes.
|
2018-03-16 19:22:34 +01:00
|
|
|
func checkValidMD5(h http.Header) ([]byte, error) {
|
|
|
|
md5B64, ok := h["Content-Md5"]
|
|
|
|
if ok {
|
|
|
|
if md5B64[0] == "" {
|
|
|
|
return nil, fmt.Errorf("Content-Md5 header set to empty value")
|
|
|
|
}
|
|
|
|
return base64.StdEncoding.DecodeString(md5B64[0])
|
|
|
|
}
|
|
|
|
return []byte{}, nil
|
2015-04-23 01:28:13 +02:00
|
|
|
}
|
2015-04-29 11:19:51 +02:00
|
|
|
|
2015-04-29 19:51:59 +02:00
|
|
|
/// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html
|
2015-04-29 11:19:51 +02:00
|
|
|
const (
|
2017-11-28 08:29:02 +01:00
|
|
|
// Maximum object size per PUT request is 5TB.
|
2017-03-03 19:14:17 +01:00
|
|
|
// This is a divergence from S3 limit on purpose to support
|
|
|
|
// use cases where users are going to upload large files
|
|
|
|
// using 'curl' and presigned URL.
|
2017-11-28 08:29:02 +01:00
|
|
|
globalMaxObjectSize = 5 * humanize.TiByte
|
2017-03-03 19:14:17 +01:00
|
|
|
|
|
|
|
// Minimum Part size for multipart upload is 5MiB
|
|
|
|
globalMinPartSize = 5 * humanize.MiByte
|
|
|
|
|
|
|
|
// Maximum Part size for multipart upload is 5GiB
|
|
|
|
globalMaxPartSize = 5 * humanize.GiByte
|
|
|
|
|
|
|
|
// Maximum Part ID for multipart upload is 10000
|
|
|
|
// (Acceptable values range from 1 to 10000 inclusive)
|
|
|
|
globalMaxPartID = 10000
|
2018-07-12 23:12:40 +02:00
|
|
|
|
|
|
|
// Default values used while communicating with the cloud backends
|
|
|
|
defaultDialTimeout = 30 * time.Second
|
|
|
|
defaultDialKeepAlive = 30 * time.Second
|
2015-04-29 11:19:51 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
// isMaxObjectSize - verify if max object size
|
2015-12-28 08:00:36 +01:00
|
|
|
func isMaxObjectSize(size int64) bool {
|
2017-03-03 19:14:17 +01:00
|
|
|
return size > globalMaxObjectSize
|
|
|
|
}
|
|
|
|
|
|
|
|
// // Check if part size is more than maximum allowed size.
|
|
|
|
func isMaxAllowedPartSize(size int64) bool {
|
|
|
|
return size > globalMaxPartSize
|
2015-04-29 11:19:51 +02:00
|
|
|
}
|
2016-02-05 12:09:31 +01:00
|
|
|
|
2016-05-08 21:06:05 +02:00
|
|
|
// Check if part size is more than or equal to minimum allowed size.
|
|
|
|
func isMinAllowedPartSize(size int64) bool {
|
2017-03-03 19:14:17 +01:00
|
|
|
return size >= globalMinPartSize
|
2016-05-08 21:06:05 +02:00
|
|
|
}
|
|
|
|
|
2016-05-24 10:52:47 +02:00
|
|
|
// isMaxPartNumber - Check if part ID is greater than the maximum allowed ID.
|
|
|
|
func isMaxPartID(partID int) bool {
|
2017-03-03 19:14:17 +01:00
|
|
|
return partID > globalMaxPartID
|
2016-05-24 10:52:47 +02:00
|
|
|
}
|
|
|
|
|
2017-12-28 18:32:48 +01:00
|
|
|
func contains(slice interface{}, elem interface{}) bool {
|
|
|
|
v := reflect.ValueOf(slice)
|
|
|
|
if v.Kind() == reflect.Slice {
|
|
|
|
for i := 0; i < v.Len(); i++ {
|
|
|
|
if v.Index(i).Interface() == elem {
|
|
|
|
return true
|
|
|
|
}
|
2016-02-05 12:09:31 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
2016-08-05 22:48:31 +02:00
|
|
|
|
2018-09-19 01:46:35 +02:00
|
|
|
// profilerWrapper is created becauses pkg/profiler doesn't
|
|
|
|
// provide any API to calculate the profiler file path in the
|
|
|
|
// disk since the name of this latter is randomly generated.
|
|
|
|
type profilerWrapper struct {
|
|
|
|
stopFn func()
|
|
|
|
pathFn func() string
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p profilerWrapper) Stop() {
|
|
|
|
p.stopFn()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p profilerWrapper) Path() string {
|
|
|
|
return p.pathFn()
|
|
|
|
}
|
|
|
|
|
2019-01-14 07:44:20 +01:00
|
|
|
// Returns current profile data, returns error if there is no active
|
|
|
|
// profiling in progress. Stops an active profile.
|
|
|
|
func getProfileData() ([]byte, error) {
|
|
|
|
if globalProfiler == nil {
|
|
|
|
return nil, errors.New("profiler not enabled")
|
|
|
|
}
|
|
|
|
|
|
|
|
profilerPath := globalProfiler.Path()
|
|
|
|
|
|
|
|
// Stop the profiler
|
|
|
|
globalProfiler.Stop()
|
|
|
|
|
|
|
|
profilerFile, err := os.Open(profilerPath)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return ioutil.ReadAll(profilerFile)
|
|
|
|
}
|
|
|
|
|
2016-09-02 05:13:11 +02:00
|
|
|
// Starts a profiler returns nil if profiler is not enabled, caller needs to handle this.
|
2019-02-13 13:59:36 +01:00
|
|
|
func startProfiler(profilerType, dirPath string) (minioProfiler, error) {
|
2018-09-19 01:46:35 +02:00
|
|
|
var err error
|
|
|
|
if dirPath == "" {
|
|
|
|
dirPath, err = ioutil.TempDir("", "profile")
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
var profiler interface {
|
|
|
|
Stop()
|
|
|
|
}
|
|
|
|
|
2018-10-15 20:13:19 +02:00
|
|
|
var profilerFileName string
|
|
|
|
|
|
|
|
// Enable profiler and set the name of the file that pkg/pprof
|
|
|
|
// library creates to store profiling data.
|
2018-09-19 01:46:35 +02:00
|
|
|
switch profilerType {
|
2016-09-02 05:13:11 +02:00
|
|
|
case "cpu":
|
2018-09-19 01:46:35 +02:00
|
|
|
profiler = profile.Start(profile.CPUProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
|
2018-10-15 20:13:19 +02:00
|
|
|
profilerFileName = "cpu.pprof"
|
2016-09-02 05:13:11 +02:00
|
|
|
case "mem":
|
2018-09-19 01:46:35 +02:00
|
|
|
profiler = profile.Start(profile.MemProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
|
2018-10-15 20:13:19 +02:00
|
|
|
profilerFileName = "mem.pprof"
|
2016-09-02 05:13:11 +02:00
|
|
|
case "block":
|
2018-09-19 01:46:35 +02:00
|
|
|
profiler = profile.Start(profile.BlockProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
|
2018-10-15 20:13:19 +02:00
|
|
|
profilerFileName = "block.pprof"
|
2018-09-27 06:02:05 +02:00
|
|
|
case "mutex":
|
|
|
|
profiler = profile.Start(profile.MutexProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
|
2018-10-15 20:13:19 +02:00
|
|
|
profilerFileName = "mutex.pprof"
|
2018-09-27 06:02:05 +02:00
|
|
|
case "trace":
|
|
|
|
profiler = profile.Start(profile.TraceProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
|
2018-10-15 20:13:19 +02:00
|
|
|
profilerFileName = "trace.out"
|
2016-09-02 05:13:11 +02:00
|
|
|
default:
|
2018-09-19 01:46:35 +02:00
|
|
|
return nil, errors.New("profiler type unknown")
|
2016-09-02 05:13:11 +02:00
|
|
|
}
|
2018-09-19 01:46:35 +02:00
|
|
|
|
|
|
|
return &profilerWrapper{
|
|
|
|
stopFn: profiler.Stop,
|
|
|
|
pathFn: func() string {
|
2018-10-15 20:13:19 +02:00
|
|
|
return filepath.Join(dirPath, profilerFileName)
|
2018-09-19 01:46:35 +02:00
|
|
|
},
|
|
|
|
}, nil
|
2016-09-02 05:13:11 +02:00
|
|
|
}
|
|
|
|
|
2019-02-13 13:59:36 +01:00
|
|
|
// minioProfiler - minio profiler interface.
|
|
|
|
type minioProfiler interface {
|
2018-09-19 01:46:35 +02:00
|
|
|
// Stop the profiler
|
2016-09-02 05:13:11 +02:00
|
|
|
Stop()
|
2018-09-19 01:46:35 +02:00
|
|
|
// Return the path of the profiling file
|
|
|
|
Path() string
|
2016-09-02 05:13:11 +02:00
|
|
|
}
|
|
|
|
|
2019-02-13 13:59:36 +01:00
|
|
|
// Global profiler to be used by service go-routine.
|
|
|
|
var globalProfiler minioProfiler
|
|
|
|
|
2016-09-19 19:17:46 +02:00
|
|
|
// dump the request into a string in JSON format.
|
|
|
|
func dumpRequest(r *http.Request) string {
|
2019-09-11 19:21:43 +02:00
|
|
|
header := r.Header.Clone()
|
2016-09-19 19:17:46 +02:00
|
|
|
header.Set("Host", r.Host)
|
2017-05-31 09:11:06 +02:00
|
|
|
// Replace all '%' to '%%' so that printer format parser
|
|
|
|
// to ignore URL encoded values.
|
|
|
|
rawURI := strings.Replace(r.RequestURI, "%", "%%", -1)
|
2016-09-19 19:17:46 +02:00
|
|
|
req := struct {
|
2017-05-31 09:11:06 +02:00
|
|
|
Method string `json:"method"`
|
|
|
|
RequestURI string `json:"reqURI"`
|
|
|
|
Header http.Header `json:"header"`
|
|
|
|
}{r.Method, rawURI, header}
|
|
|
|
|
|
|
|
var buffer bytes.Buffer
|
|
|
|
enc := json.NewEncoder(&buffer)
|
|
|
|
enc.SetEscapeHTML(false)
|
|
|
|
if err := enc.Encode(&req); err != nil {
|
2017-04-07 23:37:32 +02:00
|
|
|
// Upon error just return Go-syntax representation of the value
|
|
|
|
return fmt.Sprintf("%#v", req)
|
2016-09-19 19:17:46 +02:00
|
|
|
}
|
2017-05-31 09:11:06 +02:00
|
|
|
|
|
|
|
// Formatted string.
|
2019-02-13 13:59:36 +01:00
|
|
|
return strings.TrimSpace(buffer.String())
|
2016-09-19 19:17:46 +02:00
|
|
|
}
|
2017-02-27 23:59:53 +01:00
|
|
|
|
2017-03-02 23:21:30 +01:00
|
|
|
// isFile - returns whether given path is a file or not.
|
|
|
|
func isFile(path string) bool {
|
2017-10-13 12:01:15 +02:00
|
|
|
if fi, err := os.Stat(path); err == nil {
|
2017-03-02 23:21:30 +01:00
|
|
|
return fi.Mode().IsRegular()
|
|
|
|
}
|
|
|
|
|
|
|
|
return false
|
|
|
|
}
|
2017-03-16 00:30:34 +01:00
|
|
|
|
2017-03-18 19:28:41 +01:00
|
|
|
// UTCNow - returns current UTC time.
|
|
|
|
func UTCNow() time.Time {
|
|
|
|
return time.Now().UTC()
|
|
|
|
}
|
2017-10-26 19:17:07 +02:00
|
|
|
|
2019-11-21 13:52:35 +01:00
|
|
|
// UTCNowNTP - is similar in functionality to UTCNow()
|
|
|
|
// but only used when we do not wish to rely on system
|
|
|
|
// time.
|
|
|
|
func UTCNowNTP() (time.Time, error) {
|
|
|
|
// ntp server is disabled
|
|
|
|
if ntpServer == "" {
|
|
|
|
return UTCNow(), nil
|
|
|
|
}
|
|
|
|
return ntp.Time(ntpServer)
|
|
|
|
}
|
|
|
|
|
2017-12-06 02:58:09 +01:00
|
|
|
// GenETag - generate UUID based ETag
|
|
|
|
func GenETag() string {
|
|
|
|
return ToS3ETag(getMD5Hash([]byte(mustGetUUID())))
|
2017-10-26 19:17:07 +02:00
|
|
|
}
|
|
|
|
|
2017-12-06 02:58:09 +01:00
|
|
|
// ToS3ETag - return checksum to ETag
|
|
|
|
func ToS3ETag(etag string) string {
|
2017-10-26 19:17:07 +02:00
|
|
|
etag = canonicalizeETag(etag)
|
|
|
|
|
|
|
|
if !strings.HasSuffix(etag, "-1") {
|
|
|
|
// Tools like s3cmd uses ETag as checksum of data to validate.
|
|
|
|
// Append "-1" to indicate ETag is not a checksum.
|
|
|
|
etag += "-1"
|
|
|
|
}
|
|
|
|
|
|
|
|
return etag
|
|
|
|
}
|
2017-12-06 02:58:09 +01:00
|
|
|
|
2019-11-13 21:17:45 +01:00
|
|
|
type dialContext func(ctx context.Context, network, address string) (net.Conn, error)
|
|
|
|
|
|
|
|
func newCustomDialContext(dialTimeout, dialKeepAlive time.Duration) dialContext {
|
|
|
|
return func(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
|
|
dialer := &net.Dialer{
|
|
|
|
Timeout: dialTimeout,
|
|
|
|
KeepAlive: dialKeepAlive,
|
|
|
|
DualStack: true,
|
|
|
|
}
|
|
|
|
|
|
|
|
return dialer.DialContext(ctx, network, addr)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout, dialKeepAlive time.Duration) func() *http.Transport {
|
|
|
|
// For more details about various values used here refer
|
|
|
|
// https://golang.org/pkg/net/http/#Transport documentation
|
|
|
|
tr := &http.Transport{
|
|
|
|
Proxy: http.ProxyFromEnvironment,
|
|
|
|
DialContext: newCustomDialContext(dialTimeout, dialKeepAlive),
|
|
|
|
MaxIdleConnsPerHost: 256,
|
|
|
|
IdleConnTimeout: 60 * time.Second,
|
|
|
|
TLSHandshakeTimeout: 30 * time.Second,
|
|
|
|
ExpectContinueTimeout: 10 * time.Second,
|
|
|
|
TLSClientConfig: tlsConfig,
|
|
|
|
// Go net/http automatically unzip if content-type is
|
|
|
|
// gzip disable this feature, as we are always interested
|
|
|
|
// in raw stream.
|
|
|
|
DisableCompression: true,
|
|
|
|
}
|
|
|
|
return func() *http.Transport {
|
|
|
|
return tr
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-12-06 02:58:09 +01:00
|
|
|
// NewCustomHTTPTransport returns a new http configuration
|
|
|
|
// used while communicating with the cloud backends.
|
2018-02-20 21:23:37 +01:00
|
|
|
// This sets the value for MaxIdleConnsPerHost from 2 (go default)
|
2019-11-13 21:17:45 +01:00
|
|
|
// to 256.
|
2018-05-16 03:20:22 +02:00
|
|
|
func NewCustomHTTPTransport() *http.Transport {
|
2019-11-13 21:17:45 +01:00
|
|
|
return newCustomHTTPTransport(&tls.Config{
|
|
|
|
RootCAs: globalRootCAs,
|
|
|
|
}, defaultDialTimeout, defaultDialKeepAlive)()
|
2017-12-06 02:58:09 +01:00
|
|
|
}
|
2018-01-08 23:30:55 +01:00
|
|
|
|
|
|
|
// Load the json (typically from disk file).
|
2018-02-07 00:37:48 +01:00
|
|
|
func jsonLoad(r io.ReadSeeker, data interface{}) error {
|
2018-01-08 23:30:55 +01:00
|
|
|
if _, err := r.Seek(0, io.SeekStart); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return json.NewDecoder(r).Decode(data)
|
|
|
|
}
|
2018-02-07 00:37:48 +01:00
|
|
|
|
|
|
|
// Save to disk file in json format.
|
|
|
|
func jsonSave(f interface {
|
|
|
|
io.WriteSeeker
|
|
|
|
Truncate(int64) error
|
|
|
|
}, data interface{}) error {
|
|
|
|
b, err := json.Marshal(data)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if err = f.Truncate(0); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if _, err = f.Seek(0, io.SeekStart); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
_, err = f.Write(b)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
2018-03-04 23:16:45 +01:00
|
|
|
|
|
|
|
// ceilFrac takes a numerator and denominator representing a fraction
|
|
|
|
// and returns its ceiling. If denominator is 0, it returns 0 instead
|
|
|
|
// of crashing.
|
|
|
|
func ceilFrac(numerator, denominator int64) (ceil int64) {
|
|
|
|
if denominator == 0 {
|
|
|
|
// do nothing on invalid input
|
|
|
|
return
|
|
|
|
}
|
|
|
|
// Make denominator positive
|
|
|
|
if denominator < 0 {
|
|
|
|
numerator = -numerator
|
|
|
|
denominator = -denominator
|
|
|
|
}
|
|
|
|
ceil = numerator / denominator
|
|
|
|
if numerator > 0 && numerator%denominator != 0 {
|
|
|
|
ceil++
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
2018-03-14 20:01:47 +01:00
|
|
|
|
2018-03-15 02:36:54 +01:00
|
|
|
// Returns context with ReqInfo details set in the context.
|
2018-07-21 03:46:32 +02:00
|
|
|
func newContext(r *http.Request, w http.ResponseWriter, api string) context.Context {
|
2018-03-14 20:01:47 +01:00
|
|
|
vars := mux.Vars(r)
|
|
|
|
bucket := vars["bucket"]
|
|
|
|
object := vars["object"]
|
|
|
|
prefix := vars["prefix"]
|
|
|
|
|
|
|
|
if prefix != "" {
|
|
|
|
object = prefix
|
|
|
|
}
|
2018-07-02 23:40:18 +02:00
|
|
|
reqInfo := &logger.ReqInfo{
|
2019-07-01 21:22:01 +02:00
|
|
|
DeploymentID: globalDeploymentID,
|
2019-07-03 07:34:32 +02:00
|
|
|
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
2018-11-19 23:47:03 +01:00
|
|
|
RemoteHost: handlers.GetSourceIP(r),
|
2019-07-18 18:58:37 +02:00
|
|
|
Host: getHostName(r),
|
2018-11-19 23:47:03 +01:00
|
|
|
UserAgent: r.UserAgent(),
|
|
|
|
API: api,
|
|
|
|
BucketName: bucket,
|
|
|
|
ObjectName: object,
|
2018-07-02 23:40:18 +02:00
|
|
|
}
|
2019-06-29 07:09:17 +02:00
|
|
|
return logger.SetReqInfo(r.Context(), reqInfo)
|
2018-03-14 20:01:47 +01:00
|
|
|
}
|
2018-03-28 23:14:06 +02:00
|
|
|
|
2018-10-05 02:44:06 +02:00
|
|
|
// Used for registering with rest handlers (have a look at registerStorageRESTHandlers for usage example)
|
|
|
|
// If it is passed ["aaaa", "bbbb"], it returns ["aaaa", "{aaaa:.*}", "bbbb", "{bbbb:.*}"]
|
|
|
|
func restQueries(keys ...string) []string {
|
|
|
|
var accumulator []string
|
|
|
|
for _, key := range keys {
|
|
|
|
accumulator = append(accumulator, key, "{"+key+":.*}")
|
|
|
|
}
|
|
|
|
return accumulator
|
|
|
|
}
|
2019-05-13 21:25:49 +02:00
|
|
|
|
|
|
|
// Reverse the input order of a slice of string
|
|
|
|
func reverseStringSlice(input []string) {
|
|
|
|
for left, right := 0, len(input)-1; left < right; left, right = left+1, right-1 {
|
|
|
|
input[left], input[right] = input[right], input[left]
|
|
|
|
}
|
|
|
|
}
|
2019-08-09 19:02:41 +02:00
|
|
|
|
|
|
|
// lcp finds the longest common prefix of the input strings.
|
|
|
|
// It compares by bytes instead of runes (Unicode code points).
|
|
|
|
// It's up to the caller to do Unicode normalization if desired
|
|
|
|
// (e.g. see golang.org/x/text/unicode/norm).
|
|
|
|
func lcp(l []string) string {
|
|
|
|
// Special cases first
|
|
|
|
switch len(l) {
|
|
|
|
case 0:
|
|
|
|
return ""
|
|
|
|
case 1:
|
|
|
|
return l[0]
|
|
|
|
}
|
|
|
|
// LCP of min and max (lexigraphically)
|
|
|
|
// is the LCP of the whole set.
|
|
|
|
min, max := l[0], l[0]
|
|
|
|
for _, s := range l[1:] {
|
|
|
|
switch {
|
|
|
|
case s < min:
|
|
|
|
min = s
|
|
|
|
case s > max:
|
|
|
|
max = s
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for i := 0; i < len(min) && i < len(max); i++ {
|
|
|
|
if min[i] != max[i] {
|
|
|
|
return min[:i]
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// In the case where lengths are not equal but all bytes
|
|
|
|
// are equal, min is the answer ("foo" < "foobar").
|
|
|
|
return min
|
|
|
|
}
|
2019-08-14 20:43:43 +02:00
|
|
|
|
|
|
|
// Returns the mode in which MinIO is running
|
|
|
|
func getMinioMode() string {
|
|
|
|
mode := globalMinioModeFS
|
|
|
|
if globalIsDistXL {
|
|
|
|
mode = globalMinioModeDistXL
|
|
|
|
} else if globalIsXL {
|
|
|
|
mode = globalMinioModeXL
|
|
|
|
} else if globalIsGateway {
|
|
|
|
mode = globalMinioModeGatewayPrefix + globalGatewayName
|
|
|
|
}
|
|
|
|
return mode
|
|
|
|
}
|
2019-10-23 07:59:13 +02:00
|
|
|
|
2019-12-12 15:02:37 +01:00
|
|
|
func splitN(str, delim string, num int) []string {
|
|
|
|
stdSplit := strings.SplitN(str, delim, num)
|
|
|
|
retSplit := make([]string, num)
|
|
|
|
for i := 0; i < len(stdSplit); i++ {
|
|
|
|
retSplit[i] = stdSplit[i]
|
|
|
|
}
|
|
|
|
|
|
|
|
return retSplit
|
|
|
|
}
|
|
|
|
|
2019-10-23 07:59:13 +02:00
|
|
|
func iamPolicyName() string {
|
|
|
|
return globalOpenIDConfig.ClaimPrefix + iampolicy.PolicyName
|
|
|
|
}
|
2019-11-12 23:50:18 +01:00
|
|
|
|
|
|
|
func isWORMEnabled(bucket string) (Retention, bool) {
|
2019-12-17 19:13:12 +01:00
|
|
|
if isMinioMetaBucketName(bucket) {
|
|
|
|
return Retention{}, false
|
|
|
|
}
|
2019-11-12 23:50:18 +01:00
|
|
|
if globalWORMEnabled {
|
|
|
|
return Retention{}, true
|
|
|
|
}
|
2019-11-20 22:18:09 +01:00
|
|
|
return globalBucketObjectLockConfig.Get(bucket)
|
2019-11-12 23:50:18 +01:00
|
|
|
}
|