2015-04-29 11:19:51 +02:00
|
|
|
/*
|
2020-08-04 23:55:53 +02:00
|
|
|
* MinIO Cloud Storage, (C) 2015-2020 MinIO, Inc.
|
2015-04-29 11:19:51 +02:00
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2016-08-19 01:23:42 +02:00
|
|
|
package cmd
|
2015-04-23 01:28:13 +02:00
|
|
|
|
|
|
|
import (
|
2017-05-31 09:11:06 +02:00
|
|
|
"bytes"
|
2018-03-14 20:01:47 +01:00
|
|
|
"context"
|
2017-12-06 02:58:09 +01:00
|
|
|
"crypto/tls"
|
2017-04-07 23:37:32 +02:00
|
|
|
"encoding/json"
|
2016-04-21 02:35:38 +02:00
|
|
|
"encoding/xml"
|
2018-09-19 01:46:35 +02:00
|
|
|
"errors"
|
2016-08-26 09:11:53 +02:00
|
|
|
"fmt"
|
2016-04-21 02:35:38 +02:00
|
|
|
"io"
|
2018-09-06 01:47:14 +02:00
|
|
|
"io/ioutil"
|
2021-04-17 03:58:26 +02:00
|
|
|
"net"
|
2016-09-09 18:38:07 +02:00
|
|
|
"net/http"
|
2020-02-12 04:38:02 +01:00
|
|
|
"net/url"
|
2017-10-13 12:01:15 +02:00
|
|
|
"os"
|
2021-03-09 21:58:22 +01:00
|
|
|
"path"
|
2018-09-19 01:46:35 +02:00
|
|
|
"path/filepath"
|
2017-12-28 18:32:48 +01:00
|
|
|
"reflect"
|
2020-01-11 02:19:58 +01:00
|
|
|
"runtime"
|
|
|
|
"runtime/pprof"
|
|
|
|
"runtime/trace"
|
2016-04-21 02:35:38 +02:00
|
|
|
"strings"
|
2020-01-11 02:19:58 +01:00
|
|
|
"sync"
|
2017-03-18 19:28:41 +01:00
|
|
|
"time"
|
2016-09-02 05:13:11 +02:00
|
|
|
|
2020-08-10 18:02:29 +02:00
|
|
|
humanize "github.com/dustin/go-humanize"
|
|
|
|
"github.com/gorilla/mux"
|
2019-07-03 07:34:32 +02:00
|
|
|
xhttp "github.com/minio/minio/cmd/http"
|
2018-03-15 02:36:54 +01:00
|
|
|
"github.com/minio/minio/cmd/logger"
|
2021-03-01 21:31:33 +01:00
|
|
|
"github.com/minio/minio/cmd/rest"
|
2021-03-01 18:19:13 +01:00
|
|
|
"github.com/minio/minio/pkg/certs"
|
2018-07-02 23:40:18 +02:00
|
|
|
"github.com/minio/minio/pkg/handlers"
|
2020-01-22 00:49:25 +01:00
|
|
|
"github.com/minio/minio/pkg/madmin"
|
2021-02-13 03:17:52 +01:00
|
|
|
"golang.org/x/net/http2"
|
2015-04-23 01:28:13 +02:00
|
|
|
)
|
|
|
|
|
2020-03-27 05:07:39 +01:00
|
|
|
const (
|
|
|
|
slashSeparator = "/"
|
|
|
|
)
|
|
|
|
|
2018-04-10 18:36:37 +02:00
|
|
|
// IsErrIgnored returns whether given error is ignored or not.
|
|
|
|
func IsErrIgnored(err error, ignoredErrs ...error) bool {
|
|
|
|
return IsErr(err, ignoredErrs...)
|
|
|
|
}
|
|
|
|
|
|
|
|
// IsErr returns whether given error is exact error.
|
|
|
|
func IsErr(err error, errs ...error) bool {
|
|
|
|
for _, exactErr := range errs {
|
2019-09-11 19:21:43 +02:00
|
|
|
if errors.Is(err, exactErr) {
|
2018-04-10 18:36:37 +02:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2019-06-27 03:21:54 +02:00
|
|
|
func request2BucketObjectName(r *http.Request) (bucketName, objectName string) {
|
|
|
|
path, err := getResource(r.URL.Path, r.Host, globalDomainNames)
|
|
|
|
if err != nil {
|
2020-04-09 18:30:02 +02:00
|
|
|
logger.CriticalIf(GlobalContext, err)
|
2019-06-27 03:21:54 +02:00
|
|
|
}
|
|
|
|
|
2020-01-21 23:07:49 +01:00
|
|
|
return path2BucketObject(path)
|
|
|
|
}
|
2017-01-10 20:01:23 +01:00
|
|
|
|
2020-01-21 23:07:49 +01:00
|
|
|
// path2BucketObjectWithBasePath returns bucket and prefix, if any,
|
|
|
|
// of a 'path'. basePath is trimmed from the front of the 'path'.
|
|
|
|
func path2BucketObjectWithBasePath(basePath, path string) (bucket, prefix string) {
|
|
|
|
path = strings.TrimPrefix(path, basePath)
|
|
|
|
path = strings.TrimPrefix(path, SlashSeparator)
|
|
|
|
m := strings.Index(path, SlashSeparator)
|
|
|
|
if m < 0 {
|
|
|
|
return path, ""
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
2020-01-21 23:07:49 +01:00
|
|
|
return path[:m], path[m+len(SlashSeparator):]
|
|
|
|
}
|
2017-01-10 20:01:23 +01:00
|
|
|
|
2020-01-21 23:07:49 +01:00
|
|
|
func path2BucketObject(s string) (bucket, prefix string) {
|
|
|
|
return path2BucketObjectWithBasePath("", s)
|
2017-01-10 20:01:23 +01:00
|
|
|
}
|
|
|
|
|
2020-03-31 18:32:16 +02:00
|
|
|
func getReadQuorum(drive int) int {
|
2021-01-16 21:08:02 +01:00
|
|
|
return drive - getDefaultParityBlocks(drive)
|
2020-03-31 18:32:16 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
func getWriteQuorum(drive int) int {
|
2021-01-16 21:08:02 +01:00
|
|
|
parity := getDefaultParityBlocks(drive)
|
|
|
|
quorum := drive - parity
|
|
|
|
if quorum == parity {
|
2020-08-27 21:55:32 +02:00
|
|
|
quorum++
|
|
|
|
}
|
|
|
|
return quorum
|
2020-03-31 18:32:16 +02:00
|
|
|
}
|
|
|
|
|
2020-09-10 20:37:22 +02:00
|
|
|
// cloneMSS will clone a map[string]string.
|
|
|
|
// If input is nil an empty map is returned, not nil.
|
|
|
|
func cloneMSS(v map[string]string) map[string]string {
|
|
|
|
r := make(map[string]string, len(v))
|
|
|
|
for k, v := range v {
|
|
|
|
r[k] = v
|
|
|
|
}
|
|
|
|
return r
|
|
|
|
}
|
|
|
|
|
2017-01-18 21:24:34 +01:00
|
|
|
// URI scheme constants.
|
|
|
|
const (
|
|
|
|
httpScheme = "http"
|
|
|
|
httpsScheme = "https"
|
|
|
|
)
|
|
|
|
|
2018-12-13 21:09:50 +01:00
|
|
|
// nopCharsetConverter is a dummy charset convert which just copies input to output,
|
|
|
|
// it is used to ignore custom encoding charset in S3 XML body.
|
|
|
|
func nopCharsetConverter(label string, input io.Reader) (io.Reader, error) {
|
|
|
|
return input, nil
|
|
|
|
}
|
|
|
|
|
2016-04-21 02:35:38 +02:00
|
|
|
// xmlDecoder provide decoded value in xml.
|
2016-07-19 06:20:17 +02:00
|
|
|
func xmlDecoder(body io.Reader, v interface{}, size int64) error {
|
|
|
|
var lbody io.Reader
|
|
|
|
if size > 0 {
|
|
|
|
lbody = io.LimitReader(body, size)
|
|
|
|
} else {
|
|
|
|
lbody = body
|
|
|
|
}
|
|
|
|
d := xml.NewDecoder(lbody)
|
2018-12-13 21:09:50 +01:00
|
|
|
// Ignore any encoding set in the XML body
|
|
|
|
d.CharsetReader = nopCharsetConverter
|
2016-04-21 02:35:38 +02:00
|
|
|
return d.Decode(v)
|
|
|
|
}
|
|
|
|
|
2020-03-04 16:04:12 +01:00
|
|
|
// hasContentMD5 returns true if Content-MD5 header is set.
|
|
|
|
func hasContentMD5(h http.Header) bool {
|
|
|
|
_, ok := h[xhttp.ContentMD5]
|
|
|
|
return ok
|
|
|
|
}
|
|
|
|
|
2015-04-29 19:51:59 +02:00
|
|
|
/// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html
|
2015-04-29 11:19:51 +02:00
|
|
|
const (
|
2017-11-28 08:29:02 +01:00
|
|
|
// Maximum object size per PUT request is 5TB.
|
2017-03-03 19:14:17 +01:00
|
|
|
// This is a divergence from S3 limit on purpose to support
|
|
|
|
// use cases where users are going to upload large files
|
|
|
|
// using 'curl' and presigned URL.
|
2017-11-28 08:29:02 +01:00
|
|
|
globalMaxObjectSize = 5 * humanize.TiByte
|
2017-03-03 19:14:17 +01:00
|
|
|
|
|
|
|
// Minimum Part size for multipart upload is 5MiB
|
|
|
|
globalMinPartSize = 5 * humanize.MiByte
|
|
|
|
|
|
|
|
// Maximum Part size for multipart upload is 5GiB
|
|
|
|
globalMaxPartSize = 5 * humanize.GiByte
|
|
|
|
|
|
|
|
// Maximum Part ID for multipart upload is 10000
|
|
|
|
// (Acceptable values range from 1 to 10000 inclusive)
|
|
|
|
globalMaxPartID = 10000
|
2018-07-12 23:12:40 +02:00
|
|
|
|
2020-08-04 23:55:53 +02:00
|
|
|
// Default values used while communicating for gateway communication
|
2020-04-17 20:20:56 +02:00
|
|
|
defaultDialTimeout = 5 * time.Second
|
2015-04-29 11:19:51 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
// isMaxObjectSize - verify if max object size
|
2015-12-28 08:00:36 +01:00
|
|
|
func isMaxObjectSize(size int64) bool {
|
2017-03-03 19:14:17 +01:00
|
|
|
return size > globalMaxObjectSize
|
|
|
|
}
|
|
|
|
|
|
|
|
// // Check if part size is more than maximum allowed size.
|
|
|
|
func isMaxAllowedPartSize(size int64) bool {
|
|
|
|
return size > globalMaxPartSize
|
2015-04-29 11:19:51 +02:00
|
|
|
}
|
2016-02-05 12:09:31 +01:00
|
|
|
|
2016-05-08 21:06:05 +02:00
|
|
|
// Check if part size is more than or equal to minimum allowed size.
|
|
|
|
func isMinAllowedPartSize(size int64) bool {
|
2017-03-03 19:14:17 +01:00
|
|
|
return size >= globalMinPartSize
|
2016-05-08 21:06:05 +02:00
|
|
|
}
|
|
|
|
|
2016-05-24 10:52:47 +02:00
|
|
|
// isMaxPartNumber - Check if part ID is greater than the maximum allowed ID.
|
|
|
|
func isMaxPartID(partID int) bool {
|
2017-03-03 19:14:17 +01:00
|
|
|
return partID > globalMaxPartID
|
2016-05-24 10:52:47 +02:00
|
|
|
}
|
|
|
|
|
2017-12-28 18:32:48 +01:00
|
|
|
func contains(slice interface{}, elem interface{}) bool {
|
|
|
|
v := reflect.ValueOf(slice)
|
|
|
|
if v.Kind() == reflect.Slice {
|
|
|
|
for i := 0; i < v.Len(); i++ {
|
|
|
|
if v.Index(i).Interface() == elem {
|
|
|
|
return true
|
|
|
|
}
|
2016-02-05 12:09:31 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
2016-08-05 22:48:31 +02:00
|
|
|
|
2018-09-19 01:46:35 +02:00
|
|
|
// profilerWrapper is created becauses pkg/profiler doesn't
|
|
|
|
// provide any API to calculate the profiler file path in the
|
|
|
|
// disk since the name of this latter is randomly generated.
|
|
|
|
type profilerWrapper struct {
|
2020-01-22 00:49:25 +01:00
|
|
|
// Profile recorded at start of benchmark.
|
|
|
|
base []byte
|
2020-01-11 02:19:58 +01:00
|
|
|
stopFn func() ([]byte, error)
|
2020-03-04 15:58:12 +01:00
|
|
|
ext string
|
2018-09-19 01:46:35 +02:00
|
|
|
}
|
|
|
|
|
2020-01-22 00:49:25 +01:00
|
|
|
// recordBase will record the profile and store it as the base.
|
2020-03-04 15:58:12 +01:00
|
|
|
func (p *profilerWrapper) recordBase(name string, debug int) {
|
2020-01-22 00:49:25 +01:00
|
|
|
var buf bytes.Buffer
|
|
|
|
p.base = nil
|
2020-03-04 15:58:12 +01:00
|
|
|
err := pprof.Lookup(name).WriteTo(&buf, debug)
|
2020-01-22 00:49:25 +01:00
|
|
|
if err != nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
p.base = buf.Bytes()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Base returns the recorded base if any.
|
|
|
|
func (p profilerWrapper) Base() []byte {
|
|
|
|
return p.base
|
|
|
|
}
|
|
|
|
|
|
|
|
// Stop the currently running benchmark.
|
2020-01-11 02:19:58 +01:00
|
|
|
func (p profilerWrapper) Stop() ([]byte, error) {
|
|
|
|
return p.stopFn()
|
2018-09-19 01:46:35 +02:00
|
|
|
}
|
|
|
|
|
2020-03-04 15:58:12 +01:00
|
|
|
// Extension returns the extension without dot prefix.
|
|
|
|
func (p profilerWrapper) Extension() string {
|
|
|
|
return p.ext
|
|
|
|
}
|
|
|
|
|
2019-01-14 07:44:20 +01:00
|
|
|
// Returns current profile data, returns error if there is no active
|
|
|
|
// profiling in progress. Stops an active profile.
|
2020-01-11 02:19:58 +01:00
|
|
|
func getProfileData() (map[string][]byte, error) {
|
|
|
|
globalProfilerMu.Lock()
|
|
|
|
defer globalProfilerMu.Unlock()
|
|
|
|
|
|
|
|
if len(globalProfiler) == 0 {
|
2019-01-14 07:44:20 +01:00
|
|
|
return nil, errors.New("profiler not enabled")
|
|
|
|
}
|
|
|
|
|
2020-01-11 02:19:58 +01:00
|
|
|
dst := make(map[string][]byte, len(globalProfiler))
|
|
|
|
for typ, prof := range globalProfiler {
|
|
|
|
// Stop the profiler
|
|
|
|
var err error
|
|
|
|
buf, err := prof.Stop()
|
|
|
|
delete(globalProfiler, typ)
|
|
|
|
if err == nil {
|
2020-03-04 15:58:12 +01:00
|
|
|
dst[typ+"."+prof.Extension()] = buf
|
2020-01-11 02:19:58 +01:00
|
|
|
}
|
2020-01-22 00:49:25 +01:00
|
|
|
buf = prof.Base()
|
|
|
|
if len(buf) > 0 {
|
2020-03-04 15:58:12 +01:00
|
|
|
dst[typ+"-before"+"."+prof.Extension()] = buf
|
2020-01-22 00:49:25 +01:00
|
|
|
}
|
2019-01-14 07:44:20 +01:00
|
|
|
}
|
2020-01-11 02:19:58 +01:00
|
|
|
return dst, nil
|
2019-01-14 07:44:20 +01:00
|
|
|
}
|
|
|
|
|
2020-01-22 00:49:25 +01:00
|
|
|
func setDefaultProfilerRates() {
|
|
|
|
runtime.MemProfileRate = 4096 // 512K -> 4K - Must be constant throughout application lifetime.
|
|
|
|
runtime.SetMutexProfileFraction(0) // Disable until needed
|
|
|
|
runtime.SetBlockProfileRate(0) // Disable until needed
|
|
|
|
}
|
|
|
|
|
2016-09-02 05:13:11 +02:00
|
|
|
// Starts a profiler returns nil if profiler is not enabled, caller needs to handle this.
|
2020-01-11 02:19:58 +01:00
|
|
|
func startProfiler(profilerType string) (minioProfiler, error) {
|
|
|
|
var prof profilerWrapper
|
2020-03-04 15:58:12 +01:00
|
|
|
prof.ext = "pprof"
|
2018-10-15 20:13:19 +02:00
|
|
|
// Enable profiler and set the name of the file that pkg/pprof
|
|
|
|
// library creates to store profiling data.
|
2020-01-22 00:49:25 +01:00
|
|
|
switch madmin.ProfilerType(profilerType) {
|
|
|
|
case madmin.ProfilerCPU:
|
2020-01-11 02:19:58 +01:00
|
|
|
dirPath, err := ioutil.TempDir("", "profile")
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
fn := filepath.Join(dirPath, "cpu.out")
|
|
|
|
f, err := os.Create(fn)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
err = pprof.StartCPUProfile(f)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
prof.stopFn = func() ([]byte, error) {
|
|
|
|
pprof.StopCPUProfile()
|
|
|
|
err := f.Close()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
defer os.RemoveAll(dirPath)
|
|
|
|
return ioutil.ReadFile(fn)
|
|
|
|
}
|
2020-01-22 00:49:25 +01:00
|
|
|
case madmin.ProfilerMEM:
|
|
|
|
runtime.GC()
|
2020-03-04 15:58:12 +01:00
|
|
|
prof.recordBase("heap", 0)
|
2020-01-11 02:19:58 +01:00
|
|
|
prof.stopFn = func() ([]byte, error) {
|
2020-01-22 00:49:25 +01:00
|
|
|
runtime.GC()
|
2020-01-11 02:19:58 +01:00
|
|
|
var buf bytes.Buffer
|
|
|
|
err := pprof.Lookup("heap").WriteTo(&buf, 0)
|
|
|
|
return buf.Bytes(), err
|
|
|
|
}
|
2020-01-22 00:49:25 +01:00
|
|
|
case madmin.ProfilerBlock:
|
2021-02-27 18:22:14 +01:00
|
|
|
runtime.SetBlockProfileRate(100)
|
2020-01-11 02:19:58 +01:00
|
|
|
prof.stopFn = func() ([]byte, error) {
|
|
|
|
var buf bytes.Buffer
|
|
|
|
err := pprof.Lookup("block").WriteTo(&buf, 0)
|
2020-01-22 00:49:25 +01:00
|
|
|
runtime.SetBlockProfileRate(0)
|
2020-01-11 02:19:58 +01:00
|
|
|
return buf.Bytes(), err
|
|
|
|
}
|
2020-01-22 00:49:25 +01:00
|
|
|
case madmin.ProfilerMutex:
|
2020-03-04 15:58:12 +01:00
|
|
|
prof.recordBase("mutex", 0)
|
2020-01-11 02:19:58 +01:00
|
|
|
runtime.SetMutexProfileFraction(1)
|
|
|
|
prof.stopFn = func() ([]byte, error) {
|
|
|
|
var buf bytes.Buffer
|
|
|
|
err := pprof.Lookup("mutex").WriteTo(&buf, 0)
|
2020-01-22 00:49:25 +01:00
|
|
|
runtime.SetMutexProfileFraction(0)
|
|
|
|
return buf.Bytes(), err
|
|
|
|
}
|
|
|
|
case madmin.ProfilerThreads:
|
2020-03-04 15:58:12 +01:00
|
|
|
prof.recordBase("threadcreate", 0)
|
2020-01-22 00:49:25 +01:00
|
|
|
prof.stopFn = func() ([]byte, error) {
|
|
|
|
var buf bytes.Buffer
|
|
|
|
err := pprof.Lookup("threadcreate").WriteTo(&buf, 0)
|
2020-01-11 02:19:58 +01:00
|
|
|
return buf.Bytes(), err
|
|
|
|
}
|
2020-03-04 15:58:12 +01:00
|
|
|
case madmin.ProfilerGoroutines:
|
|
|
|
prof.ext = "txt"
|
2020-03-06 22:22:47 +01:00
|
|
|
prof.recordBase("goroutine", 1)
|
2020-03-04 15:58:12 +01:00
|
|
|
prof.stopFn = func() ([]byte, error) {
|
|
|
|
var buf bytes.Buffer
|
2020-03-06 22:22:47 +01:00
|
|
|
err := pprof.Lookup("goroutine").WriteTo(&buf, 1)
|
2020-03-04 15:58:12 +01:00
|
|
|
return buf.Bytes(), err
|
|
|
|
}
|
2020-01-22 00:49:25 +01:00
|
|
|
case madmin.ProfilerTrace:
|
2020-01-11 02:19:58 +01:00
|
|
|
dirPath, err := ioutil.TempDir("", "profile")
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
fn := filepath.Join(dirPath, "trace.out")
|
|
|
|
f, err := os.Create(fn)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
err = trace.Start(f)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2020-03-04 15:58:12 +01:00
|
|
|
prof.ext = "trace"
|
2020-01-11 02:19:58 +01:00
|
|
|
prof.stopFn = func() ([]byte, error) {
|
|
|
|
trace.Stop()
|
|
|
|
err := f.Close()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
defer os.RemoveAll(dirPath)
|
|
|
|
return ioutil.ReadFile(fn)
|
|
|
|
}
|
2016-09-02 05:13:11 +02:00
|
|
|
default:
|
2018-09-19 01:46:35 +02:00
|
|
|
return nil, errors.New("profiler type unknown")
|
2016-09-02 05:13:11 +02:00
|
|
|
}
|
2018-09-19 01:46:35 +02:00
|
|
|
|
2020-01-11 02:19:58 +01:00
|
|
|
return prof, nil
|
2016-09-02 05:13:11 +02:00
|
|
|
}
|
|
|
|
|
2019-02-13 13:59:36 +01:00
|
|
|
// minioProfiler - minio profiler interface.
|
|
|
|
type minioProfiler interface {
|
2020-01-22 00:49:25 +01:00
|
|
|
// Return base profile. 'nil' if none.
|
|
|
|
Base() []byte
|
2018-09-19 01:46:35 +02:00
|
|
|
// Stop the profiler
|
2020-01-11 02:19:58 +01:00
|
|
|
Stop() ([]byte, error)
|
2020-03-04 15:58:12 +01:00
|
|
|
// Return extension of profile
|
|
|
|
Extension() string
|
2016-09-02 05:13:11 +02:00
|
|
|
}
|
|
|
|
|
2019-02-13 13:59:36 +01:00
|
|
|
// Global profiler to be used by service go-routine.
|
2020-01-11 02:19:58 +01:00
|
|
|
var globalProfiler map[string]minioProfiler
|
|
|
|
var globalProfilerMu sync.Mutex
|
2019-02-13 13:59:36 +01:00
|
|
|
|
2016-09-19 19:17:46 +02:00
|
|
|
// dump the request into a string in JSON format.
|
|
|
|
func dumpRequest(r *http.Request) string {
|
2019-09-11 19:21:43 +02:00
|
|
|
header := r.Header.Clone()
|
2016-09-19 19:17:46 +02:00
|
|
|
header.Set("Host", r.Host)
|
2017-05-31 09:11:06 +02:00
|
|
|
// Replace all '%' to '%%' so that printer format parser
|
|
|
|
// to ignore URL encoded values.
|
|
|
|
rawURI := strings.Replace(r.RequestURI, "%", "%%", -1)
|
2016-09-19 19:17:46 +02:00
|
|
|
req := struct {
|
2017-05-31 09:11:06 +02:00
|
|
|
Method string `json:"method"`
|
|
|
|
RequestURI string `json:"reqURI"`
|
|
|
|
Header http.Header `json:"header"`
|
|
|
|
}{r.Method, rawURI, header}
|
|
|
|
|
|
|
|
var buffer bytes.Buffer
|
|
|
|
enc := json.NewEncoder(&buffer)
|
|
|
|
enc.SetEscapeHTML(false)
|
|
|
|
if err := enc.Encode(&req); err != nil {
|
2017-04-07 23:37:32 +02:00
|
|
|
// Upon error just return Go-syntax representation of the value
|
|
|
|
return fmt.Sprintf("%#v", req)
|
2016-09-19 19:17:46 +02:00
|
|
|
}
|
2017-05-31 09:11:06 +02:00
|
|
|
|
|
|
|
// Formatted string.
|
2019-02-13 13:59:36 +01:00
|
|
|
return strings.TrimSpace(buffer.String())
|
2016-09-19 19:17:46 +02:00
|
|
|
}
|
2017-02-27 23:59:53 +01:00
|
|
|
|
2017-03-02 23:21:30 +01:00
|
|
|
// isFile - returns whether given path is a file or not.
|
|
|
|
func isFile(path string) bool {
|
2017-10-13 12:01:15 +02:00
|
|
|
if fi, err := os.Stat(path); err == nil {
|
2017-03-02 23:21:30 +01:00
|
|
|
return fi.Mode().IsRegular()
|
|
|
|
}
|
|
|
|
|
|
|
|
return false
|
|
|
|
}
|
2017-03-16 00:30:34 +01:00
|
|
|
|
2017-03-18 19:28:41 +01:00
|
|
|
// UTCNow - returns current UTC time.
|
|
|
|
func UTCNow() time.Time {
|
|
|
|
return time.Now().UTC()
|
|
|
|
}
|
2017-10-26 19:17:07 +02:00
|
|
|
|
2017-12-06 02:58:09 +01:00
|
|
|
// GenETag - generate UUID based ETag
|
|
|
|
func GenETag() string {
|
|
|
|
return ToS3ETag(getMD5Hash([]byte(mustGetUUID())))
|
2017-10-26 19:17:07 +02:00
|
|
|
}
|
|
|
|
|
2017-12-06 02:58:09 +01:00
|
|
|
// ToS3ETag - return checksum to ETag
|
|
|
|
func ToS3ETag(etag string) string {
|
2017-10-26 19:17:07 +02:00
|
|
|
etag = canonicalizeETag(etag)
|
|
|
|
|
|
|
|
if !strings.HasSuffix(etag, "-1") {
|
|
|
|
// Tools like s3cmd uses ETag as checksum of data to validate.
|
|
|
|
// Append "-1" to indicate ETag is not a checksum.
|
|
|
|
etag += "-1"
|
|
|
|
}
|
|
|
|
|
|
|
|
return etag
|
|
|
|
}
|
2017-12-06 02:58:09 +01:00
|
|
|
|
2020-12-22 06:42:38 +01:00
|
|
|
func newInternodeHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) func() http.RoundTripper {
|
2020-08-04 23:55:53 +02:00
|
|
|
// For more details about various values used here refer
|
|
|
|
// https://golang.org/pkg/net/http/#Transport documentation
|
|
|
|
tr := &http.Transport{
|
|
|
|
Proxy: http.ProxyFromEnvironment,
|
2020-10-16 23:49:05 +02:00
|
|
|
DialContext: xhttp.DialContextWithDNSCache(globalDNSCache, xhttp.NewInternodeDialContext(dialTimeout)),
|
2020-10-12 23:19:46 +02:00
|
|
|
MaxIdleConnsPerHost: 1024,
|
[feat]: change erasure coding default block size from 10MiB to 1MiB (#11721)
major performance improvements in range GETs to avoid large
read amplification when ranges are tiny and random
```
-------------------
Operation: GET
Operations: 142014 -> 339421
Duration: 4m50s -> 4m56s
* Average: +139.41% (+1177.3 MiB/s) throughput, +139.11% (+658.4) obj/s
* Fastest: +125.24% (+1207.4 MiB/s) throughput, +132.32% (+612.9) obj/s
* 50% Median: +139.06% (+1175.7 MiB/s) throughput, +133.46% (+660.9) obj/s
* Slowest: +203.40% (+1267.9 MiB/s) throughput, +198.59% (+753.5) obj/s
```
TTFB from 10MiB BlockSize
```
* First Access TTFB: Avg: 81ms, Median: 61ms, Best: 20ms, Worst: 2.056s
```
TTFB from 1MiB BlockSize
```
* First Access TTFB: Avg: 22ms, Median: 21ms, Best: 8ms, Worst: 91ms
```
Full object reads however do see a slight change which won't be
noticeable in real world, so not doing any comparisons
TTFB still had improvements with full object reads with 1MiB
```
* First Access TTFB: Avg: 68ms, Median: 35ms, Best: 11ms, Worst: 1.16s
```
v/s
TTFB with 10MiB
```
* First Access TTFB: Avg: 388ms, Median: 98ms, Best: 20ms, Worst: 4.156s
```
This change should affect all new uploads, previous uploads should
continue to work with business as usual. But dramatic improvements can
be seen with these changes.
2021-03-06 23:09:34 +01:00
|
|
|
WriteBufferSize: 32 << 10, // 32KiB moving up from 4KiB default
|
|
|
|
ReadBufferSize: 32 << 10, // 32KiB moving up from 4KiB default
|
2020-10-12 23:19:46 +02:00
|
|
|
IdleConnTimeout: 15 * time.Second,
|
2020-08-04 23:55:53 +02:00
|
|
|
ResponseHeaderTimeout: 3 * time.Minute, // Set conservative timeouts for MinIO internode.
|
2020-09-25 05:10:07 +02:00
|
|
|
TLSHandshakeTimeout: 15 * time.Second,
|
|
|
|
ExpectContinueTimeout: 15 * time.Second,
|
2020-08-04 23:55:53 +02:00
|
|
|
TLSClientConfig: tlsConfig,
|
|
|
|
// Go net/http automatically unzip if content-type is
|
|
|
|
// gzip disable this feature, as we are always interested
|
|
|
|
// in raw stream.
|
|
|
|
DisableCompression: true,
|
|
|
|
}
|
2020-08-10 18:02:29 +02:00
|
|
|
|
2021-02-12 00:53:04 +01:00
|
|
|
// https://github.com/golang/go/issues/23559
|
|
|
|
// https://github.com/golang/go/issues/42534
|
|
|
|
// https://github.com/golang/go/issues/43989
|
|
|
|
// https://github.com/golang/go/issues/33425
|
|
|
|
// https://github.com/golang/go/issues/29246
|
|
|
|
// if tlsConfig != nil {
|
|
|
|
// trhttp2, _ := http2.ConfigureTransports(tr)
|
|
|
|
// if trhttp2 != nil {
|
|
|
|
// // ReadIdleTimeout is the timeout after which a health check using ping
|
|
|
|
// // frame will be carried out if no frame is received on the
|
|
|
|
// // connection. 5 minutes is sufficient time for any idle connection.
|
|
|
|
// trhttp2.ReadIdleTimeout = 5 * time.Minute
|
|
|
|
// // PingTimeout is the timeout after which the connection will be closed
|
|
|
|
// // if a response to Ping is not received.
|
|
|
|
// trhttp2.PingTimeout = dialTimeout
|
|
|
|
// // DisableCompression, if true, prevents the Transport from
|
|
|
|
// // requesting compression with an "Accept-Encoding: gzip"
|
|
|
|
// trhttp2.DisableCompression = true
|
|
|
|
// }
|
|
|
|
// }
|
2020-08-10 18:02:29 +02:00
|
|
|
|
2020-12-22 06:42:38 +01:00
|
|
|
return func() http.RoundTripper {
|
2020-08-04 23:55:53 +02:00
|
|
|
return tr
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-10-12 23:19:46 +02:00
|
|
|
// Used by only proxied requests, specifically only supports HTTP/1.1
|
|
|
|
func newCustomHTTPProxyTransport(tlsConfig *tls.Config, dialTimeout time.Duration) func() *http.Transport {
|
2020-10-08 21:32:32 +02:00
|
|
|
// For more details about various values used here refer
|
|
|
|
// https://golang.org/pkg/net/http/#Transport documentation
|
|
|
|
tr := &http.Transport{
|
|
|
|
Proxy: http.ProxyFromEnvironment,
|
2020-10-16 23:49:05 +02:00
|
|
|
DialContext: xhttp.DialContextWithDNSCache(globalDNSCache, xhttp.NewInternodeDialContext(dialTimeout)),
|
2020-10-12 23:19:46 +02:00
|
|
|
MaxIdleConnsPerHost: 1024,
|
[feat]: change erasure coding default block size from 10MiB to 1MiB (#11721)
major performance improvements in range GETs to avoid large
read amplification when ranges are tiny and random
```
-------------------
Operation: GET
Operations: 142014 -> 339421
Duration: 4m50s -> 4m56s
* Average: +139.41% (+1177.3 MiB/s) throughput, +139.11% (+658.4) obj/s
* Fastest: +125.24% (+1207.4 MiB/s) throughput, +132.32% (+612.9) obj/s
* 50% Median: +139.06% (+1175.7 MiB/s) throughput, +133.46% (+660.9) obj/s
* Slowest: +203.40% (+1267.9 MiB/s) throughput, +198.59% (+753.5) obj/s
```
TTFB from 10MiB BlockSize
```
* First Access TTFB: Avg: 81ms, Median: 61ms, Best: 20ms, Worst: 2.056s
```
TTFB from 1MiB BlockSize
```
* First Access TTFB: Avg: 22ms, Median: 21ms, Best: 8ms, Worst: 91ms
```
Full object reads however do see a slight change which won't be
noticeable in real world, so not doing any comparisons
TTFB still had improvements with full object reads with 1MiB
```
* First Access TTFB: Avg: 68ms, Median: 35ms, Best: 11ms, Worst: 1.16s
```
v/s
TTFB with 10MiB
```
* First Access TTFB: Avg: 388ms, Median: 98ms, Best: 20ms, Worst: 4.156s
```
This change should affect all new uploads, previous uploads should
continue to work with business as usual. But dramatic improvements can
be seen with these changes.
2021-03-06 23:09:34 +01:00
|
|
|
WriteBufferSize: 16 << 10, // 16KiB moving up from 4KiB default
|
|
|
|
ReadBufferSize: 16 << 10, // 16KiB moving up from 4KiB default
|
2020-10-12 23:19:46 +02:00
|
|
|
IdleConnTimeout: 15 * time.Second,
|
|
|
|
ResponseHeaderTimeout: 30 * time.Minute, // Set larger timeouts for proxied requests.
|
2020-10-08 21:32:32 +02:00
|
|
|
TLSHandshakeTimeout: 10 * time.Second,
|
|
|
|
ExpectContinueTimeout: 10 * time.Second,
|
|
|
|
TLSClientConfig: tlsConfig,
|
|
|
|
// Go net/http automatically unzip if content-type is
|
|
|
|
// gzip disable this feature, as we are always interested
|
|
|
|
// in raw stream.
|
|
|
|
DisableCompression: true,
|
|
|
|
}
|
|
|
|
|
|
|
|
return func() *http.Transport {
|
|
|
|
return tr
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-02-13 03:17:52 +01:00
|
|
|
func newCustomHTTPTransportWithHTTP2(tlsConfig *tls.Config, dialTimeout time.Duration) func() *http.Transport {
|
|
|
|
// For more details about various values used here refer
|
|
|
|
// https://golang.org/pkg/net/http/#Transport documentation
|
|
|
|
tr := &http.Transport{
|
|
|
|
Proxy: http.ProxyFromEnvironment,
|
|
|
|
DialContext: xhttp.DialContextWithDNSCache(globalDNSCache, xhttp.NewInternodeDialContext(dialTimeout)),
|
|
|
|
MaxIdleConnsPerHost: 1024,
|
|
|
|
IdleConnTimeout: 15 * time.Second,
|
|
|
|
ResponseHeaderTimeout: 1 * time.Minute,
|
|
|
|
TLSHandshakeTimeout: 10 * time.Second,
|
|
|
|
ExpectContinueTimeout: 10 * time.Second,
|
|
|
|
TLSClientConfig: tlsConfig,
|
|
|
|
// Go net/http automatically unzip if content-type is
|
|
|
|
// gzip disable this feature, as we are always interested
|
|
|
|
// in raw stream.
|
|
|
|
DisableCompression: true,
|
|
|
|
}
|
|
|
|
|
|
|
|
if tlsConfig != nil {
|
|
|
|
trhttp2, _ := http2.ConfigureTransports(tr)
|
|
|
|
if trhttp2 != nil {
|
|
|
|
// ReadIdleTimeout is the timeout after which a health check using ping
|
|
|
|
// frame will be carried out if no frame is received on the
|
|
|
|
// connection. 5 minutes is sufficient time for any idle connection.
|
|
|
|
trhttp2.ReadIdleTimeout = 5 * time.Minute
|
|
|
|
// PingTimeout is the timeout after which the connection will be closed
|
|
|
|
// if a response to Ping is not received.
|
|
|
|
trhttp2.PingTimeout = dialTimeout
|
|
|
|
// DisableCompression, if true, prevents the Transport from
|
|
|
|
// requesting compression with an "Accept-Encoding: gzip"
|
|
|
|
trhttp2.DisableCompression = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return func() *http.Transport {
|
|
|
|
return tr
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-04-17 20:20:56 +02:00
|
|
|
func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) func() *http.Transport {
|
2019-11-13 21:17:45 +01:00
|
|
|
// For more details about various values used here refer
|
|
|
|
// https://golang.org/pkg/net/http/#Transport documentation
|
|
|
|
tr := &http.Transport{
|
2020-07-10 21:08:14 +02:00
|
|
|
Proxy: http.ProxyFromEnvironment,
|
2020-10-16 23:49:05 +02:00
|
|
|
DialContext: xhttp.DialContextWithDNSCache(globalDNSCache, xhttp.NewInternodeDialContext(dialTimeout)),
|
2020-10-12 23:19:46 +02:00
|
|
|
MaxIdleConnsPerHost: 1024,
|
[feat]: change erasure coding default block size from 10MiB to 1MiB (#11721)
major performance improvements in range GETs to avoid large
read amplification when ranges are tiny and random
```
-------------------
Operation: GET
Operations: 142014 -> 339421
Duration: 4m50s -> 4m56s
* Average: +139.41% (+1177.3 MiB/s) throughput, +139.11% (+658.4) obj/s
* Fastest: +125.24% (+1207.4 MiB/s) throughput, +132.32% (+612.9) obj/s
* 50% Median: +139.06% (+1175.7 MiB/s) throughput, +133.46% (+660.9) obj/s
* Slowest: +203.40% (+1267.9 MiB/s) throughput, +198.59% (+753.5) obj/s
```
TTFB from 10MiB BlockSize
```
* First Access TTFB: Avg: 81ms, Median: 61ms, Best: 20ms, Worst: 2.056s
```
TTFB from 1MiB BlockSize
```
* First Access TTFB: Avg: 22ms, Median: 21ms, Best: 8ms, Worst: 91ms
```
Full object reads however do see a slight change which won't be
noticeable in real world, so not doing any comparisons
TTFB still had improvements with full object reads with 1MiB
```
* First Access TTFB: Avg: 68ms, Median: 35ms, Best: 11ms, Worst: 1.16s
```
v/s
TTFB with 10MiB
```
* First Access TTFB: Avg: 388ms, Median: 98ms, Best: 20ms, Worst: 4.156s
```
This change should affect all new uploads, previous uploads should
continue to work with business as usual. But dramatic improvements can
be seen with these changes.
2021-03-06 23:09:34 +01:00
|
|
|
WriteBufferSize: 16 << 10, // 16KiB moving up from 4KiB default
|
|
|
|
ReadBufferSize: 16 << 10, // 16KiB moving up from 4KiB default
|
2020-10-12 23:19:46 +02:00
|
|
|
IdleConnTimeout: 15 * time.Second,
|
2020-03-22 06:10:13 +01:00
|
|
|
ResponseHeaderTimeout: 3 * time.Minute, // Set conservative timeouts for MinIO internode.
|
2020-01-10 11:35:06 +01:00
|
|
|
TLSHandshakeTimeout: 10 * time.Second,
|
2020-02-01 04:01:55 +01:00
|
|
|
ExpectContinueTimeout: 10 * time.Second,
|
2019-11-13 21:17:45 +01:00
|
|
|
TLSClientConfig: tlsConfig,
|
|
|
|
// Go net/http automatically unzip if content-type is
|
|
|
|
// gzip disable this feature, as we are always interested
|
|
|
|
// in raw stream.
|
|
|
|
DisableCompression: true,
|
|
|
|
}
|
2020-08-10 18:02:29 +02:00
|
|
|
|
2021-02-12 00:53:04 +01:00
|
|
|
// https://github.com/golang/go/issues/23559
|
|
|
|
// https://github.com/golang/go/issues/42534
|
|
|
|
// https://github.com/golang/go/issues/43989
|
|
|
|
// https://github.com/golang/go/issues/33425
|
|
|
|
// https://github.com/golang/go/issues/29246
|
|
|
|
// if tlsConfig != nil {
|
|
|
|
// trhttp2, _ := http2.ConfigureTransports(tr)
|
|
|
|
// if trhttp2 != nil {
|
|
|
|
// // ReadIdleTimeout is the timeout after which a health check using ping
|
|
|
|
// // frame will be carried out if no frame is received on the
|
|
|
|
// // connection. 5 minutes is sufficient time for any idle connection.
|
|
|
|
// trhttp2.ReadIdleTimeout = 5 * time.Minute
|
|
|
|
// // PingTimeout is the timeout after which the connection will be closed
|
|
|
|
// // if a response to Ping is not received.
|
|
|
|
// trhttp2.PingTimeout = dialTimeout
|
|
|
|
// // DisableCompression, if true, prevents the Transport from
|
|
|
|
// // requesting compression with an "Accept-Encoding: gzip"
|
|
|
|
// trhttp2.DisableCompression = true
|
|
|
|
// }
|
|
|
|
// }
|
2020-08-10 18:02:29 +02:00
|
|
|
|
2019-11-13 21:17:45 +01:00
|
|
|
return func() *http.Transport {
|
|
|
|
return tr
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-03-01 18:19:13 +01:00
|
|
|
// NewGatewayHTTPTransportWithClientCerts returns a new http configuration
|
|
|
|
// used while communicating with the cloud backends.
|
|
|
|
func NewGatewayHTTPTransportWithClientCerts(clientCert, clientKey string) *http.Transport {
|
|
|
|
transport := newGatewayHTTPTransport(1 * time.Minute)
|
|
|
|
if clientCert != "" && clientKey != "" {
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
|
|
defer cancel()
|
|
|
|
c, err := certs.NewManager(ctx, clientCert, clientKey, tls.LoadX509KeyPair)
|
|
|
|
if err != nil {
|
|
|
|
logger.LogIf(ctx, fmt.Errorf("failed to load client key and cert, please check your endpoint configuration: %s",
|
|
|
|
err.Error()))
|
|
|
|
}
|
|
|
|
if c != nil {
|
|
|
|
transport.TLSClientConfig.GetClientCertificate = c.GetClientCertificate
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return transport
|
|
|
|
}
|
|
|
|
|
2020-03-22 06:10:13 +01:00
|
|
|
// NewGatewayHTTPTransport returns a new http configuration
|
2017-12-06 02:58:09 +01:00
|
|
|
// used while communicating with the cloud backends.
|
2020-03-22 06:10:13 +01:00
|
|
|
func NewGatewayHTTPTransport() *http.Transport {
|
2020-07-07 21:19:57 +02:00
|
|
|
return newGatewayHTTPTransport(1 * time.Minute)
|
|
|
|
}
|
|
|
|
|
|
|
|
func newGatewayHTTPTransport(timeout time.Duration) *http.Transport {
|
2020-03-22 06:10:13 +01:00
|
|
|
tr := newCustomHTTPTransport(&tls.Config{
|
2019-11-13 21:17:45 +01:00
|
|
|
RootCAs: globalRootCAs,
|
2020-04-17 20:20:56 +02:00
|
|
|
}, defaultDialTimeout)()
|
|
|
|
|
2020-10-12 23:19:46 +02:00
|
|
|
// Customize response header timeout for gateway transport.
|
2020-07-07 21:19:57 +02:00
|
|
|
tr.ResponseHeaderTimeout = timeout
|
2020-03-22 06:10:13 +01:00
|
|
|
return tr
|
2017-12-06 02:58:09 +01:00
|
|
|
}
|
2018-01-08 23:30:55 +01:00
|
|
|
|
2021-04-17 03:58:26 +02:00
|
|
|
// NewRemoteTargetHTTPTransport returns a new http configuration
|
|
|
|
// used while communicating with the remote replication targets.
|
|
|
|
func NewRemoteTargetHTTPTransport() *http.Transport {
|
|
|
|
// For more details about various values used here refer
|
|
|
|
// https://golang.org/pkg/net/http/#Transport documentation
|
|
|
|
tr := &http.Transport{
|
|
|
|
Proxy: http.ProxyFromEnvironment,
|
|
|
|
DialContext: (&net.Dialer{
|
|
|
|
Timeout: 15 * time.Second,
|
|
|
|
KeepAlive: 30 * time.Second,
|
|
|
|
}).DialContext,
|
|
|
|
MaxIdleConnsPerHost: 1024,
|
|
|
|
WriteBufferSize: 16 << 10, // 16KiB moving up from 4KiB default
|
|
|
|
ReadBufferSize: 16 << 10, // 16KiB moving up from 4KiB default
|
|
|
|
IdleConnTimeout: 15 * time.Second,
|
|
|
|
TLSHandshakeTimeout: 5 * time.Second,
|
|
|
|
ExpectContinueTimeout: 5 * time.Second,
|
|
|
|
TLSClientConfig: &tls.Config{
|
|
|
|
RootCAs: globalRootCAs,
|
|
|
|
},
|
|
|
|
// Go net/http automatically unzip if content-type is
|
|
|
|
// gzip disable this feature, as we are always interested
|
|
|
|
// in raw stream.
|
|
|
|
DisableCompression: true,
|
|
|
|
}
|
|
|
|
return tr
|
|
|
|
}
|
|
|
|
|
2018-01-08 23:30:55 +01:00
|
|
|
// Load the json (typically from disk file).
|
2018-02-07 00:37:48 +01:00
|
|
|
func jsonLoad(r io.ReadSeeker, data interface{}) error {
|
2018-01-08 23:30:55 +01:00
|
|
|
if _, err := r.Seek(0, io.SeekStart); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return json.NewDecoder(r).Decode(data)
|
|
|
|
}
|
2018-02-07 00:37:48 +01:00
|
|
|
|
|
|
|
// Save to disk file in json format.
|
|
|
|
func jsonSave(f interface {
|
|
|
|
io.WriteSeeker
|
|
|
|
Truncate(int64) error
|
|
|
|
}, data interface{}) error {
|
|
|
|
b, err := json.Marshal(data)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if err = f.Truncate(0); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if _, err = f.Seek(0, io.SeekStart); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
_, err = f.Write(b)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
2018-03-04 23:16:45 +01:00
|
|
|
|
|
|
|
// ceilFrac takes a numerator and denominator representing a fraction
|
|
|
|
// and returns its ceiling. If denominator is 0, it returns 0 instead
|
|
|
|
// of crashing.
|
|
|
|
func ceilFrac(numerator, denominator int64) (ceil int64) {
|
|
|
|
if denominator == 0 {
|
|
|
|
// do nothing on invalid input
|
|
|
|
return
|
|
|
|
}
|
|
|
|
// Make denominator positive
|
|
|
|
if denominator < 0 {
|
|
|
|
numerator = -numerator
|
|
|
|
denominator = -denominator
|
|
|
|
}
|
|
|
|
ceil = numerator / denominator
|
|
|
|
if numerator > 0 && numerator%denominator != 0 {
|
|
|
|
ceil++
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
2018-03-14 20:01:47 +01:00
|
|
|
|
2021-04-16 01:32:13 +02:00
|
|
|
// pathClean is like path.Clean but does not return "." for
|
|
|
|
// empty inputs, instead returns "empty" as is.
|
|
|
|
func pathClean(p string) string {
|
|
|
|
cp := path.Clean(p)
|
|
|
|
if cp == "." {
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
return cp
|
|
|
|
}
|
|
|
|
|
2021-03-11 22:57:03 +01:00
|
|
|
func trimLeadingSlash(ep string) string {
|
2021-03-09 21:58:22 +01:00
|
|
|
if len(ep) > 0 && ep[0] == '/' {
|
|
|
|
// Path ends with '/' preserve it
|
2021-03-11 22:57:03 +01:00
|
|
|
if ep[len(ep)-1] == '/' && len(ep) > 1 {
|
2021-03-09 21:58:22 +01:00
|
|
|
ep = path.Clean(ep)
|
|
|
|
ep += slashSeparator
|
|
|
|
} else {
|
|
|
|
ep = path.Clean(ep)
|
|
|
|
}
|
|
|
|
ep = ep[1:]
|
2020-02-12 04:38:02 +01:00
|
|
|
}
|
2021-03-11 22:57:03 +01:00
|
|
|
return ep
|
|
|
|
}
|
|
|
|
|
|
|
|
// unescapeGeneric is similar to url.PathUnescape or url.QueryUnescape
|
|
|
|
// depending on input, additionally also handles situations such as
|
|
|
|
// `//` are normalized as `/`, also removes any `/` prefix before
|
|
|
|
// returning.
|
|
|
|
func unescapeGeneric(p string, escapeFn func(string) (string, error)) (string, error) {
|
|
|
|
ep, err := escapeFn(p)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
return trimLeadingSlash(ep), nil
|
2021-03-09 21:58:22 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// unescapePath is similar to unescapeGeneric but for specifically
|
|
|
|
// path unescaping.
|
|
|
|
func unescapePath(p string) (string, error) {
|
|
|
|
return unescapeGeneric(p, url.PathUnescape)
|
|
|
|
}
|
|
|
|
|
|
|
|
// similar to unescapeGeneric but never returns any error if the unescaping
|
|
|
|
// fails, returns the input as is in such occasion, not meant to be
|
|
|
|
// used where strict validation is expected.
|
|
|
|
func likelyUnescapeGeneric(p string, escapeFn func(string) (string, error)) string {
|
|
|
|
ep, err := unescapeGeneric(p, escapeFn)
|
2020-02-12 04:38:02 +01:00
|
|
|
if err != nil {
|
2021-03-09 21:58:22 +01:00
|
|
|
return p
|
2020-02-12 04:38:02 +01:00
|
|
|
}
|
2021-03-09 21:58:22 +01:00
|
|
|
return ep
|
|
|
|
}
|
|
|
|
|
|
|
|
// Returns context with ReqInfo details set in the context.
|
|
|
|
func newContext(r *http.Request, w http.ResponseWriter, api string) context.Context {
|
|
|
|
vars := mux.Vars(r)
|
|
|
|
bucket := vars["bucket"]
|
|
|
|
object := likelyUnescapeGeneric(vars["object"], url.PathUnescape)
|
|
|
|
prefix := likelyUnescapeGeneric(vars["prefix"], url.QueryUnescape)
|
2018-03-14 20:01:47 +01:00
|
|
|
if prefix != "" {
|
|
|
|
object = prefix
|
|
|
|
}
|
2018-07-02 23:40:18 +02:00
|
|
|
reqInfo := &logger.ReqInfo{
|
2019-07-01 21:22:01 +02:00
|
|
|
DeploymentID: globalDeploymentID,
|
2019-07-03 07:34:32 +02:00
|
|
|
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
2018-11-19 23:47:03 +01:00
|
|
|
RemoteHost: handlers.GetSourceIP(r),
|
2019-07-18 18:58:37 +02:00
|
|
|
Host: getHostName(r),
|
2018-11-19 23:47:03 +01:00
|
|
|
UserAgent: r.UserAgent(),
|
|
|
|
API: api,
|
|
|
|
BucketName: bucket,
|
|
|
|
ObjectName: object,
|
2018-07-02 23:40:18 +02:00
|
|
|
}
|
2019-06-29 07:09:17 +02:00
|
|
|
return logger.SetReqInfo(r.Context(), reqInfo)
|
2018-03-14 20:01:47 +01:00
|
|
|
}
|
2018-03-28 23:14:06 +02:00
|
|
|
|
2018-10-05 02:44:06 +02:00
|
|
|
// Used for registering with rest handlers (have a look at registerStorageRESTHandlers for usage example)
|
|
|
|
// If it is passed ["aaaa", "bbbb"], it returns ["aaaa", "{aaaa:.*}", "bbbb", "{bbbb:.*}"]
|
|
|
|
func restQueries(keys ...string) []string {
|
|
|
|
var accumulator []string
|
|
|
|
for _, key := range keys {
|
|
|
|
accumulator = append(accumulator, key, "{"+key+":.*}")
|
|
|
|
}
|
|
|
|
return accumulator
|
|
|
|
}
|
2019-05-13 21:25:49 +02:00
|
|
|
|
2021-01-28 20:44:48 +01:00
|
|
|
// Suffix returns the longest common suffix of the provided strings
|
|
|
|
func lcpSuffix(strs []string) string {
|
|
|
|
return lcp(strs, false)
|
|
|
|
}
|
|
|
|
|
|
|
|
func lcp(strs []string, pre bool) string {
|
|
|
|
// short-circuit empty list
|
|
|
|
if len(strs) == 0 {
|
2019-08-09 19:02:41 +02:00
|
|
|
return ""
|
|
|
|
}
|
2021-01-28 20:44:48 +01:00
|
|
|
xfix := strs[0]
|
|
|
|
// short-circuit single-element list
|
|
|
|
if len(strs) == 1 {
|
|
|
|
return xfix
|
|
|
|
}
|
|
|
|
// compare first to rest
|
|
|
|
for _, str := range strs[1:] {
|
|
|
|
xfixl := len(xfix)
|
|
|
|
strl := len(str)
|
|
|
|
// short-circuit empty strings
|
|
|
|
if xfixl == 0 || strl == 0 {
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
// maximum possible length
|
|
|
|
maxl := xfixl
|
|
|
|
if strl < maxl {
|
|
|
|
maxl = strl
|
|
|
|
}
|
|
|
|
// compare letters
|
|
|
|
if pre {
|
|
|
|
// prefix, iterate left to right
|
|
|
|
for i := 0; i < maxl; i++ {
|
|
|
|
if xfix[i] != str[i] {
|
|
|
|
xfix = xfix[:i]
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// suffix, iterate right to left
|
|
|
|
for i := 0; i < maxl; i++ {
|
|
|
|
xi := xfixl - i - 1
|
|
|
|
si := strl - i - 1
|
|
|
|
if xfix[xi] != str[si] {
|
|
|
|
xfix = xfix[xi+1:]
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
2019-08-09 19:02:41 +02:00
|
|
|
}
|
|
|
|
}
|
2021-01-28 20:44:48 +01:00
|
|
|
return xfix
|
2019-08-09 19:02:41 +02:00
|
|
|
}
|
2019-08-14 20:43:43 +02:00
|
|
|
|
|
|
|
// Returns the mode in which MinIO is running
|
|
|
|
func getMinioMode() string {
|
|
|
|
mode := globalMinioModeFS
|
2020-06-13 05:04:01 +02:00
|
|
|
if globalIsDistErasure {
|
|
|
|
mode = globalMinioModeDistErasure
|
|
|
|
} else if globalIsErasure {
|
|
|
|
mode = globalMinioModeErasure
|
2019-08-14 20:43:43 +02:00
|
|
|
} else if globalIsGateway {
|
|
|
|
mode = globalMinioModeGatewayPrefix + globalGatewayName
|
|
|
|
}
|
|
|
|
return mode
|
|
|
|
}
|
2019-10-23 07:59:13 +02:00
|
|
|
|
2020-03-23 22:17:18 +01:00
|
|
|
func iamPolicyClaimNameOpenID() string {
|
2020-01-09 02:21:58 +01:00
|
|
|
return globalOpenIDConfig.ClaimPrefix + globalOpenIDConfig.ClaimName
|
2019-10-23 07:59:13 +02:00
|
|
|
}
|
2019-11-12 23:50:18 +01:00
|
|
|
|
2020-03-23 22:17:18 +01:00
|
|
|
func iamPolicyClaimNameSA() string {
|
|
|
|
return "sa-policy"
|
|
|
|
}
|
2020-05-26 21:52:24 +02:00
|
|
|
|
|
|
|
// timedValue contains a synchronized value that is considered valid
|
|
|
|
// for a specific amount of time.
|
|
|
|
// An Update function must be set to provide an updated value when needed.
|
|
|
|
type timedValue struct {
|
|
|
|
// Update must return an updated value.
|
|
|
|
// If an error is returned the cached value is not set.
|
|
|
|
// Only one caller will call this function at any time, others will be blocking.
|
|
|
|
// The returned value can no longer be modified once returned.
|
|
|
|
// Should be set before calling Get().
|
|
|
|
Update func() (interface{}, error)
|
|
|
|
|
|
|
|
// TTL for a cached value.
|
|
|
|
// If not set 1 second TTL is assumed.
|
|
|
|
// Should be set before calling Get().
|
|
|
|
TTL time.Duration
|
|
|
|
|
|
|
|
// Once can be used to initialize values for lazy initialization.
|
|
|
|
// Should be set before calling Get().
|
|
|
|
Once sync.Once
|
|
|
|
|
|
|
|
// Managed values.
|
|
|
|
value interface{}
|
|
|
|
lastUpdate time.Time
|
2021-02-13 03:17:52 +01:00
|
|
|
mu sync.RWMutex
|
2020-05-26 21:52:24 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Get will return a cached value or fetch a new one.
|
|
|
|
// If the Update function returns an error the value is forwarded as is and not cached.
|
|
|
|
func (t *timedValue) Get() (interface{}, error) {
|
2021-02-13 03:17:52 +01:00
|
|
|
v := t.get()
|
|
|
|
if v != nil {
|
|
|
|
return v, nil
|
2020-05-26 21:52:24 +02:00
|
|
|
}
|
2021-02-13 03:17:52 +01:00
|
|
|
|
2020-05-26 21:52:24 +02:00
|
|
|
v, err := t.Update()
|
|
|
|
if err != nil {
|
|
|
|
return v, err
|
|
|
|
}
|
2021-02-13 03:17:52 +01:00
|
|
|
|
|
|
|
t.update(v)
|
2020-05-26 21:52:24 +02:00
|
|
|
return v, nil
|
|
|
|
}
|
|
|
|
|
2021-02-13 03:17:52 +01:00
|
|
|
func (t *timedValue) get() (v interface{}) {
|
|
|
|
ttl := t.TTL
|
|
|
|
if ttl <= 0 {
|
|
|
|
ttl = time.Second
|
|
|
|
}
|
|
|
|
t.mu.RLock()
|
|
|
|
defer t.mu.RUnlock()
|
|
|
|
v = t.value
|
|
|
|
if time.Since(t.lastUpdate) < ttl {
|
|
|
|
return v
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (t *timedValue) update(v interface{}) {
|
2020-05-26 21:52:24 +02:00
|
|
|
t.mu.Lock()
|
2021-02-13 03:17:52 +01:00
|
|
|
defer t.mu.Unlock()
|
|
|
|
t.value = v
|
|
|
|
t.lastUpdate = time.Now()
|
2020-05-26 21:52:24 +02:00
|
|
|
}
|
2020-09-19 17:39:41 +02:00
|
|
|
|
|
|
|
// On MinIO a directory object is stored as a regular object with "__XLDIR__" suffix.
|
|
|
|
// For ex. "prefix/" is stored as "prefix__XLDIR__"
|
|
|
|
func encodeDirObject(object string) string {
|
|
|
|
if HasSuffix(object, slashSeparator) {
|
|
|
|
return strings.TrimSuffix(object, slashSeparator) + globalDirSuffix
|
|
|
|
}
|
|
|
|
return object
|
|
|
|
}
|
|
|
|
|
|
|
|
// Reverse process of encodeDirObject()
|
|
|
|
func decodeDirObject(object string) string {
|
|
|
|
if HasSuffix(object, globalDirSuffix) {
|
|
|
|
return strings.TrimSuffix(object, globalDirSuffix) + slashSeparator
|
|
|
|
}
|
|
|
|
return object
|
|
|
|
}
|
2021-03-01 21:31:33 +01:00
|
|
|
|
|
|
|
// This is used by metrics to show the number of failed RPC calls
|
|
|
|
// between internodes
|
|
|
|
func loadAndResetRPCNetworkErrsCounter() uint64 {
|
|
|
|
defer rest.ResetNetworkErrsCounter()
|
|
|
|
return rest.GetNetworkErrsCounter()
|
|
|
|
}
|