The following commit f44f2e341c
fix was incomplete and we still had presigned URLs printing
in query strings in wrong fashion.
This PR fixes this properly. Avoid double encoding
percent encoded strings such as
`s3%!!(MISSING)A(MISSING)`
Print properly as json encoded.
`s3%3AObjectCreated%3A%2A`
212 lines
5.4 KiB
Go
212 lines
5.4 KiB
Go
/*
|
|
* Minio Cloud Storage, (C) 2015, 2016, 2017 Minio, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package cmd
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"encoding/xml"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
humanize "github.com/dustin/go-humanize"
|
|
"github.com/pkg/profile"
|
|
)
|
|
|
|
// make a copy of http.Header
|
|
func cloneHeader(h http.Header) http.Header {
|
|
h2 := make(http.Header, len(h))
|
|
for k, vv := range h {
|
|
vv2 := make([]string, len(vv))
|
|
copy(vv2, vv)
|
|
h2[k] = vv2
|
|
|
|
}
|
|
return h2
|
|
}
|
|
|
|
// Convert url path into bucket and object name.
|
|
func urlPath2BucketObjectName(u *url.URL) (bucketName, objectName string) {
|
|
if u == nil {
|
|
// Empty url, return bucket and object names.
|
|
return
|
|
}
|
|
|
|
// Trim any preceding slash separator.
|
|
urlPath := strings.TrimPrefix(u.Path, slashSeparator)
|
|
|
|
// Split urlpath using slash separator into a given number of
|
|
// expected tokens.
|
|
tokens := strings.SplitN(urlPath, slashSeparator, 2)
|
|
bucketName = tokens[0]
|
|
if len(tokens) == 2 {
|
|
objectName = tokens[1]
|
|
}
|
|
|
|
// Success.
|
|
return bucketName, objectName
|
|
}
|
|
|
|
// URI scheme constants.
|
|
const (
|
|
httpScheme = "http"
|
|
httpsScheme = "https"
|
|
)
|
|
|
|
// xmlDecoder provide decoded value in xml.
|
|
func xmlDecoder(body io.Reader, v interface{}, size int64) error {
|
|
var lbody io.Reader
|
|
if size > 0 {
|
|
lbody = io.LimitReader(body, size)
|
|
} else {
|
|
lbody = body
|
|
}
|
|
d := xml.NewDecoder(lbody)
|
|
return d.Decode(v)
|
|
}
|
|
|
|
// checkValidMD5 - verify if valid md5, returns md5 in bytes.
|
|
func checkValidMD5(md5 string) ([]byte, error) {
|
|
return base64.StdEncoding.DecodeString(strings.TrimSpace(md5))
|
|
}
|
|
|
|
/// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html
|
|
const (
|
|
// Maximum object size per PUT request is 16GiB.
|
|
// This is a divergence from S3 limit on purpose to support
|
|
// use cases where users are going to upload large files
|
|
// using 'curl' and presigned URL.
|
|
globalMaxObjectSize = 16 * humanize.GiByte
|
|
|
|
// Minimum Part size for multipart upload is 5MiB
|
|
globalMinPartSize = 5 * humanize.MiByte
|
|
|
|
// Maximum Part size for multipart upload is 5GiB
|
|
globalMaxPartSize = 5 * humanize.GiByte
|
|
|
|
// Maximum Part ID for multipart upload is 10000
|
|
// (Acceptable values range from 1 to 10000 inclusive)
|
|
globalMaxPartID = 10000
|
|
)
|
|
|
|
// isMaxObjectSize - verify if max object size
|
|
func isMaxObjectSize(size int64) bool {
|
|
return size > globalMaxObjectSize
|
|
}
|
|
|
|
// // Check if part size is more than maximum allowed size.
|
|
func isMaxAllowedPartSize(size int64) bool {
|
|
return size > globalMaxPartSize
|
|
}
|
|
|
|
// Check if part size is more than or equal to minimum allowed size.
|
|
func isMinAllowedPartSize(size int64) bool {
|
|
return size >= globalMinPartSize
|
|
}
|
|
|
|
// isMaxPartNumber - Check if part ID is greater than the maximum allowed ID.
|
|
func isMaxPartID(partID int) bool {
|
|
return partID > globalMaxPartID
|
|
}
|
|
|
|
func contains(stringList []string, element string) bool {
|
|
for _, e := range stringList {
|
|
if e == element {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Starts a profiler returns nil if profiler is not enabled, caller needs to handle this.
|
|
func startProfiler(profiler string) interface {
|
|
Stop()
|
|
} {
|
|
// Enable profiler if ``_MINIO_PROFILER`` is set. Supported options are [cpu, mem, block].
|
|
switch profiler {
|
|
case "cpu":
|
|
return profile.Start(profile.CPUProfile, profile.NoShutdownHook)
|
|
case "mem":
|
|
return profile.Start(profile.MemProfile, profile.NoShutdownHook)
|
|
case "block":
|
|
return profile.Start(profile.BlockProfile, profile.NoShutdownHook)
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// Global profiler to be used by service go-routine.
|
|
var globalProfiler interface {
|
|
Stop()
|
|
}
|
|
|
|
// dump the request into a string in JSON format.
|
|
func dumpRequest(r *http.Request) string {
|
|
header := cloneHeader(r.Header)
|
|
header.Set("Host", r.Host)
|
|
// Replace all '%' to '%%' so that printer format parser
|
|
// to ignore URL encoded values.
|
|
rawURI := strings.Replace(r.RequestURI, "%", "%%", -1)
|
|
req := struct {
|
|
Method string `json:"method"`
|
|
RequestURI string `json:"reqURI"`
|
|
Header http.Header `json:"header"`
|
|
}{r.Method, rawURI, header}
|
|
|
|
var buffer bytes.Buffer
|
|
enc := json.NewEncoder(&buffer)
|
|
enc.SetEscapeHTML(false)
|
|
if err := enc.Encode(&req); err != nil {
|
|
// Upon error just return Go-syntax representation of the value
|
|
return fmt.Sprintf("%#v", req)
|
|
}
|
|
|
|
// Formatted string.
|
|
return strings.TrimSpace(string(buffer.Bytes()))
|
|
}
|
|
|
|
// isFile - returns whether given path is a file or not.
|
|
func isFile(path string) bool {
|
|
if fi, err := osStat(path); err == nil {
|
|
return fi.Mode().IsRegular()
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
// checkURL - checks if passed address correspond
|
|
func checkURL(urlStr string) (*url.URL, error) {
|
|
if urlStr == "" {
|
|
return nil, errors.New("Address cannot be empty")
|
|
}
|
|
u, err := url.Parse(urlStr)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("`%s` invalid: %s", urlStr, err.Error())
|
|
}
|
|
return u, nil
|
|
}
|
|
|
|
// UTCNow - returns current UTC time.
|
|
func UTCNow() time.Time {
|
|
return time.Now().UTC()
|
|
}
|