2016-04-29 05:01:11 +02:00
/ *
2019-04-09 20:39:42 +02:00
* MinIO Cloud Storage , ( C ) 2015 , 2016 , 2017 MinIO , Inc .
2016-04-29 05:01:11 +02:00
*
* Licensed under the Apache License , Version 2.0 ( the "License" ) ;
* you may not use this file except in compliance with the License .
* You may obtain a copy of the License at
*
* http : //www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing , software
* distributed under the License is distributed on an "AS IS" BASIS ,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
* See the License for the specific language governing permissions and
* limitations under the License .
* /
2016-08-19 01:23:42 +02:00
package cmd
2016-04-29 05:01:11 +02:00
import (
2018-08-01 23:19:11 +02:00
"bytes"
2018-04-06 00:04:40 +02:00
"context"
2019-11-04 18:30:59 +01:00
"fmt"
2016-04-29 05:01:11 +02:00
"io"
2018-08-01 23:19:11 +02:00
"io/ioutil"
2016-07-24 07:51:12 +02:00
"mime/multipart"
2017-11-15 01:56:24 +01:00
"net"
2016-07-19 06:20:17 +02:00
"net/http"
2017-03-13 22:41:13 +01:00
"net/url"
2019-11-04 18:30:59 +01:00
"regexp"
2016-07-23 05:31:45 +02:00
"strings"
2017-10-25 04:04:51 +02:00
2019-07-03 07:34:32 +02:00
xhttp "github.com/minio/minio/cmd/http"
2018-04-06 00:04:40 +02:00
"github.com/minio/minio/cmd/logger"
2018-11-30 02:35:11 +01:00
"github.com/minio/minio/pkg/auth"
2020-01-27 23:12:34 +01:00
"github.com/minio/minio/pkg/bucket/object/tagging"
2018-07-02 23:40:18 +02:00
"github.com/minio/minio/pkg/handlers"
2019-11-04 18:30:59 +01:00
"github.com/minio/minio/pkg/madmin"
2020-01-20 17:45:59 +01:00
)
const (
copyDirective = "COPY"
replaceDirective = "REPLACE"
2016-04-29 05:01:11 +02:00
)
2017-04-03 23:50:09 +02:00
// Parses location constraint from the incoming reader.
func parseLocationConstraint ( r * http . Request ) ( location string , s3Error APIErrorCode ) {
2016-07-19 06:20:17 +02:00
// If the request has no body with content-length set to 0,
// we do not have to validate location constraint. Bucket will
// be created at default region.
locationConstraint := createBucketLocationConfiguration { }
2016-09-30 00:51:00 +02:00
err := xmlDecoder ( r . Body , & locationConstraint , r . ContentLength )
2019-01-31 16:19:09 +01:00
if err != nil && r . ContentLength != 0 {
2018-04-06 00:04:40 +02:00
logger . LogIf ( context . Background ( ) , err )
2017-04-03 23:50:09 +02:00
// Treat all other failures as XML parsing errors.
return "" , ErrMalformedXML
} // else for both err as nil or io.EOF
location = locationConstraint . Location
if location == "" {
2019-10-23 07:59:13 +02:00
location = globalServerRegion
2016-04-29 05:01:11 +02:00
}
2017-04-03 23:50:09 +02:00
return location , ErrNone
}
// Validates input location is same as configured region
2019-04-09 20:39:42 +02:00
// of MinIO server.
2017-04-03 23:50:09 +02:00
func isValidLocation ( location string ) bool {
2019-10-23 07:59:13 +02:00
return globalServerRegion == "" || globalServerRegion == location
2016-04-29 05:01:11 +02:00
}
2016-07-23 05:31:45 +02:00
// Supported headers that needs to be extracted.
var supportedHeaders = [ ] string {
"content-type" ,
"cache-control" ,
2018-03-14 10:57:32 +01:00
"content-language" ,
2016-07-23 05:31:45 +02:00
"content-encoding" ,
"content-disposition" ,
2019-10-07 07:50:24 +02:00
xhttp . AmzStorageClass ,
2020-01-20 17:45:59 +01:00
xhttp . AmzObjectTagging ,
2018-03-28 23:14:06 +02:00
"expires" ,
2016-07-23 05:31:45 +02:00
// Add more supported headers here.
}
2020-01-20 17:45:59 +01:00
// isDirectiveValid - check if tagging-directive is valid.
func isDirectiveValid ( v string ) bool {
// Check if set metadata-directive is valid.
return isDirectiveCopy ( v ) || isDirectiveReplace ( v )
2016-12-27 01:29:26 +01:00
}
2020-01-20 17:45:59 +01:00
// Check if the directive COPY is requested.
func isDirectiveCopy ( value string ) bool {
// By default if directive is not set we
// treat it as 'COPY' this function returns true.
return value == copyDirective || value == ""
2016-12-27 01:29:26 +01:00
}
2020-01-20 17:45:59 +01:00
// Check if the directive REPLACE is requested.
func isDirectiveReplace ( value string ) bool {
return value == replaceDirective
2016-12-27 01:29:26 +01:00
}
2017-08-23 01:53:35 +02:00
// userMetadataKeyPrefixes contains the prefixes of used-defined metadata keys.
// All values stored with a key starting with one of the following prefixes
// must be extracted from the header.
var userMetadataKeyPrefixes = [ ] string {
"X-Amz-Meta-" ,
"X-Minio-Meta-" ,
}
2018-07-11 05:27:10 +02:00
// extractMetadata extracts metadata from HTTP header and HTTP queryString.
func extractMetadata ( ctx context . Context , r * http . Request ) ( metadata map [ string ] string , err error ) {
query := r . URL . Query ( )
header := r . Header
metadata = make ( map [ string ] string )
// Extract all query values.
err = extractMetadataFromMap ( ctx , query , metadata )
if err != nil {
return nil , err
}
// Extract all header values.
err = extractMetadataFromMap ( ctx , header , metadata )
if err != nil {
return nil , err
2017-03-13 22:41:13 +01:00
}
2017-12-22 12:28:13 +01:00
2018-12-19 23:31:45 +01:00
// Set content-type to default value if it is not set.
if _ , ok := metadata [ "content-type" ] ; ! ok {
metadata [ "content-type" ] = "application/octet-stream"
}
2018-07-11 05:27:10 +02:00
// Success.
return metadata , nil
}
// extractMetadata extracts metadata from map values.
func extractMetadataFromMap ( ctx context . Context , v map [ string ] [ ] string , m map [ string ] string ) error {
if v == nil {
logger . LogIf ( ctx , errInvalidArgument )
return errInvalidArgument
}
2017-12-22 12:28:13 +01:00
// Save all supported headers.
2016-07-23 05:31:45 +02:00
for _ , supportedHeader := range supportedHeaders {
2018-07-11 05:27:10 +02:00
if value , ok := v [ http . CanonicalHeaderKey ( supportedHeader ) ] ; ok {
m [ supportedHeader ] = value [ 0 ]
} else if value , ok := v [ supportedHeader ] ; ok {
m [ supportedHeader ] = value [ 0 ]
2016-07-23 05:31:45 +02:00
}
}
2018-07-11 05:27:10 +02:00
for key := range v {
2017-08-23 01:53:35 +02:00
for _ , prefix := range userMetadataKeyPrefixes {
2018-07-11 05:27:10 +02:00
if ! strings . HasPrefix ( strings . ToLower ( key ) , strings . ToLower ( prefix ) ) {
continue
}
value , ok := v [ key ]
if ok {
2018-07-12 18:40:14 +02:00
m [ key ] = strings . Join ( value , "," )
2017-08-23 01:53:35 +02:00
break
}
2016-07-23 05:31:45 +02:00
}
}
2018-07-11 05:27:10 +02:00
return nil
2016-12-20 01:14:04 +01:00
}
2020-01-20 17:45:59 +01:00
// extractTags extracts tag key and value from given http header. It then
// - Parses the input format X-Amz-Tagging:"Key1=Value1&Key2=Value2" into a map[string]string
// with entries in the format X-Amg-Tag-Key1:Value1, X-Amz-Tag-Key2:Value2
// - Validates the tags
// - Returns the Tag in original string format "Key1=Value1&Key2=Value2"
func extractTags ( ctx context . Context , tags string ) ( string , error ) {
// Check if the metadata has tagging related header
if tags != "" {
tagging , err := tagging . FromString ( tags )
if err != nil {
return "" , err
}
if err := tagging . Validate ( ) ; err != nil {
return "" , err
}
return tagging . String ( ) , nil
}
return "" , nil
}
2017-03-13 22:41:13 +01:00
// The Query string for the redirect URL the client is
// redirected on successful upload.
func getRedirectPostRawQuery ( objInfo ObjectInfo ) string {
redirectValues := make ( url . Values )
redirectValues . Set ( "bucket" , objInfo . Bucket )
redirectValues . Set ( "key" , objInfo . Name )
2017-05-14 21:05:51 +02:00
redirectValues . Set ( "etag" , "\"" + objInfo . ETag + "\"" )
2017-03-13 22:41:13 +01:00
return redirectValues . Encode ( )
}
2018-11-30 02:35:11 +01:00
// Returns access credentials in the request Authorization header.
func getReqAccessCred ( r * http . Request , region string ) ( cred auth . Credentials ) {
2019-02-28 02:46:55 +01:00
cred , _ , _ = getReqAccessKeyV4 ( r , region , serviceS3 )
2018-11-07 15:40:03 +01:00
if cred . AccessKey == "" {
cred , _ , _ = getReqAccessKeyV2 ( r )
2018-11-03 02:40:08 +01:00
}
2018-12-19 14:13:47 +01:00
if cred . AccessKey == "" {
claims , owner , _ := webRequestAuthenticate ( r )
if owner {
2019-10-23 07:59:13 +02:00
return globalActiveCred
2018-12-19 14:13:47 +01:00
}
2019-10-23 07:59:13 +02:00
cred , _ = globalIAMSys . GetUser ( claims . AccessKey ( ) )
2018-12-19 14:13:47 +01:00
}
2018-11-30 02:35:11 +01:00
return cred
2018-11-03 02:40:08 +01:00
}
2017-03-13 22:41:13 +01:00
// Extract request params to be sent with event notifiation.
func extractReqParams ( r * http . Request ) map [ string ] string {
if r == nil {
return nil
2016-12-20 01:14:04 +01:00
}
2017-03-13 22:41:13 +01:00
2019-10-23 07:59:13 +02:00
region := globalServerRegion
2018-11-30 02:35:11 +01:00
cred := getReqAccessCred ( r , region )
2018-12-19 14:13:47 +01:00
2017-03-13 22:41:13 +01:00
// Success.
return map [ string ] string {
2018-11-03 02:40:08 +01:00
"region" : region ,
2018-11-30 02:35:11 +01:00
"accessKey" : cred . AccessKey ,
2018-07-02 23:40:18 +02:00
"sourceIPAddress" : handlers . GetSourceIP ( r ) ,
2017-03-13 22:41:13 +01:00
// Add more fields here.
}
}
2018-08-23 23:40:54 +02:00
// Extract response elements to be sent with event notifiation.
func extractRespElements ( w http . ResponseWriter ) map [ string ] string {
return map [ string ] string {
2019-07-03 07:34:32 +02:00
"requestId" : w . Header ( ) . Get ( xhttp . AmzRequestID ) ,
"content-length" : w . Header ( ) . Get ( xhttp . ContentLength ) ,
2018-08-23 23:40:54 +02:00
// Add more fields here.
}
}
2017-03-28 02:02:04 +02:00
// Trims away `aws-chunked` from the content-encoding header if present.
// Streaming signature clients can have custom content-encoding such as
// `aws-chunked,gzip` here we need to only save `gzip`.
// For more refer http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
func trimAwsChunkedContentEncoding ( contentEnc string ) ( trimmedContentEnc string ) {
if contentEnc == "" {
return contentEnc
}
var newEncs [ ] string
for _ , enc := range strings . Split ( contentEnc , "," ) {
if enc != streamingContentEncoding {
newEncs = append ( newEncs , enc )
}
}
return strings . Join ( newEncs , "," )
}
2017-03-13 22:41:13 +01:00
// Validate form field size for s3 specification requirement.
2018-04-06 00:04:40 +02:00
func validateFormFieldSize ( ctx context . Context , formValues http . Header ) error {
2017-03-13 22:41:13 +01:00
// Iterate over form values
for k := range formValues {
// Check if value's field exceeds S3 limit
if int64 ( len ( formValues . Get ( k ) ) ) > maxFormFieldSize {
2018-04-06 00:04:40 +02:00
logger . LogIf ( ctx , errSizeUnexpected )
return errSizeUnexpected
2016-12-20 01:14:04 +01:00
}
}
2017-03-13 22:41:13 +01:00
// Success.
return nil
2016-07-23 05:31:45 +02:00
}
2016-07-24 07:51:12 +02:00
2016-07-28 21:02:22 +02:00
// Extract form fields and file data from a HTTP POST Policy
2018-04-06 00:04:40 +02:00
func extractPostPolicyFormValues ( ctx context . Context , form * multipart . Form ) ( filePart io . ReadCloser , fileName string , fileSize int64 , formValues http . Header , err error ) {
2016-07-24 07:51:12 +02:00
/// HTML Form values
2016-07-28 21:02:22 +02:00
fileName = ""
2017-02-02 19:45:00 +01:00
2017-03-13 22:41:13 +01:00
// Canonicalize the form values into http.Header.
formValues = make ( http . Header )
2017-02-02 19:45:00 +01:00
for k , v := range form . Value {
2017-03-13 22:41:13 +01:00
formValues [ http . CanonicalHeaderKey ( k ) ] = v
}
// Validate form values.
2018-04-06 00:04:40 +02:00
if err = validateFormFieldSize ( ctx , formValues ) ; err != nil {
2017-03-13 22:41:13 +01:00
return nil , "" , 0 , nil , err
2017-02-02 19:45:00 +01:00
}
2018-08-01 23:19:11 +02:00
// this means that filename="" was not specified for file key and Go has
// an ugly way of handling this situation. Refer here
// https://golang.org/src/mime/multipart/formdata.go#L61
if len ( form . File ) == 0 {
var b = & bytes . Buffer { }
for _ , v := range formValues [ "File" ] {
b . WriteString ( v )
}
fileSize = int64 ( b . Len ( ) )
filePart = ioutil . NopCloser ( b )
return filePart , fileName , fileSize , formValues , nil
}
2017-02-02 19:45:00 +01:00
// Iterator until we find a valid File field and break
for k , v := range form . File {
canonicalFormName := http . CanonicalHeaderKey ( k )
if canonicalFormName == "File" {
if len ( v ) == 0 {
2018-04-06 00:04:40 +02:00
logger . LogIf ( ctx , errInvalidArgument )
return nil , "" , 0 , nil , errInvalidArgument
2016-07-24 07:51:12 +02:00
}
2017-02-02 19:45:00 +01:00
// Fetch fileHeader which has the uploaded file information
fileHeader := v [ 0 ]
// Set filename
fileName = fileHeader . Filename
// Open the uploaded part
filePart , err = fileHeader . Open ( )
2017-02-09 21:37:32 +01:00
if err != nil {
2018-04-06 00:04:40 +02:00
logger . LogIf ( ctx , err )
return nil , "" , 0 , nil , err
2017-02-09 21:37:32 +01:00
}
2017-02-02 19:45:00 +01:00
// Compute file size
fileSize , err = filePart . ( io . Seeker ) . Seek ( 0 , 2 )
if err != nil {
2018-04-06 00:04:40 +02:00
logger . LogIf ( ctx , err )
return nil , "" , 0 , nil , err
2017-02-02 19:45:00 +01:00
}
// Reset Seek to the beginning
_ , err = filePart . ( io . Seeker ) . Seek ( 0 , 0 )
if err != nil {
2018-04-06 00:04:40 +02:00
logger . LogIf ( ctx , err )
return nil , "" , 0 , nil , err
2017-02-02 19:45:00 +01:00
}
// File found and ready for reading
break
2016-07-24 07:51:12 +02:00
}
}
2017-02-02 19:45:00 +01:00
return filePart , fileName , fileSize , formValues , nil
2016-07-24 07:51:12 +02:00
}
2017-10-25 04:04:51 +02:00
// Log headers and body.
func httpTraceAll ( f http . HandlerFunc ) http . HandlerFunc {
2019-06-09 00:54:41 +02:00
return func ( w http . ResponseWriter , r * http . Request ) {
2019-06-27 07:41:12 +02:00
if ! globalHTTPTrace . HasSubscribers ( ) {
2019-06-09 00:54:41 +02:00
f . ServeHTTP ( w , r )
return
}
trace := Trace ( f , true , w , r )
2019-06-27 07:41:12 +02:00
globalHTTPTrace . Publish ( trace )
2017-10-25 04:04:51 +02:00
}
}
// Log only the headers.
func httpTraceHdrs ( f http . HandlerFunc ) http . HandlerFunc {
2019-06-09 00:54:41 +02:00
return func ( w http . ResponseWriter , r * http . Request ) {
2019-06-27 07:41:12 +02:00
if ! globalHTTPTrace . HasSubscribers ( ) {
2019-06-09 00:54:41 +02:00
f . ServeHTTP ( w , r )
return
}
trace := Trace ( f , false , w , r )
2019-06-27 07:41:12 +02:00
globalHTTPTrace . Publish ( trace )
2017-10-25 04:04:51 +02:00
}
}
2017-11-15 01:56:24 +01:00
2019-10-23 06:01:14 +02:00
func collectAPIStats ( api string , f http . HandlerFunc ) http . HandlerFunc {
return func ( w http . ResponseWriter , r * http . Request ) {
isS3Request := ! strings . HasPrefix ( r . URL . Path , minioReservedBucketPath )
apiStatsWriter := & recordAPIStats { w , UTCNow ( ) , false , 0 , isS3Request }
// Time start before the call is about to start.
tBefore := UTCNow ( )
if isS3Request {
globalHTTPStats . currentS3Requests . Inc ( api )
}
// Execute the request
f . ServeHTTP ( apiStatsWriter , r )
if isS3Request {
globalHTTPStats . currentS3Requests . Dec ( api )
}
// Firstbyte read.
tAfter := apiStatsWriter . TTFB
// Time duration in secs since the call started.
//
// We don't need to do nanosecond precision in this
// simply for the fact that it is not human readable.
durationSecs := tAfter . Sub ( tBefore ) . Seconds ( )
// Update http statistics
globalHTTPStats . updateStats ( api , r , apiStatsWriter , durationSecs )
}
}
2017-11-15 01:56:24 +01:00
// Returns "/bucketName/objectName" for path-style or virtual-host-style requests.
2019-02-23 04:18:01 +01:00
func getResource ( path string , host string , domains [ ] string ) ( string , error ) {
if len ( domains ) == 0 {
2017-11-15 01:56:24 +01:00
return path , nil
}
// If virtual-host-style is enabled construct the "resource" properly.
if strings . Contains ( host , ":" ) {
// In bucket.mydomain.com:9000, strip out :9000
var err error
if host , _ , err = net . SplitHostPort ( host ) ; err != nil {
2018-04-06 00:04:40 +02:00
reqInfo := ( & logger . ReqInfo { } ) . AppendTags ( "host" , host )
reqInfo . AppendTags ( "path" , path )
ctx := logger . SetReqInfo ( context . Background ( ) , reqInfo )
logger . LogIf ( ctx , err )
2017-11-15 01:56:24 +01:00
return "" , err
}
}
2019-02-23 04:18:01 +01:00
for _ , domain := range domains {
if ! strings . HasSuffix ( host , "." + domain ) {
continue
}
bucket := strings . TrimSuffix ( host , "." + domain )
2019-08-06 21:08:58 +02:00
return SlashSeparator + pathJoin ( bucket , path ) , nil
2017-11-15 01:56:24 +01:00
}
2019-02-23 04:18:01 +01:00
return path , nil
2017-11-15 01:56:24 +01:00
}
2019-11-04 18:30:59 +01:00
var regexVersion = regexp . MustCompile ( ` (\w\d+) ` )
func extractAPIVersion ( r * http . Request ) string {
return regexVersion . FindString ( r . URL . Path )
2017-11-15 01:56:24 +01:00
}
2019-07-06 05:41:35 +02:00
2019-11-04 18:30:59 +01:00
// If none of the http routes match respond with appropriate errors
func errorResponseHandler ( w http . ResponseWriter , r * http . Request ) {
version := extractAPIVersion ( r )
2019-10-03 09:08:12 +02:00
switch {
2019-11-04 18:30:59 +01:00
case strings . HasPrefix ( r . URL . Path , peerRESTPrefix ) :
desc := fmt . Sprintf ( "Expected 'peer' API version '%s', instead found '%s', please upgrade the servers" ,
peerRESTVersion , version )
writeErrorResponseString ( r . Context ( ) , w , APIError {
Code : "XMinioPeerVersionMismatch" ,
Description : desc ,
HTTPStatusCode : http . StatusBadRequest ,
} , r . URL )
case strings . HasPrefix ( r . URL . Path , storageRESTPrefix ) :
desc := fmt . Sprintf ( "Expected 'storage' API version '%s', instead found '%s', please upgrade the servers" ,
storageRESTVersion , version )
writeErrorResponseString ( r . Context ( ) , w , APIError {
Code : "XMinioStorageVersionMismatch" ,
Description : desc ,
HTTPStatusCode : http . StatusBadRequest ,
} , r . URL )
case strings . HasPrefix ( r . URL . Path , lockRESTPrefix ) :
desc := fmt . Sprintf ( "Expected 'lock' API version '%s', instead found '%s', please upgrade the servers" ,
lockRESTVersion , version )
writeErrorResponseString ( r . Context ( ) , w , APIError {
Code : "XMinioLockVersionMismatch" ,
Description : desc ,
HTTPStatusCode : http . StatusBadRequest ,
} , r . URL )
case strings . HasPrefix ( r . URL . Path , adminPathPrefix ) :
var desc string
if version == "v1" {
desc = fmt . Sprintf ( "Server expects client requests with 'admin' API version '%s', found '%s', please upgrade the client to latest releases" , madmin . AdminAPIVersion , version )
} else if version == madmin . AdminAPIVersion {
desc = fmt . Sprintf ( "This 'admin' API is not supported by server in '%s'" , getMinioMode ( ) )
} else {
desc = fmt . Sprintf ( "Unexpected client 'admin' API version found '%s', expected '%s', please downgrade the client to older releases" , version , madmin . AdminAPIVersion )
}
writeErrorResponseJSON ( r . Context ( ) , w , APIError {
Code : "XMinioAdminVersionMismatch" ,
Description : desc ,
HTTPStatusCode : http . StatusBadRequest ,
} , r . URL )
2019-10-03 09:08:12 +02:00
default :
2019-11-04 18:30:59 +01:00
desc := fmt . Sprintf ( "Unknown API request at %s" , r . URL . Path )
writeErrorResponse ( r . Context ( ) , w , APIError {
Code : "XMinioUnknownAPIRequest" ,
Description : desc ,
HTTPStatusCode : http . StatusBadRequest ,
} , r . URL , guessIsBrowserReq ( r ) )
2019-10-03 09:08:12 +02:00
}
}
2019-07-06 05:41:35 +02:00
// gets host name for current node
2019-07-18 18:58:37 +02:00
func getHostName ( r * http . Request ) ( hostName string ) {
2019-07-06 05:41:35 +02:00
if globalIsDistXL {
2019-07-18 18:58:37 +02:00
hostName = GetLocalPeer ( globalEndpoints )
} else {
hostName = r . Host
2019-07-06 05:41:35 +02:00
}
return
}