maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add rpc signature handler

Browse source
This commit is contained in:
Harshavardhana 2015-10-08 19:56:41 -07:00
parent 312af12fd5
commit 7fea9cb550
17 changed files with 720 additions and 194 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

139
common.go Normal file
View file

@ -0,0 +1,139 @@
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"crypto/hmac"
"encoding/hex"
"io"
"regexp"
"strings"
"unicode/utf8"
"github.com/minio/minio/pkg/crypto/sha256"
)
///
/// Excerpts from @lsegal - https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258
///
/// User-Agent:
///
/// This is ignored from signing because signing this causes problems with generating pre-signed URLs
/// (that are executed by other agents) or when customers pass requests through proxies, which may
/// modify the user-agent.
///
/// Content-Length:
///
/// This is ignored from signing because generating a pre-signed URL should not provide a content-length
/// constraint, specifically when vending a S3 pre-signed PUT URL. The corollary to this is that when
/// sending regular requests (non-pre-signed), the signature contains a checksum of the body, which
/// implicitly validates the payload length (since changing the number of bytes would change the checksum)
/// and therefore this header is not valuable in the signature.
///
/// Content-Type:
///
/// Signing this header causes quite a number of problems in browser environments, where browsers
/// like to modify and normalize the content-type header in different ways. There is more information
/// on this in https://github.com/aws/aws-sdk-js/issues/244. Avoiding this field simplifies logic
/// and reduces the possibility of future bugs
///
/// Authorization:
///
/// Is skipped for obvious reasons
///
var ignoredHeaders = map[string]bool{
"Authorization": true,
"Content-Type": true,
"Content-Length": true,
"User-Agent": true,
}
// sum256Reader calculate sha256 sum for an input read seeker
func sum256Reader(reader io.ReadSeeker) ([]byte, error) {
h := sha256.New()
var err error
start, _ := reader.Seek(0, 1)
defer reader.Seek(start, 0)
for err == nil {
length := 0
byteBuffer := make([]byte, 1024*1024)
length, err = reader.Read(byteBuffer)
byteBuffer = byteBuffer[0:length]
h.Write(byteBuffer)
}
if err != io.EOF {
return nil, err
}
return h.Sum(nil), nil
}
// sum256 calculate sha256 sum for an input byte array
func sum256(data []byte) []byte {
hash := sha256.New()
hash.Write(data)
return hash.Sum(nil)
}
// sumHMAC calculate hmac between two input byte array
func sumHMAC(key []byte, data []byte) []byte {
hash := hmac.New(sha256.New, key)
hash.Write(data)
return hash.Sum(nil)
}
// getURLEncodedName encode the strings from UTF-8 byte representations to HTML hex escape sequences
//
// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8
// non english characters cannot be parsed due to the nature in which url.Encode() is written
//
// This function on the other hand is a direct replacement for url.Encode() technique to support
// pretty much every UTF-8 character.
func getURLEncodedName(name string) string {
// if object matches reserved string, no need to encode them
reservedNames := regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")
if reservedNames.MatchString(name) {
return name
}
var encodedName string
for _, s := range name {
if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark)
encodedName = encodedName + string(s)
continue
}
switch s {
case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark)
encodedName = encodedName + string(s)
continue
default:
len := utf8.RuneLen(s)
if len < 0 {
return name
}
u := make([]byte, len)
utf8.EncodeRune(u, s)
for _, r := range u {
hex := hex.EncodeToString([]byte{r})
encodedName = encodedName + "%" + strings.ToUpper(hex)
}
}
}
return encodedName
}

2
controller-main.go
View file

@ -100,7 +100,7 @@ func configureControllerRPC(conf minioConfig, rpcHandler http.Handler) (*http.Se
// startController starts a minio controller
func startController(conf minioConfig) *probe.Error {
rpcServer, err := configureControllerRPC(conf, getControllerRPCHandler())
rpcServer, err := configureControllerRPC(conf, getControllerRPCHandler(conf.Anonymous))
if err != nil {
return err.Trace()
}

13
controller-router.go
View file

@ -25,7 +25,14 @@ import (
)
// getControllerRPCHandler rpc handler for controller
func getControllerRPCHandler() http.Handler {
func getControllerRPCHandler(anonymous bool) http.Handler {
var mwHandlers = []MiddlewareHandler{
TimeValidityHandler,
}
if !anonymous {
mwHandlers = append(mwHandlers, RPCSignatureHandler)
}
s := jsonrpc.NewServer()
codec := json.NewCodec()
s.RegisterCodec(codec, "application/json")
@ -35,5 +42,7 @@ func getControllerRPCHandler() http.Handler {
// Add new RPC services here
mux.Handle("/rpc", s)
mux.Handle("/{file:.*}", http.FileServer(assetFS()))
return mux
rpcHandler := registerCustomMiddleware(mux, mwHandlers...)
return rpcHandler
}

243
controller-rpc-signature-handler.go Normal file
View file

@ -0,0 +1,243 @@
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"bytes"
"encoding/hex"
"io"
"net/http"
"sort"
"strings"
"time"
"github.com/minio/minio/pkg/crypto/sha256"
"github.com/minio/minio/pkg/probe"
)
type rpcSignatureHandler struct {
handler http.Handler
}
// RPCSignatureHandler to validate authorization header for the incoming request.
func RPCSignatureHandler(h http.Handler) http.Handler {
return signatureHandler{h}
}
type rpcSignature struct {
AccessKeyID string
SecretAccessKey string
Signature string
SignedHeaders []string
Request *http.Request
}
// getCanonicalHeaders generate a list of request headers with their values
func (r *rpcSignature) getCanonicalHeaders(signedHeaders map[string][]string) string {
var headers []string
vals := make(map[string][]string)
for k, vv := range signedHeaders {
headers = append(headers, strings.ToLower(k))
vals[strings.ToLower(k)] = vv
}
headers = append(headers, "host")
sort.Strings(headers)
var buf bytes.Buffer
for _, k := range headers {
buf.WriteString(k)
buf.WriteByte(':')
switch {
case k == "host":
buf.WriteString(r.Request.Host)
fallthrough
default:
for idx, v := range vals[k] {
if idx > 0 {
buf.WriteByte(',')
}
buf.WriteString(v)
}
buf.WriteByte('\n')
}
}
return buf.String()
}
// getSignedHeaders generate a string i.e alphabetically sorted, semicolon-separated list of lowercase request header names
func (r *rpcSignature) getSignedHeaders(signedHeaders map[string][]string) string {
var headers []string
for k := range signedHeaders {
headers = append(headers, strings.ToLower(k))
}
headers = append(headers, "host")
sort.Strings(headers)
return strings.Join(headers, ";")
}
// extractSignedHeaders extract signed headers from Authorization header
func (r rpcSignature) extractSignedHeaders() map[string][]string {
extractedSignedHeadersMap := make(map[string][]string)
for _, header := range r.SignedHeaders {
val, ok := r.Request.Header[http.CanonicalHeaderKey(header)]
if !ok {
// if not found continue, we will fail later
continue
}
extractedSignedHeadersMap[header] = val
}
return extractedSignedHeadersMap
}
// getCanonicalRequest generate a canonical request of style
//
// canonicalRequest =
// <HTTPMethod>\n
// <CanonicalURI>\n
// <CanonicalQueryString>\n
// <CanonicalHeaders>\n
// <SignedHeaders>\n
// <HashedPayload>
//
func (r *rpcSignature) getCanonicalRequest() string {
payload := r.Request.Header.Get(http.CanonicalHeaderKey("x-amz-content-sha256"))
r.Request.URL.RawQuery = strings.Replace(r.Request.URL.Query().Encode(), "+", "%20", -1)
encodedPath := getURLEncodedName(r.Request.URL.Path)
// convert any space strings back to "+"
encodedPath = strings.Replace(encodedPath, "+", "%20", -1)
canonicalRequest := strings.Join([]string{
r.Request.Method,
encodedPath,
r.Request.URL.RawQuery,
r.getCanonicalHeaders(r.extractSignedHeaders()),
r.getSignedHeaders(r.extractSignedHeaders()),
payload,
}, "\n")
return canonicalRequest
}
// getScope generate a string of a specific date, an AWS region, and a service
func (r rpcSignature) getScope(t time.Time) string {
scope := strings.Join([]string{
t.Format(yyyymmdd),
"milkyway",
"rpc",
"rpc_request",
}, "/")
return scope
}
// getStringToSign a string based on selected query values
func (r rpcSignature) getStringToSign(canonicalRequest string, t time.Time) string {
stringToSign := authHeaderPrefix + "\n" + t.Format(iso8601Format) + "\n"
stringToSign = stringToSign + r.getScope(t) + "\n"
stringToSign = stringToSign + hex.EncodeToString(sha256.Sum256([]byte(canonicalRequest)))
return stringToSign
}
// getSigningKey hmac seed to calculate final signature
func (r rpcSignature) getSigningKey(t time.Time) []byte {
secret := r.SecretAccessKey
date := sumHMAC([]byte("MINIO"+secret), []byte(t.Format(yyyymmdd)))
region := sumHMAC(date, []byte("milkyway"))
service := sumHMAC(region, []byte("rpc"))
signingKey := sumHMAC(service, []byte("rpc_request"))
return signingKey
}
// getSignature final signature in hexadecimal form
func (r rpcSignature) getSignature(signingKey []byte, stringToSign string) string {
return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
}
func (r rpcSignature) DoesSignatureMatch(hashedPayload string) (bool, *probe.Error) {
// set new calulated payload
r.Request.Header.Set("X-Minio-Content-Sha256", hashedPayload)
// Add date if not present throw error
var date string
if date = r.Request.Header.Get(http.CanonicalHeaderKey("x-minio-date")); date == "" {
if date = r.Request.Header.Get("Date"); date == "" {
return false, probe.NewError(errMissingDateHeader)
}
}
t, err := time.Parse(iso8601Format, date)
if err != nil {
return false, probe.NewError(err)
}
canonicalRequest := r.getCanonicalRequest()
stringToSign := r.getStringToSign(canonicalRequest, t)
signingKey := r.getSigningKey(t)
newSignature := r.getSignature(signingKey, stringToSign)
if newSignature != r.Signature {
return false, nil
}
return true, nil
}
func isRequestSignatureRPC(req *http.Request) bool {
if _, ok := req.Header["Authorization"]; ok {
return ok
}
return false
}
func (s rpcSignatureHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
var signature *rpcSignature
if isRequestSignatureRPC(r) {
// Init signature V4 verification
var err *probe.Error
signature, err = initSignatureRPC(r)
if err != nil {
switch err.ToGoError() {
case errInvalidRegion:
errorIf(err.Trace(), "Unknown region in authorization header.", nil)
writeErrorResponse(w, r, AuthorizationHeaderMalformed, r.URL.Path)
return
case errAccessKeyIDInvalid:
errorIf(err.Trace(), "Invalid access key id.", nil)
writeErrorResponse(w, r, InvalidAccessKeyID, r.URL.Path)
return
default:
errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
writeErrorResponse(w, r, InternalError, r.URL.Path)
return
}
}
buffer := new(bytes.Buffer)
if _, err := io.Copy(buffer, r.Body); err != nil {
errorIf(probe.NewError(err), "Unable to read payload from request body.", nil)
writeErrorResponse(w, r, InternalError, r.URL.Path)
return
}
value := sha256.Sum256(buffer.Bytes())
ok, err := signature.DoesSignatureMatch(hex.EncodeToString(value[:]))
if err != nil {
errorIf(err.Trace(), "Unable to verify signature.", nil)
writeErrorResponse(w, r, InternalError, r.URL.Path)
return
}
if !ok {
writeErrorResponse(w, r, SignatureDoesNotMatch, r.URL.Path)
return
}
s.handler.ServeHTTP(w, r)
return
}
writeErrorResponse(w, r, AccessDenied, r.URL.Path)
}

123
controller-rpc-signature.go Normal file
View file

@ -0,0 +1,123 @@
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"net/http"
"strings"
"github.com/minio/minio/pkg/probe"
)
const (
rpcAuthHeaderPrefix = "MINIORPC"
)
// getRPCCredentialsFromAuth parse credentials tag from authorization value
// Authorization:
// Authorization: MINIORPC Credential=admin/20130524/milkyway/rpc/rpc_request,
// SignedHeaders=host;x-minio-date, Signature=fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024
func getRPCCredentialsFromAuth(authValue string) ([]string, *probe.Error) {
if authValue == "" {
return nil, probe.NewError(errMissingAuthHeaderValue)
}
authFields := strings.Split(strings.TrimSpace(authValue), ",")
if len(authFields) != 3 {
return nil, probe.NewError(errInvalidAuthHeaderValue)
}
authPrefixFields := strings.Fields(authFields[0])
if len(authPrefixFields) != 2 {
return nil, probe.NewError(errMissingFieldsAuthHeader)
}
if authPrefixFields[0] != rpcAuthHeaderPrefix {
return nil, probe.NewError(errInvalidAuthHeaderPrefix)
}
credentials := strings.Split(strings.TrimSpace(authPrefixFields[1]), "=")
if len(credentials) != 2 {
return nil, probe.NewError(errMissingFieldsCredentialTag)
}
if len(strings.Split(strings.TrimSpace(authFields[1]), "=")) != 2 {
return nil, probe.NewError(errMissingFieldsSignedHeadersTag)
}
if len(strings.Split(strings.TrimSpace(authFields[2]), "=")) != 2 {
return nil, probe.NewError(errMissingFieldsSignatureTag)
}
credentialElements := strings.Split(strings.TrimSpace(credentials[1]), "/")
if len(credentialElements) != 5 {
return nil, probe.NewError(errCredentialTagMalformed)
}
return credentialElements, nil
}
// verify if rpcAuthHeader value has valid region
func isValidRPCRegion(authHeaderValue string) *probe.Error {
credentialElements, err := getRPCCredentialsFromAuth(authHeaderValue)
if err != nil {
return err.Trace()
}
region := credentialElements[2]
if region != "milkyway" {
return probe.NewError(errInvalidRegion)
}
return nil
}
// stripRPCAccessKeyID - strip only access key id from auth header
func stripRPCAccessKeyID(authHeaderValue string) (string, *probe.Error) {
if err := isValidRegion(authHeaderValue); err != nil {
return "", err.Trace()
}
credentialElements, err := getRPCCredentialsFromAuth(authHeaderValue)
if err != nil {
return "", err.Trace()
}
accessKeyID := credentialElements[0]
if !IsValidAccessKey(accessKeyID) {
return "", probe.NewError(errAccessKeyIDInvalid)
}
return accessKeyID, nil
}
// initSignatureRPC initializing rpc signature verification
func initSignatureRPC(req *http.Request) (*rpcSignature, *probe.Error) {
// strip auth from authorization header
authHeaderValue := req.Header.Get("Authorization")
accessKeyID, err := stripAccessKeyID(authHeaderValue)
if err != nil {
return nil, err.Trace()
}
authConfig, err := LoadConfig()
if err != nil {
return nil, err.Trace()
}
authFields := strings.Split(strings.TrimSpace(authHeaderValue), ",")
signedHeaders := strings.Split(strings.Split(strings.TrimSpace(authFields[1]), "=")[1], ";")
signature := strings.Split(strings.TrimSpace(authFields[2]), "=")[1]
for _, user := range authConfig.Users {
if user.AccessKeyID == accessKeyID {
signature := &rpcSignature{
AccessKeyID: user.AccessKeyID,
SecretAccessKey: user.SecretAccessKey,
Signature: signature,
SignedHeaders: signedHeaders,
Request: req,
}
return signature, nil
}
}
return nil, probe.NewError(errAccessKeyIDInvalid)
}

44
controller-rpc.go
View file

@ -150,6 +150,14 @@ func (s *controllerRPCService) ResetAuth(r *http.Request, args *AuthArgs, reply
return nil
}
func readAuthConfig() (*AuthConfig, *probe.Error) {
authConfig, err := LoadConfig()
if err != nil {
return nil, err.Trace()
}
return authConfig, nil
}
func proxyRequest(method, host string, ssl bool, res interface{}) *probe.Error {
u := &url.URL{}
if ssl {
@ -169,7 +177,11 @@ func proxyRequest(method, host string, ssl bool, res interface{}) *probe.Error {
Method: method,
Request: ServerArg{},
}
request, err := newRPCRequest(u.String(), op, nil)
authConfig, err := readAuthConfig()
if err != nil {
return err.Trace()
}
request, err := newRPCRequest(authConfig, u.String(), op, nil)
if err != nil {
return err.Trace()
}
@ -216,39 +228,11 @@ func (s *controllerRPCService) DiscoverServers(r *http.Request, args *DiscoverAr
defer close(c)
for _, host := range args.Hosts {
go func(c chan DiscoverRepEntry, host string) {
u := &url.URL{}
if args.SSL {
u.Scheme = "https"
} else {
u.Scheme = "http"
}
if args.Port != 0 {
u.Host = host + ":" + string(args.Port)
} else {
u.Host = host + ":9002"
}
u.Path = "/rpc"
op := rpcOperation{
Method: "Server.Version",
Request: ServerArg{},
}
versionrep := VersionRep{}
request, err := newRPCRequest(u.String(), op, nil)
err := proxyRequest("Server.Version", host, args.SSL, rep)
if err != nil {
c <- DiscoverRepEntry{host, err.ToGoError().Error()}
return
}
var resp *http.Response
resp, err = request.Do()
if err != nil {
c <- DiscoverRepEntry{host, err.ToGoError().Error()}
return
}
if err := json.DecodeClientResponse(resp.Body, &versionrep); err != nil {
c <- DiscoverRepEntry{host, err.Error()}
return
}
c <- DiscoverRepEntry{host, ""}
}(c, host)
}

48
controller_rpc_test.go
View file

@ -17,6 +17,7 @@
package main
import (
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
@ -28,8 +29,11 @@ import (
)
type ControllerRPCSuite struct {
root string
url *url.URL
root string
url *url.URL
req *http.Request
body io.ReadSeeker
config *AuthConfig
}
var _ = Suite(&ControllerRPCSuite{})
@ -45,8 +49,24 @@ func (s *ControllerRPCSuite) SetUpSuite(c *C) {
s.root = root
SetAuthConfigPath(root)
testControllerRPC = httptest.NewServer(getControllerRPCHandler())
testServerRPC = httptest.NewUnstartedServer(getServerRPCHandler())
secretAccessKey, perr := generateSecretAccessKey()
c.Assert(perr, IsNil)
authConf := &AuthConfig{}
authConf.Users = make(map[string]*AuthUser)
authConf.Users["admin"] = &AuthUser{
Name: "admin",
AccessKeyID: "admin",
SecretAccessKey: string(secretAccessKey),
}
s.config = authConf
SetAuthConfigPath(root)
perr = SaveConfig(authConf)
c.Assert(perr, IsNil)
testControllerRPC = httptest.NewServer(getControllerRPCHandler(false))
testServerRPC = httptest.NewUnstartedServer(getServerRPCHandler(false))
testServerRPC.Config.Addr = ":9002"
testServerRPC.Start()
@ -66,7 +86,7 @@ func (s *ControllerRPCSuite) TestMemStats(c *C) {
Method: "Controller.GetServerMemStats",
Request: ControllerArgs{Host: s.url.Host},
}
req, err := newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
req, err := newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
c.Assert(err, IsNil)
c.Assert(req.Get("Content-Type"), Equals, "application/json")
resp, err := req.Do()
@ -84,7 +104,7 @@ func (s *ControllerRPCSuite) TestDiskStats(c *C) {
Method: "Controller.GetServerDiskStats",
Request: ControllerArgs{Host: s.url.Host},
}
req, err := newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
req, err := newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
c.Assert(err, IsNil)
c.Assert(req.Get("Content-Type"), Equals, "application/json")
resp, err := req.Do()
@ -102,7 +122,7 @@ func (s *ControllerRPCSuite) TestSysInfo(c *C) {
Method: "Controller.GetServerSysInfo",
Request: ControllerArgs{Host: s.url.Host},
}
req, err := newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
req, err := newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
c.Assert(err, IsNil)
c.Assert(req.Get("Content-Type"), Equals, "application/json")
resp, err := req.Do()
@ -120,7 +140,7 @@ func (s *ControllerRPCSuite) TestServerList(c *C) {
Method: "Controller.ListServers",
Request: ControllerArgs{Host: s.url.Host},
}
req, err := newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
req, err := newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
c.Assert(err, IsNil)
c.Assert(req.Get("Content-Type"), Equals, "application/json")
resp, err := req.Do()
@ -138,7 +158,7 @@ func (s *ControllerRPCSuite) TestServerAdd(c *C) {
Method: "Controller.AddServer",
Request: ControllerArgs{Host: s.url.Host},
}
req, err := newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
req, err := newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
c.Assert(err, IsNil)
c.Assert(req.Get("Content-Type"), Equals, "application/json")
resp, err := req.Do()
@ -156,7 +176,7 @@ func (s *ControllerRPCSuite) TestAuth(c *C) {
Method: "Controller.GenerateAuth",
Request: AuthArgs{User: "newuser"},
}
req, err := newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
req, err := newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
c.Assert(err, IsNil)
c.Assert(req.Get("Content-Type"), Equals, "application/json")
resp, err := req.Do()
@ -175,7 +195,7 @@ func (s *ControllerRPCSuite) TestAuth(c *C) {
Method: "Controller.FetchAuth",
Request: AuthArgs{User: "newuser"},
}
req, err = newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
req, err = newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
c.Assert(err, IsNil)
c.Assert(req.Get("Content-Type"), Equals, "application/json")
resp, err = req.Do()
@ -194,7 +214,7 @@ func (s *ControllerRPCSuite) TestAuth(c *C) {
Method: "Controller.ResetAuth",
Request: AuthArgs{User: "newuser"},
}
req, err = newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
req, err = newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
c.Assert(err, IsNil)
c.Assert(req.Get("Content-Type"), Equals, "application/json")
resp, err = req.Do()
@ -216,7 +236,7 @@ func (s *ControllerRPCSuite) TestAuth(c *C) {
Method: "Controller.GenerateAuth",
Request: AuthArgs{User: "newuser"},
}
req, err = newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
req, err = newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
c.Assert(err, IsNil)
c.Assert(req.Get("Content-Type"), Equals, "application/json")
resp, err = req.Do()
@ -228,7 +248,7 @@ func (s *ControllerRPCSuite) TestAuth(c *C) {
Method: "Controller.GenerateAuth",
Request: AuthArgs{User: ""},
}
req, err = newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
req, err = newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport)
c.Assert(err, IsNil)
c.Assert(req.Get("Content-Type"), Equals, "application/json")
resp, err = req.Do()

27
server-api-generic-handlers.go → generic-handlers.go
View file

@ -22,17 +22,23 @@ import (
"strings"
"time"
router "github.com/gorilla/mux"
"github.com/rs/cors"
)
// MiddlewareHandler - useful to chain different middleware http.Handler
type MiddlewareHandler func(http.Handler) http.Handler
type timeHandler struct {
handler http.Handler
func registerCustomMiddleware(mux *router.Router, mwHandlers ...MiddlewareHandler) http.Handler {
var f http.Handler
f = mux
for _, mw := range mwHandlers {
f = mw(f)
}
return f
}
type validateAuthHandler struct {
type timeHandler struct {
handler http.Handler
}
@ -54,6 +60,19 @@ func parseDate(req *http.Request) (time.Time, error) {
return time.Parse(iso8601Format, amzDate)
}
}
minioDate := req.Header.Get(http.CanonicalHeaderKey("x-minio-date"))
switch {
case minioDate != "":
if _, err := time.Parse(time.RFC1123, minioDate); err == nil {
return time.Parse(time.RFC1123, minioDate)
}
if _, err := time.Parse(time.RFC1123Z, minioDate); err == nil {
return time.Parse(time.RFC1123Z, minioDate)
}
if _, err := time.Parse(iso8601Format, minioDate); err == nil {
return time.Parse(iso8601Format, minioDate)
}
}
date := req.Header.Get("Date")
switch {
case date != "":
@ -78,7 +97,7 @@ func TimeValidityHandler(h http.Handler) http.Handler {
func (h timeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Verify if date headers are set, if not reject the request
if r.Header.Get("Authorization") != "" {
if r.Header.Get(http.CanonicalHeaderKey("x-amz-date")) == "" && r.Header.Get("Date") == "" {
if r.Header.Get(http.CanonicalHeaderKey("x-amz-date")) == "" && r.Header.Get(http.CanonicalHeaderKey("x-minio-date")) == "" && r.Header.Get("Date") == "" {
// there is no way to knowing if this is a valid request, could be a attack reject such clients
writeErrorResponse(w, r, RequestTimeTooSkewed, r.URL.Path)
return

0
donut-metadata.md → pkg/donut/donut-metadata.md
View file

14
pkg/signature/signature-v4.go
View file

@ -97,7 +97,7 @@ func getURLEncodedName(name string) string {
}
// getCanonicalHeaders generate a list of request headers with their values
func (r *Signature) getCanonicalHeaders(signedHeaders map[string][]string) string {
func (r Signature) getCanonicalHeaders(signedHeaders map[string][]string) string {
var headers []string
vals := make(map[string][]string)
for k, vv := range signedHeaders {
@ -129,7 +129,7 @@ func (r *Signature) getCanonicalHeaders(signedHeaders map[string][]string) strin
}
// getSignedHeaders generate a string i.e alphabetically sorted, semicolon-separated list of lowercase request header names
func (r *Signature) getSignedHeaders(signedHeaders map[string][]string) string {
func (r Signature) getSignedHeaders(signedHeaders map[string][]string) string {
var headers []string
for k := range signedHeaders {
headers = append(headers, strings.ToLower(k))
@ -190,7 +190,7 @@ func (r *Signature) getCanonicalRequest() string {
// <SignedHeaders>\n
// <HashedPayload>
//
func (r *Signature) getPresignedCanonicalRequest(presignedQuery string) string {
func (r Signature) getPresignedCanonicalRequest(presignedQuery string) string {
rawQuery := strings.Replace(presignedQuery, "+", "%20", -1)
encodedPath := getURLEncodedName(r.Request.URL.Path)
// convert any space strings back to "+"
@ -207,7 +207,7 @@ func (r *Signature) getPresignedCanonicalRequest(presignedQuery string) string {
}
// getScope generate a string of a specific date, an AWS region, and a service
func (r *Signature) getScope(t time.Time) string {
func (r Signature) getScope(t time.Time) string {
scope := strings.Join([]string{
t.Format(yyyymmdd),
"milkyway",
@ -218,7 +218,7 @@ func (r *Signature) getScope(t time.Time) string {
}
// getStringToSign a string based on selected query values
func (r *Signature) getStringToSign(canonicalRequest string, t time.Time) string {
func (r Signature) getStringToSign(canonicalRequest string, t time.Time) string {
stringToSign := authHeaderPrefix + "\n" + t.Format(iso8601Format) + "\n"
stringToSign = stringToSign + r.getScope(t) + "\n"
stringToSign = stringToSign + hex.EncodeToString(sha256.Sum256([]byte(canonicalRequest)))
@ -226,7 +226,7 @@ func (r *Signature) getStringToSign(canonicalRequest string, t time.Time) string
}
// getSigningKey hmac seed to calculate final signature
func (r *Signature) getSigningKey(t time.Time) []byte {
func (r Signature) getSigningKey(t time.Time) []byte {
secret := r.SecretAccessKey
date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd)))
region := sumHMAC(date, []byte("milkyway"))
@ -236,7 +236,7 @@ func (r *Signature) getSigningKey(t time.Time) []byte {
}
// getSignature final signature in hexadecimal form
func (r *Signature) getSignature(signingKey []byte, stringToSign string) string {
func (r Signature) getSignature(signingKey []byte, stringToSign string) string {
return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
}

109
rpc-request.go
View file

@ -18,7 +18,12 @@ package main
import (
"bytes"
"encoding/hex"
"fmt"
"net/http"
"sort"
"strings"
"time"
"github.com/gorilla/rpc/v2/json"
"github.com/minio/minio/pkg/probe"
@ -37,18 +42,116 @@ type rpcRequest struct {
}
// newRPCRequest initiate a new client RPC request
func newRPCRequest(url string, op rpcOperation, transport http.RoundTripper) (*rpcRequest, *probe.Error) {
func newRPCRequest(config *AuthConfig, url string, op rpcOperation, transport http.RoundTripper) (*rpcRequest, *probe.Error) {
t := time.Now().UTC()
params, err := json.EncodeClientRequest(op.Method, op.Request)
if err != nil {
return nil, probe.NewError(err)
}
req, err := http.NewRequest("POST", url, bytes.NewReader(params))
body := bytes.NewReader(params)
req, err := http.NewRequest("POST", url, body)
if err != nil {
return nil, probe.NewError(err)
}
req.Header.Set("x-minio-date", t.Format(iso8601Format))
// save for subsequent use
hash := func() string {
sum256Bytes, _ := sum256Reader(body)
return hex.EncodeToString(sum256Bytes)
}
hashedPayload := hash()
req.Header.Set("Content-Type", "application/json")
req.Header.Set("x-amz-content-sha256", hashedPayload)
var headers []string
vals := make(map[string][]string)
for k, vv := range req.Header {
if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok {
continue // ignored header
}
headers = append(headers, strings.ToLower(k))
vals[strings.ToLower(k)] = vv
}
headers = append(headers, "host")
sort.Strings(headers)
var canonicalHeaders bytes.Buffer
for _, k := range headers {
canonicalHeaders.WriteString(k)
canonicalHeaders.WriteByte(':')
switch {
case k == "host":
canonicalHeaders.WriteString(req.URL.Host)
fallthrough
default:
for idx, v := range vals[k] {
if idx > 0 {
canonicalHeaders.WriteByte(',')
}
canonicalHeaders.WriteString(v)
}
canonicalHeaders.WriteByte('\n')
}
}
signedHeaders := strings.Join(headers, ";")
req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1)
encodedPath := getURLEncodedName(req.URL.Path)
// convert any space strings back to "+"
encodedPath = strings.Replace(encodedPath, "+", "%20", -1)
//
// canonicalRequest =
// <HTTPMethod>\n
// <CanonicalURI>\n
// <CanonicalQueryString>\n
// <CanonicalHeaders>\n
// <SignedHeaders>\n
// <HashedPayload>
//
canonicalRequest := strings.Join([]string{
req.Method,
encodedPath,
req.URL.RawQuery,
canonicalHeaders.String(),
signedHeaders,
hashedPayload,
}, "\n")
scope := strings.Join([]string{
t.Format(yyyymmdd),
"milkyway",
"rpc",
"rpc_request",
}, "/")
stringToSign := rpcAuthHeaderPrefix + "\n" + t.Format(iso8601Format) + "\n"
stringToSign = stringToSign + scope + "\n"
stringToSign = stringToSign + hex.EncodeToString(sum256([]byte(canonicalRequest)))
fmt.Println(config)
date := sumHMAC([]byte("MINIO"+config.Users["admin"].SecretAccessKey), []byte(t.Format(yyyymmdd)))
region := sumHMAC(date, []byte("milkyway"))
service := sumHMAC(region, []byte("rpc"))
signingKey := sumHMAC(service, []byte("rpc_request"))
signature := hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
// final Authorization header
parts := []string{
rpcAuthHeaderPrefix + " Credential=" + config.Users["admin"].AccessKeyID + "/" + scope,
"SignedHeaders=" + signedHeaders,
"Signature=" + signature,
}
auth := strings.Join(parts, ", ")
req.Header.Set("Authorization", auth)
rpcReq := &rpcRequest{}
rpcReq.req = req
rpcReq.req.Header.Set("Content-Type", "application/json")
if transport == nil {
transport = http.DefaultTransport
}

5
server-api-signature-handler.go
View file

@ -17,10 +17,10 @@
package main
import (
"crypto/sha256"
"encoding/hex"
"net/http"
"github.com/minio/minio/pkg/crypto/sha256"
"github.com/minio/minio/pkg/probe"
signv4 "github.com/minio/minio/pkg/signature"
)
@ -73,8 +73,7 @@ func (s signatureHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
}
value := sha256.Sum256([]byte(""))
ok, err := signature.DoesSignatureMatch(hex.EncodeToString(value[:]))
ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sha256.Sum256([]byte(""))))
if err != nil {
errorIf(err.Trace(), "Unable to verify signature.", nil)
writeErrorResponse(w, r, InternalError, r.URL.Path)

2
server-main.go
View file

@ -136,7 +136,7 @@ func startServer(conf minioConfig) *probe.Error {
if err != nil {
return err.Trace()
}
rpcServer, err := configureServerRPC(conf, getServerRPCHandler())
rpcServer, err := configureServerRPC(conf, getServerRPCHandler(conf.Anonymous))
// start ticket master
go startTM(minioAPI)

22
server-router.go
View file

@ -50,15 +50,6 @@ func registerAPI(mux *router.Router, a API) {
mux.HandleFunc("/{bucket}/{object:.*}", a.DeleteObjectHandler).Methods("DELETE")
}
func registerCustomMiddleware(mux *router.Router, mwHandlers ...MiddlewareHandler) http.Handler {
var f http.Handler
f = mux
for _, mw := range mwHandlers {
f = mw(f)
}
return f
}
// APIOperation container for individual operations read by Ticket Master
type APIOperation struct {
ProceedCh chan struct{}
@ -99,12 +90,21 @@ func getAPIHandler(anonymous bool, api API) http.Handler {
return apiHandler
}
func getServerRPCHandler() http.Handler {
func getServerRPCHandler(anonymous bool) http.Handler {
var mwHandlers = []MiddlewareHandler{
TimeValidityHandler,
}
if !anonymous {
mwHandlers = append(mwHandlers, RPCSignatureHandler)
}
s := jsonrpc.NewServer()
s.RegisterCodec(json.NewCodec(), "application/json")
s.RegisterService(new(serverRPCService), "Server")
s.RegisterService(new(donutRPCService), "Donut")
mux := router.NewRouter()
mux.Handle("/rpc", s)
return mux
rpcHandler := registerCustomMiddleware(mux, mwHandlers...)
return rpcHandler
}

2
server_signV4_donut_cache_test.go
View file

@ -159,7 +159,7 @@ func (s *MyAPIDonutCacheSuite) newRequest(method, urlStr string, contentLength i
signedHeaders := strings.Join(headers, ";")
req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1)
encodedPath, _ := urlEncodeName(req.URL.Path)
encodedPath := getURLEncodedName(req.URL.Path)
// convert any space strings back to "+"
encodedPath = strings.Replace(encodedPath, "+", "%20", -1)

118
server_signV4_donut_test.go
View file

@ -18,19 +18,14 @@ package main
import (
"bytes"
"crypto/hmac"
"crypto/sha256"
"errors"
"io"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"time"
"unicode/utf8"
"encoding/hex"
"encoding/xml"
@ -113,117 +108,6 @@ func (s *MyAPISignatureV4Suite) TearDownSuite(c *C) {
testSignatureV4Server.Close()
}
///
/// Excerpts from @lsegal - https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258
///
/// User-Agent:
///
/// This is ignored from signing because signing this causes problems with generating pre-signed URLs
/// (that are executed by other agents) or when customers pass requests through proxies, which may
/// modify the user-agent.
///
/// Content-Length:
///
/// This is ignored from signing because generating a pre-signed URL should not provide a content-length
/// constraint, specifically when vending a S3 pre-signed PUT URL. The corollary to this is that when
/// sending regular requests (non-pre-signed), the signature contains a checksum of the body, which
/// implicitly validates the payload length (since changing the number of bytes would change the checksum)
/// and therefore this header is not valuable in the signature.
///
/// Content-Type:
///
/// Signing this header causes quite a number of problems in browser environments, where browsers
/// like to modify and normalize the content-type header in different ways. There is more information
/// on this in https://github.com/aws/aws-sdk-js/issues/244. Avoiding this field simplifies logic
/// and reduces the possibility of future bugs
///
/// Authorization:
///
/// Is skipped for obvious reasons
///
var ignoredHeaders = map[string]bool{
"Authorization": true,
"Content-Type": true,
"Content-Length": true,
"User-Agent": true,
}
// urlEncodedName encode the strings from UTF-8 byte representations to HTML hex escape sequences
//
// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8
// non english characters cannot be parsed due to the nature in which url.Encode() is written
//
// This function on the other hand is a direct replacement for url.Encode() technique to support
// pretty much every UTF-8 character.
func urlEncodeName(name string) (string, error) {
// if object matches reserved string, no need to encode them
reservedNames := regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")
if reservedNames.MatchString(name) {
return name, nil
}
var encodedName string
for _, s := range name {
if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark)
encodedName = encodedName + string(s)
continue
}
switch s {
case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark)
encodedName = encodedName + string(s)
continue
default:
len := utf8.RuneLen(s)
if len < 0 {
return "", errors.New("invalid utf-8")
}
u := make([]byte, len)
utf8.EncodeRune(u, s)
for _, r := range u {
hex := hex.EncodeToString([]byte{r})
encodedName = encodedName + "%" + strings.ToUpper(hex)
}
}
}
return encodedName, nil
}
// sum256Reader calculate sha256 sum for an input read seeker
func sum256Reader(reader io.ReadSeeker) ([]byte, error) {
h := sha256.New()
var err error
start, _ := reader.Seek(0, 1)
defer reader.Seek(start, 0)
for err == nil {
length := 0
byteBuffer := make([]byte, 1024*1024)
length, err = reader.Read(byteBuffer)
byteBuffer = byteBuffer[0:length]
h.Write(byteBuffer)
}
if err != io.EOF {
return nil, err
}
return h.Sum(nil), nil
}
// sum256 calculate sha256 sum for an input byte array
func sum256(data []byte) []byte {
hash := sha256.New()
hash.Write(data)
return hash.Sum(nil)
}
// sumHMAC calculate hmac between two input byte array
func sumHMAC(key []byte, data []byte) []byte {
hash := hmac.New(sha256.New, key)
hash.Write(data)
return hash.Sum(nil)
}
func (s *MyAPISignatureV4Suite) newRequest(method, urlStr string, contentLength int64, body io.ReadSeeker) (*http.Request, error) {
t := time.Now().UTC()
req, err := http.NewRequest(method, urlStr, nil)
@ -294,7 +178,7 @@ func (s *MyAPISignatureV4Suite) newRequest(method, urlStr string, contentLength
signedHeaders := strings.Join(headers, ";")
req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1)
encodedPath, _ := urlEncodeName(req.URL.Path)
encodedPath := getURLEncodedName(req.URL.Path)
// convert any space strings back to "+"
encodedPath = strings.Replace(encodedPath, "+", "%20", -1)

3
server-api-typed-errors.go → typed-errors.go
View file

@ -68,3 +68,6 @@ var errPolicyAlreadyExpired = errors.New("Policy already expired")
// errPolicyMissingFields means that form values and policy header have some fields missing.
var errPolicyMissingFields = errors.New("Some fields are missing or do not match in policy")
// errMissingDateHeader means that date header is missing
var errMissingDateHeader = errors.New("Missing date header on the request")