diff --git a/cmd/crypto/kes.go b/cmd/crypto/kes.go index 8d122e304..536e6eee2 100644 --- a/cmd/crypto/kes.go +++ b/cmd/crypto/kes.go @@ -177,11 +177,10 @@ func (kes *kesService) CreateKey(keyID string) error { return kes.client.CreateK // named key referenced by keyID. It also binds the generated key // cryptographically to the provided context. func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) { - var context bytes.Buffer - ctx.WriteTo(&context) + context := ctx.AppendTo(make([]byte, 0, 128)) var plainKey []byte - plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context.Bytes()) + plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context) if err != nil { return key, nil, err } @@ -200,11 +199,10 @@ func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea // The context must be same context as the one provided while // generating the plaintext key / sealedKey. func (kes *kesService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) { - var context bytes.Buffer - ctx.WriteTo(&context) + context := ctx.AppendTo(make([]byte, 0, 128)) var plainKey []byte - plainKey, err = kes.client.DecryptDataKey(keyID, sealedKey, context.Bytes()) + plainKey, err = kes.client.DecryptDataKey(keyID, sealedKey, context) if err != nil { return key, err } diff --git a/cmd/crypto/key.go b/cmd/crypto/key.go index f069159df..561d749ea 100644 --- a/cmd/crypto/key.go +++ b/cmd/crypto/key.go @@ -103,7 +103,6 @@ func (key ObjectKey) Seal(extKey, iv [32]byte, domain, bucket, object string) Se func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucket, object string) error { var ( unsealConfig sio.Config - decryptedKey bytes.Buffer ) switch sealedKey.Algorithm { default: @@ -122,10 +121,9 @@ func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucke unsealConfig = sio.Config{MinVersion: sio.Version10, Key: sha.Sum(nil)} } - if n, err := sio.Decrypt(&decryptedKey, bytes.NewReader(sealedKey.Key[:]), unsealConfig); n != 32 || err != nil { + if out, err := sio.DecryptBuffer(key[:0], sealedKey.Key[:], unsealConfig); len(out) != 32 || err != nil { return ErrSecretKeyMismatch } - copy(key[:], decryptedKey.Bytes()) return nil } @@ -165,11 +163,7 @@ func (key ObjectKey) UnsealETag(etag []byte) ([]byte, error) { if !IsETagSealed(etag) { return etag, nil } - var buffer bytes.Buffer mac := hmac.New(sha256.New, key[:]) mac.Write([]byte("SSE-etag")) - if _, err := sio.Decrypt(&buffer, bytes.NewReader(etag), sio.Config{Key: mac.Sum(nil)}); err != nil { - return nil, err - } - return buffer.Bytes(), nil + return sio.DecryptBuffer(make([]byte, 0, len(etag)), etag, sio.Config{Key: mac.Sum(nil)}) } diff --git a/cmd/crypto/kms.go b/cmd/crypto/kms.go index b52e6e4f0..088833794 100644 --- a/cmd/crypto/kms.go +++ b/cmd/crypto/kms.go @@ -39,6 +39,8 @@ type Context map[string]string // // WriteTo sorts the context keys and writes the sorted // key-value pairs as canonical JSON object to w. +// +// Note that neither keys nor values are escaped for JSON. func (c Context) WriteTo(w io.Writer) (n int64, err error) { sortedKeys := make(sort.StringSlice, 0, len(c)) for k := range c { @@ -67,6 +69,53 @@ func (c Context) WriteTo(w io.Writer) (n int64, err error) { return n + int64(nn), err } +// AppendTo appends the context in a canonical from to dst. +// +// AppendTo sorts the context keys and writes the sorted +// key-value pairs as canonical JSON object to w. +// +// Note that neither keys nor values are escaped for JSON. +func (c Context) AppendTo(dst []byte) (output []byte) { + if len(c) == 0 { + return append(dst, '{', '}') + } + + // out should not escape. + out := bytes.NewBuffer(dst) + + // No need to copy+sort + if len(c) == 1 { + for k, v := range c { + out.WriteString(`{"`) + out.WriteString(k) + out.WriteString(`":"`) + out.WriteString(v) + out.WriteString(`"}`) + } + return out.Bytes() + } + + sortedKeys := make([]string, 0, len(c)) + for k := range c { + sortedKeys = append(sortedKeys, k) + } + sort.Strings(sortedKeys) + + out.WriteByte('{') + for i, k := range sortedKeys { + out.WriteByte('"') + out.WriteString(k) + out.WriteString(`":"`) + out.WriteString(c[k]) + out.WriteByte('"') + if i < len(sortedKeys)-1 { + out.WriteByte(',') + } + } + out.WriteByte('}') + return out.Bytes() +} + // KMS represents an active and authenticted connection // to a Key-Management-Service. It supports generating // data key generation and unsealing of KMS-generated @@ -155,13 +204,12 @@ func (kms *masterKeyKMS) Info() (info KMSInfo) { func (kms *masterKeyKMS) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) { var ( - buffer bytes.Buffer derivedKey = kms.deriveKey(keyID, ctx) ) - if n, err := sio.Decrypt(&buffer, bytes.NewReader(sealedKey), sio.Config{Key: derivedKey[:]}); err != nil || n != 32 { + out, err := sio.DecryptBuffer(key[:0], sealedKey, sio.Config{Key: derivedKey[:]}) + if err != nil || len(out) != 32 { return key, err // TODO(aead): upgrade sio to use sio.Error } - copy(key[:], buffer.Bytes()) return key, nil } @@ -171,7 +219,7 @@ func (kms *masterKeyKMS) deriveKey(keyID string, context Context) (key [32]byte) } mac := hmac.New(sha256.New, kms.masterKey[:]) mac.Write([]byte(keyID)) - context.WriteTo(mac) + mac.Write(context.AppendTo(make([]byte, 0, 128))) mac.Sum(key[:0]) return key } diff --git a/cmd/crypto/kms_test.go b/cmd/crypto/kms_test.go index b2e7239ba..a57871799 100644 --- a/cmd/crypto/kms_test.go +++ b/cmd/crypto/kms_test.go @@ -16,6 +16,7 @@ package crypto import ( "bytes" + "fmt" "path" "strings" "testing" @@ -83,3 +84,32 @@ func TestContextWriteTo(t *testing.T) { } } } + +func TestContextAppendTo(t *testing.T) { + for i, test := range contextWriteToTests { + dst := make([]byte, 0, 1024) + dst = test.Context.AppendTo(dst) + if s := string(dst); s != test.ExpectedJSON { + t.Errorf("Test %d: JSON representation differ - got: '%s' want: '%s'", i, s, test.ExpectedJSON) + } + // Append one more + dst = test.Context.AppendTo(dst) + if s := string(dst); s != test.ExpectedJSON+test.ExpectedJSON { + t.Errorf("Test %d: JSON representation differ - got: '%s' want: '%s'", i, s, test.ExpectedJSON+test.ExpectedJSON) + } + } +} + +func BenchmarkContext_AppendTo(b *testing.B) { + tests := []Context{{}, {"bucket": "warp-benchmark-bucket"}, {"0": "1", "-": "2", ".": "#"}, {"34trg": "dfioutr89", "ikjfdghkjf": "jkedfhgfjkhg", "sdfhsdjkh": "if88889", "asddsirfh804": "kjfdshgdfuhgfg78-45604586#$%"}} + for _, test := range tests { + b.Run(fmt.Sprintf("%d-elems", len(test)), func(b *testing.B) { + dst := make([]byte, 0, 1024) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + dst = test.AppendTo(dst[:0]) + } + }) + } +} diff --git a/cmd/crypto/metadata.go b/cmd/crypto/metadata.go index 42ceb13ca..5887c7064 100644 --- a/cmd/crypto/metadata.go +++ b/cmd/crypto/metadata.go @@ -204,15 +204,17 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte } // Check whether all extracted values are well-formed - iv, err := base64.StdEncoding.DecodeString(b64IV) - if err != nil || len(iv) != 32 { + var iv [32]byte + n, err := base64.StdEncoding.Decode(iv[:], []byte(b64IV)) + if err != nil || n != 32 { return keyID, kmsKey, sealedKey, errInvalidInternalIV } if algorithm != SealAlgorithm { return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm } - encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey) - if err != nil || len(encryptedKey) != 64 { + var encryptedKey [64]byte + n, err = base64.StdEncoding.Decode(encryptedKey[:], []byte(b64SealedKey)) + if err != nil || n != 64 { return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid") } if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key. @@ -223,8 +225,8 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte } sealedKey.Algorithm = algorithm - copy(sealedKey.IV[:], iv) - copy(sealedKey.Key[:], encryptedKey) + sealedKey.IV = iv + sealedKey.Key = encryptedKey return keyID, kmsKey, sealedKey, nil } diff --git a/cmd/crypto/vault.go b/cmd/crypto/vault.go index 007623daa..09550eb21 100644 --- a/cmd/crypto/vault.go +++ b/cmd/crypto/vault.go @@ -15,7 +15,6 @@ package crypto import ( - "bytes" "encoding/base64" "errors" "fmt" @@ -224,11 +223,10 @@ func (v *vaultService) CreateKey(keyID string) error { // named key referenced by keyID. It also binds the generated key // cryptographically to the provided context. func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) { - var contextStream bytes.Buffer - ctx.WriteTo(&contextStream) + context := ctx.AppendTo(make([]byte, 0, 128)) payload := map[string]interface{}{ - "context": base64.StdEncoding.EncodeToString(contextStream.Bytes()), + "context": base64.StdEncoding.EncodeToString(context), } s, err := v.client.Logical().Write(fmt.Sprintf("/transit/datakey/plaintext/%s", keyID), payload) if err != nil { @@ -260,12 +258,11 @@ func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea // The context must be same context as the one provided while // generating the plaintext key / sealedKey. func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) { - var contextStream bytes.Buffer - ctx.WriteTo(&contextStream) + context := ctx.AppendTo(make([]byte, 0, 128)) payload := map[string]interface{}{ "ciphertext": string(sealedKey), - "context": base64.StdEncoding.EncodeToString(contextStream.Bytes()), + "context": base64.StdEncoding.EncodeToString(context), } s, err := v.client.Logical().Write(fmt.Sprintf("/transit/decrypt/%s", keyID), payload) @@ -294,12 +291,11 @@ func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k // The context must be same context as the one provided while // generating the plaintext key / sealedKey. func (v *vaultService) UpdateKey(keyID string, sealedKey []byte, ctx Context) (rotatedKey []byte, err error) { - var contextStream bytes.Buffer - ctx.WriteTo(&contextStream) + context := ctx.AppendTo(make([]byte, 0, 128)) payload := map[string]interface{}{ "ciphertext": string(sealedKey), - "context": base64.StdEncoding.EncodeToString(contextStream.Bytes()), + "context": base64.StdEncoding.EncodeToString(context), } s, err := v.client.Logical().Write(fmt.Sprintf("/transit/rewrap/%s", keyID), payload) if err != nil { diff --git a/cmd/encryption-v1_test.go b/cmd/encryption-v1_test.go index 88ea2ab1a..b8e2de488 100644 --- a/cmd/encryption-v1_test.go +++ b/cmd/encryption-v1_test.go @@ -19,10 +19,14 @@ package cmd import ( "bytes" "encoding/base64" + "encoding/json" + "fmt" "net/http" + "os" "testing" humanize "github.com/dustin/go-humanize" + "github.com/klauspost/compress/zstd" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/minio/minio/cmd/crypto" "github.com/minio/sio" @@ -622,3 +626,89 @@ func TestGetDefaultOpts(t *testing.T) { } } } +func Test_decryptObjectInfo(t *testing.T) { + var testSet []struct { + Bucket string + Name string + UserDef map[string]string + } + file, err := os.Open("testdata/decryptObjectInfo.json.zst") + if err != nil { + t.Fatal(err) + } + defer file.Close() + dec, err := zstd.NewReader(file) + if err != nil { + t.Fatal(err) + } + defer dec.Close() + js := json.NewDecoder(dec) + err = js.Decode(&testSet) + if err != nil { + t.Fatal(err) + } + + os.Setenv("MINIO_KMS_MASTER_KEY", "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574") + defer os.Setenv("MINIO_KMS_MASTER_KEY", "") + GlobalKMS, err = crypto.NewKMS(crypto.KMSConfig{}) + if err != nil { + t.Fatal(err) + } + + var dst [32]byte + for i := range testSet { + t.Run(fmt.Sprint("case-", i), func(t *testing.T) { + test := &testSet[i] + _, err := decryptObjectInfo(dst[:], test.Bucket, test.Name, test.UserDef) + if err != nil { + t.Fatal(err) + } + }) + } +} + +func Benchmark_decryptObjectInfo(b *testing.B) { + var testSet []struct { + Bucket string + Name string + UserDef map[string]string + } + file, err := os.Open("testdata/decryptObjectInfo.json.zst") + if err != nil { + b.Fatal(err) + } + defer file.Close() + dec, err := zstd.NewReader(file) + if err != nil { + b.Fatal(err) + } + defer dec.Close() + js := json.NewDecoder(dec) + err = js.Decode(&testSet) + if err != nil { + b.Fatal(err) + } + + os.Setenv("MINIO_KMS_MASTER_KEY", "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574") + defer os.Setenv("MINIO_KMS_MASTER_KEY", "") + GlobalKMS, err = crypto.NewKMS(crypto.KMSConfig{}) + if err != nil { + b.Fatal(err) + } + + b.ReportAllocs() + b.ResetTimer() + b.SetBytes(int64(len(testSet))) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + var dst [32]byte + for i := range testSet { + test := &testSet[i] + _, err := decryptObjectInfo(dst[:], test.Bucket, test.Name, test.UserDef) + if err != nil { + b.Fatal(err) + } + } + } + }) +} diff --git a/cmd/testdata/decryptObjectInfo.json.zst b/cmd/testdata/decryptObjectInfo.json.zst new file mode 100644 index 000000000..c27a82798 Binary files /dev/null and b/cmd/testdata/decryptObjectInfo.json.zst differ diff --git a/go.mod b/go.mod index fa53ef6c8..c12ba1abe 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( github.com/minio/selfupdate v0.3.1 github.com/minio/sha256-simd v0.1.1 github.com/minio/simdjson-go v0.1.5 - github.com/minio/sio v0.2.0 + github.com/minio/sio v0.2.1 github.com/mitchellh/go-homedir v1.1.0 github.com/mmcloughlin/avo v0.0.0-20200803215136-443f81d77104 // indirect github.com/montanaflynn/stats v0.5.0 diff --git a/go.sum b/go.sum index b5d0789ac..650afa283 100644 --- a/go.sum +++ b/go.sum @@ -318,6 +318,8 @@ github.com/minio/simdjson-go v0.1.5 h1:6T5mHh7r3kUvgwhmFWQAjoPV5Yt5oD/VPjAI9ViH1 github.com/minio/simdjson-go v0.1.5/go.mod h1:oKURrZZEBtqObgJrSjN1Ln2n9MJj2icuBTkeJzZnvSI= github.com/minio/sio v0.2.0 h1:NCRCFLx0r5pRbXf65LVNjxbCGZgNQvNFQkgX3XF4BoA= github.com/minio/sio v0.2.0/go.mod h1:nKM5GIWSrqbOZp0uhyj6M1iA0X6xQzSGtYSaTKSCut0= +github.com/minio/sio v0.2.1 h1:NjzKiIMSMcHediVQR0AFVx2tp7Wxh9tKPfDI3kH7aHQ= +github.com/minio/sio v0.2.1/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=