maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add encryption buffer (#8626)

Browse source 
Quite hard to measure difference:

```
λ warp cmp put-before.csv.zst put-after2.csv.zst
Operation: PUT
Operations: 340 -> 353
* Average: +4.11% (+22.7 MB/s) throughput, +4.11% (+0.2) obj/s
* 50% Median: +1.58% (+7.3 MB/s) throughput, +1.58% (+0.1) obj/s
```

Difference is likely bigger on Intel platforms due to higher syscall costs.
This commit is contained in:
Klaus Post 2019-12-12 10:01:15 -08:00 committed by Harshavardhana
parent abc266caa1
commit 3211cb5df6
2 changed files with 20 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
cmd/encryption-v1.go
View file

@ -17,6 +17,7 @@
package cmd
import (
"bufio"
"context"
"crypto/hmac"
"crypto/rand"
@ -225,10 +226,8 @@ func setEncryptionMetadata(r *http.Request, bucket, object string, metadata map[
// with the client provided key. It also marks the object as client-side-encrypted
// and sets the correct headers.
func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, metadata map[string]string) (reader io.Reader, objEncKey []byte, err error) {
var key []byte
var (
key []byte
)
if crypto.S3.IsRequested(r.Header) && crypto.SSEC.IsRequested(r.Header) {
return nil, objEncKey, crypto.ErrIncompatibleEncryptionMethod
}
@ -238,6 +237,11 @@ func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, m
return nil, objEncKey, err
}
}
if r.ContentLength > encryptBufferThreshold {
// The encryption reads in blocks of 64KB.
// We add a buffer on bigger files to reduce the number of syscalls upstream.
content = bufio.NewReaderSize(content, encryptBufferSize)
}
return newEncryptReader(content, key, bucket, object, metadata, crypto.S3.IsRequested(r.Header))
}

14
cmd/object-handlers.go
View file

@ -17,6 +17,7 @@
package cmd
import (
"bufio"
"context"
"crypto/hmac"
"encoding/binary"
@ -64,6 +65,11 @@ var supportedHeadGetReqParams = map[string]string{
const (
compressionAlgorithmV1 = "golang/snappy/LZ77"
compressionAlgorithmV2 = "klauspost/compress/s2"
// When an upload exceeds encryptBufferThreshold ...
encryptBufferThreshold = 1 << 20
// add an input buffer of this size.
encryptBufferSize = 1 << 20
)
// setHeadGetRespHeaders - set any requested parameters as response headers.
@ -1975,7 +1981,13 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
mac.Write(partIDbin[:])
partEncryptionKey := mac.Sum(nil)
reader, err = sio.EncryptReader(hashReader, sio.Config{Key: partEncryptionKey})
in := io.Reader(hashReader)
if size > encryptBufferThreshold {
// The encryption reads in blocks of 64KB.
// We add a buffer on bigger files to reduce the number of syscalls upstream.
in = bufio.NewReaderSize(hashReader, encryptBufferSize)
}
reader, err = sio.EncryptReader(in, sio.Config{Key: partEncryptionKey})
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return