minio/cmd/encryption-v1.go
Harshavardhana e79874f58e
[feat] Preserve version supplied by client (#9854)
Just like GET/DELETE APIs it is possible to preserve
client supplied versionId's, of course the versionIds
have to be uuid, if an existing versionId is found
it is overwritten if no object locking policies
are found.

- PUT /bucketname/objectname?versionId=<id>
- POST /bucketname/objectname?uploads=&versionId=<id>
- PUT /bucketname/objectname?verisonId=<id> (with x-amz-copy-source)
2020-06-17 11:13:41 -07:00

1054 lines
33 KiB
Go

/*
* MinIO Cloud Storage, (C) 2017, 2018 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"bufio"
"context"
"crypto/hmac"
"crypto/rand"
"crypto/subtle"
"encoding/binary"
"encoding/hex"
"errors"
"io"
"net/http"
"path"
"strconv"
"strings"
"github.com/google/uuid"
"github.com/minio/minio-go/v6/pkg/encrypt"
"github.com/minio/minio/cmd/crypto"
"github.com/minio/minio/cmd/logger"
sha256 "github.com/minio/sha256-simd"
"github.com/minio/sio"
)
var (
// AWS errors for invalid SSE-C requests.
errEncryptedObject = errors.New("The object was stored using a form of SSE")
errInvalidSSEParameters = errors.New("The SSE-C key for key-rotation is not correct") // special access denied
errKMSNotConfigured = errors.New("KMS not configured for a server side encrypted object")
// Additional MinIO errors for SSE-C requests.
errObjectTampered = errors.New("The requested object was modified and may be compromised")
// error returned when invalid encryption parameters are specified
errInvalidEncryptionParameters = errors.New("The encryption parameters are not applicable to this object")
)
const (
// SSECustomerKeySize is the size of valid client provided encryption keys in bytes.
// Currently AWS supports only AES256. So the SSE-C key size is fixed to 32 bytes.
SSECustomerKeySize = 32
// SSEIVSize is the size of the IV data
SSEIVSize = 32 // 32 bytes
// SSEDAREPackageBlockSize - SSE dare package block size.
SSEDAREPackageBlockSize = 64 * 1024 // 64KiB bytes
// SSEDAREPackageMetaSize - SSE dare package meta padding bytes.
SSEDAREPackageMetaSize = 32 // 32 bytes
)
// isEncryptedMultipart returns true if the current object is
// uploaded by the user using multipart mechanism:
// initiate new multipart, upload part, complete upload
func isEncryptedMultipart(objInfo ObjectInfo) bool {
if len(objInfo.Parts) == 0 {
return false
}
if !crypto.IsMultiPart(objInfo.UserDefined) {
return false
}
for _, part := range objInfo.Parts {
_, err := sio.DecryptedSize(uint64(part.Size))
if err != nil {
return false
}
}
// Further check if this object is uploaded using multipart mechanism
// by the user and it is not about Erasure internally splitting the
// object into parts in PutObject()
return !(objInfo.backendType == BackendErasure && len(objInfo.ETag) == 32)
}
// ParseSSECopyCustomerRequest parses the SSE-C header fields of the provided request.
// It returns the client provided key on success.
func ParseSSECopyCustomerRequest(h http.Header, metadata map[string]string) (key []byte, err error) {
if crypto.S3.IsEncrypted(metadata) && crypto.SSECopy.IsRequested(h) {
return nil, crypto.ErrIncompatibleEncryptionMethod
}
k, err := crypto.SSECopy.ParseHTTP(h)
return k[:], err
}
// ParseSSECustomerRequest parses the SSE-C header fields of the provided request.
// It returns the client provided key on success.
func ParseSSECustomerRequest(r *http.Request) (key []byte, err error) {
return ParseSSECustomerHeader(r.Header)
}
// ParseSSECustomerHeader parses the SSE-C header fields and returns
// the client provided key on success.
func ParseSSECustomerHeader(header http.Header) (key []byte, err error) {
if crypto.S3.IsRequested(header) && crypto.SSEC.IsRequested(header) {
return key, crypto.ErrIncompatibleEncryptionMethod
}
k, err := crypto.SSEC.ParseHTTP(header)
return k[:], err
}
// This function rotates old to new key.
func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map[string]string) error {
switch {
default:
return errObjectTampered
case crypto.SSEC.IsEncrypted(metadata):
sealedKey, err := crypto.SSEC.ParseMetadata(metadata)
if err != nil {
return err
}
var objectKey crypto.ObjectKey
var extKey [32]byte
copy(extKey[:], oldKey)
if err = objectKey.Unseal(extKey, sealedKey, crypto.SSEC.String(), bucket, object); err != nil {
if subtle.ConstantTimeCompare(oldKey, newKey) == 1 {
return errInvalidSSEParameters // AWS returns special error for equal but invalid keys.
}
return crypto.ErrInvalidCustomerKey // To provide strict AWS S3 compatibility we return: access denied.
}
if subtle.ConstantTimeCompare(oldKey, newKey) == 1 && sealedKey.Algorithm == crypto.SealAlgorithm {
return nil // don't rotate on equal keys if seal algorithm is latest
}
copy(extKey[:], newKey)
sealedKey = objectKey.Seal(extKey, sealedKey.IV, crypto.SSEC.String(), bucket, object)
crypto.SSEC.CreateMetadata(metadata, sealedKey)
return nil
case crypto.S3.IsEncrypted(metadata):
if GlobalKMS == nil {
return errKMSNotConfigured
}
keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(metadata)
if err != nil {
return err
}
oldKey, err := GlobalKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
if err != nil {
return err
}
var objectKey crypto.ObjectKey
if err = objectKey.Unseal(oldKey, sealedKey, crypto.S3.String(), bucket, object); err != nil {
return err
}
newKey, encKey, err := GlobalKMS.GenerateKey(GlobalKMS.KeyID(), crypto.Context{bucket: path.Join(bucket, object)})
if err != nil {
return err
}
sealedKey = objectKey.Seal(newKey, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
crypto.S3.CreateMetadata(metadata, GlobalKMS.KeyID(), encKey, sealedKey)
return nil
}
}
func newEncryptMetadata(key []byte, bucket, object string, metadata map[string]string, sseS3 bool) (crypto.ObjectKey, error) {
var sealedKey crypto.SealedKey
if sseS3 {
if GlobalKMS == nil {
return crypto.ObjectKey{}, errKMSNotConfigured
}
key, encKey, err := GlobalKMS.GenerateKey(GlobalKMS.KeyID(), crypto.Context{bucket: path.Join(bucket, object)})
if err != nil {
return crypto.ObjectKey{}, err
}
objectKey := crypto.GenerateKey(key, rand.Reader)
sealedKey = objectKey.Seal(key, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
crypto.S3.CreateMetadata(metadata, GlobalKMS.KeyID(), encKey, sealedKey)
return objectKey, nil
}
var extKey [32]byte
copy(extKey[:], key)
objectKey := crypto.GenerateKey(extKey, rand.Reader)
sealedKey = objectKey.Seal(extKey, crypto.GenerateIV(rand.Reader), crypto.SSEC.String(), bucket, object)
crypto.SSEC.CreateMetadata(metadata, sealedKey)
return objectKey, nil
}
func newEncryptReader(content io.Reader, key []byte, bucket, object string, metadata map[string]string, sseS3 bool) (io.Reader, crypto.ObjectKey, error) {
objectEncryptionKey, err := newEncryptMetadata(key, bucket, object, metadata, sseS3)
if err != nil {
return nil, crypto.ObjectKey{}, err
}
reader, err := sio.EncryptReader(content, sio.Config{Key: objectEncryptionKey[:], MinVersion: sio.Version20})
if err != nil {
return nil, crypto.ObjectKey{}, crypto.ErrInvalidCustomerKey
}
return reader, objectEncryptionKey, nil
}
// set new encryption metadata from http request headers for SSE-C and generated key from KMS in the case of
// SSE-S3
func setEncryptionMetadata(r *http.Request, bucket, object string, metadata map[string]string) (err error) {
var (
key []byte
)
if crypto.SSEC.IsRequested(r.Header) {
key, err = ParseSSECustomerRequest(r)
if err != nil {
return
}
}
_, err = newEncryptMetadata(key, bucket, object, metadata, crypto.S3.IsRequested(r.Header))
return
}
// EncryptRequest takes the client provided content and encrypts the data
// with the client provided key. It also marks the object as client-side-encrypted
// and sets the correct headers.
func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, metadata map[string]string) (io.Reader, crypto.ObjectKey, error) {
if crypto.S3.IsRequested(r.Header) && crypto.SSEC.IsRequested(r.Header) {
return nil, crypto.ObjectKey{}, crypto.ErrIncompatibleEncryptionMethod
}
if r.ContentLength > encryptBufferThreshold {
// The encryption reads in blocks of 64KB.
// We add a buffer on bigger files to reduce the number of syscalls upstream.
content = bufio.NewReaderSize(content, encryptBufferSize)
}
var key []byte
if crypto.SSEC.IsRequested(r.Header) {
var err error
key, err = ParseSSECustomerRequest(r)
if err != nil {
return nil, crypto.ObjectKey{}, err
}
}
return newEncryptReader(content, key, bucket, object, metadata, crypto.S3.IsRequested(r.Header))
}
func decryptObjectInfo(key []byte, bucket, object string, metadata map[string]string) ([]byte, error) {
switch {
default:
return nil, errObjectTampered
case crypto.S3.IsEncrypted(metadata) && isCacheEncrypted(metadata):
if globalCacheKMS == nil {
return nil, errKMSNotConfigured
}
keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(metadata)
if err != nil {
return nil, err
}
extKey, err := globalCacheKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
if err != nil {
return nil, err
}
var objectKey crypto.ObjectKey
if err = objectKey.Unseal(extKey, sealedKey, crypto.S3.String(), bucket, object); err != nil {
return nil, err
}
return objectKey[:], nil
case crypto.S3.IsEncrypted(metadata):
if GlobalKMS == nil {
return nil, errKMSNotConfigured
}
keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(metadata)
if err != nil {
return nil, err
}
extKey, err := GlobalKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
if err != nil {
return nil, err
}
var objectKey crypto.ObjectKey
if err = objectKey.Unseal(extKey, sealedKey, crypto.S3.String(), bucket, object); err != nil {
return nil, err
}
return objectKey[:], nil
case crypto.SSEC.IsEncrypted(metadata):
var extKey [32]byte
copy(extKey[:], key)
sealedKey, err := crypto.SSEC.ParseMetadata(metadata)
if err != nil {
return nil, err
}
var objectKey crypto.ObjectKey
if err = objectKey.Unseal(extKey, sealedKey, crypto.SSEC.String(), bucket, object); err != nil {
return nil, err
}
return objectKey[:], nil
}
}
// Adding support for reader based interface
// DecryptRequestWithSequenceNumberR - same as
// DecryptRequestWithSequenceNumber but with a reader
func DecryptRequestWithSequenceNumberR(client io.Reader, h http.Header, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
if crypto.S3.IsEncrypted(metadata) {
return newDecryptReader(client, nil, bucket, object, seqNumber, metadata)
}
key, err := ParseSSECustomerHeader(h)
if err != nil {
return nil, err
}
return newDecryptReader(client, key, bucket, object, seqNumber, metadata)
}
// DecryptCopyRequestR - same as DecryptCopyRequest, but with a
// Reader
func DecryptCopyRequestR(client io.Reader, h http.Header, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
var (
key []byte
err error
)
if crypto.SSECopy.IsRequested(h) {
key, err = ParseSSECopyCustomerRequest(h, metadata)
if err != nil {
return nil, err
}
}
return newDecryptReader(client, key, bucket, object, seqNumber, metadata)
}
func newDecryptReader(client io.Reader, key []byte, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
objectEncryptionKey, err := decryptObjectInfo(key, bucket, object, metadata)
if err != nil {
return nil, err
}
return newDecryptReaderWithObjectKey(client, objectEncryptionKey, seqNumber)
}
func newDecryptReaderWithObjectKey(client io.Reader, objectEncryptionKey []byte, seqNumber uint32) (io.Reader, error) {
reader, err := sio.DecryptReader(client, sio.Config{
Key: objectEncryptionKey,
SequenceNumber: seqNumber,
})
if err != nil {
return nil, crypto.ErrInvalidCustomerKey
}
return reader, nil
}
// DecryptBlocksRequestR - same as DecryptBlocksRequest but with a
// reader
func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
length int64, seqNumber uint32, partStart int, oi ObjectInfo, copySource bool) (
io.Reader, error) {
bucket, object := oi.Bucket, oi.Name
// Single part case
if !isEncryptedMultipart(oi) {
var reader io.Reader
var err error
if copySource {
reader, err = DecryptCopyRequestR(inputReader, h, bucket, object, seqNumber, oi.UserDefined)
} else {
reader, err = DecryptRequestWithSequenceNumberR(inputReader, h, bucket, object, seqNumber, oi.UserDefined)
}
if err != nil {
return nil, err
}
return reader, nil
}
partDecRelOffset := int64(seqNumber) * SSEDAREPackageBlockSize
partEncRelOffset := int64(seqNumber) * (SSEDAREPackageBlockSize + SSEDAREPackageMetaSize)
w := &DecryptBlocksReader{
reader: inputReader,
startSeqNum: seqNumber,
partDecRelOffset: partDecRelOffset,
partEncRelOffset: partEncRelOffset,
parts: oi.Parts,
partIndex: partStart,
header: h,
bucket: bucket,
object: object,
customerKeyHeader: h.Get(crypto.SSECKey),
copySource: copySource,
}
w.metadata = map[string]string{}
// Copy encryption metadata for internal use.
for k, v := range oi.UserDefined {
w.metadata[k] = v
}
if w.copySource {
w.customerKeyHeader = h.Get(crypto.SSECopyKey)
}
if err := w.buildDecrypter(w.parts[w.partIndex].Number); err != nil {
return nil, err
}
return w, nil
}
// DecryptBlocksReader - decrypts multipart parts, while implementing
// a io.Reader compatible interface.
type DecryptBlocksReader struct {
// Source of the encrypted content that will be decrypted
reader io.Reader
// Current decrypter for the current encrypted data block
decrypter io.Reader
// Start sequence number
startSeqNum uint32
// Current part index
partIndex int
// Parts information
parts []ObjectPartInfo
header http.Header
bucket, object string
metadata map[string]string
partDecRelOffset, partEncRelOffset int64
copySource bool
// Customer Key
customerKeyHeader string
}
func (d *DecryptBlocksReader) buildDecrypter(partID int) error {
m := make(map[string]string)
for k, v := range d.metadata {
m[k] = v
}
// Initialize the first decrypter; new decrypters will be
// initialized in Read() operation as needed.
var key []byte
var err error
if d.copySource {
if crypto.SSEC.IsEncrypted(d.metadata) {
d.header.Set(crypto.SSECopyKey, d.customerKeyHeader)
key, err = ParseSSECopyCustomerRequest(d.header, d.metadata)
}
} else {
if crypto.SSEC.IsEncrypted(d.metadata) {
d.header.Set(crypto.SSECKey, d.customerKeyHeader)
key, err = ParseSSECustomerHeader(d.header)
}
}
if err != nil {
return err
}
objectEncryptionKey, err := decryptObjectInfo(key, d.bucket, d.object, m)
if err != nil {
return err
}
var partIDbin [4]byte
binary.LittleEndian.PutUint32(partIDbin[:], uint32(partID)) // marshal part ID
mac := hmac.New(sha256.New, objectEncryptionKey) // derive part encryption key from part ID and object key
mac.Write(partIDbin[:])
partEncryptionKey := mac.Sum(nil)
// Limit the reader, so the decryptor doesnt receive bytes
// from the next part (different DARE stream)
encLenToRead := d.parts[d.partIndex].Size - d.partEncRelOffset
decrypter, err := newDecryptReaderWithObjectKey(io.LimitReader(d.reader, encLenToRead), partEncryptionKey, d.startSeqNum)
if err != nil {
return err
}
d.decrypter = decrypter
return nil
}
func (d *DecryptBlocksReader) Read(p []byte) (int, error) {
var err error
var n1 int
decPartSize, _ := sio.DecryptedSize(uint64(d.parts[d.partIndex].Size))
unreadPartLen := int64(decPartSize) - d.partDecRelOffset
if int64(len(p)) < unreadPartLen {
n1, err = d.decrypter.Read(p)
if err != nil {
return 0, err
}
d.partDecRelOffset += int64(n1)
} else {
n1, err = io.ReadFull(d.decrypter, p[:unreadPartLen])
if err != nil {
return 0, err
}
// We should now proceed to next part, reset all
// values appropriately.
d.partEncRelOffset = 0
d.partDecRelOffset = 0
d.startSeqNum = 0
d.partIndex++
if d.partIndex == len(d.parts) {
return n1, io.EOF
}
err = d.buildDecrypter(d.parts[d.partIndex].Number)
if err != nil {
return 0, err
}
n1, err = d.decrypter.Read(p[n1:])
if err != nil {
return 0, err
}
d.partDecRelOffset += int64(n1)
}
return len(p), nil
}
// DecryptedSize returns the size of the object after decryption in bytes.
// It returns an error if the object is not encrypted or marked as encrypted
// but has an invalid size.
func (o *ObjectInfo) DecryptedSize() (int64, error) {
if !crypto.IsEncrypted(o.UserDefined) {
return 0, errors.New("Cannot compute decrypted size of an unencrypted object")
}
if !isEncryptedMultipart(*o) {
size, err := sio.DecryptedSize(uint64(o.Size))
if err != nil {
err = errObjectTampered // assign correct error type
}
return int64(size), err
}
var size int64
for _, part := range o.Parts {
partSize, err := sio.DecryptedSize(uint64(part.Size))
if err != nil {
return 0, errObjectTampered
}
size += int64(partSize)
}
return size, nil
}
// DecryptETag decrypts the ETag that is part of given object
// with the given object encryption key.
//
// However, DecryptETag does not try to decrypt the ETag if
// it consists of a 128 bit hex value (32 hex chars) and exactly
// one '-' followed by a 32-bit number.
// This special case adresses randomly-generated ETags generated
// by the MinIO server when running in non-compat mode. These
// random ETags are not encrypt.
//
// Calling DecryptETag with a non-randomly generated ETag will
// fail.
func DecryptETag(key crypto.ObjectKey, object ObjectInfo) (string, error) {
if n := strings.Count(object.ETag, "-"); n > 0 {
if n != 1 {
return "", errObjectTampered
}
i := strings.IndexByte(object.ETag, '-')
if len(object.ETag[:i]) != 32 {
return "", errObjectTampered
}
if _, err := hex.DecodeString(object.ETag[:32]); err != nil {
return "", errObjectTampered
}
if _, err := strconv.ParseInt(object.ETag[i+1:], 10, 32); err != nil {
return "", errObjectTampered
}
return object.ETag, nil
}
etag, err := hex.DecodeString(object.ETag)
if err != nil {
return "", err
}
etag, err = key.UnsealETag(etag)
if err != nil {
return "", err
}
return hex.EncodeToString(etag), nil
}
// For encrypted objects, the ETag sent by client if available
// is stored in encrypted form in the backend. Decrypt the ETag
// if ETag was previously encrypted.
func getDecryptedETag(headers http.Header, objInfo ObjectInfo, copySource bool) (decryptedETag string) {
var (
key [32]byte
err error
)
// If ETag is contentMD5Sum return it as is.
if len(objInfo.ETag) == 32 {
return objInfo.ETag
}
if crypto.IsMultiPart(objInfo.UserDefined) {
return objInfo.ETag
}
if crypto.SSECopy.IsRequested(headers) {
key, err = crypto.SSECopy.ParseHTTP(headers)
if err != nil {
return objInfo.ETag
}
}
// As per AWS S3 Spec, ETag for SSE-C encrypted objects need not be MD5Sum of the data.
// Since server side copy with same source and dest just replaces the ETag, we save
// encrypted content MD5Sum as ETag for both SSE-C and SSE-S3, we standardize the ETag
//encryption across SSE-C and SSE-S3, and only return last 32 bytes for SSE-C
if crypto.SSEC.IsEncrypted(objInfo.UserDefined) && !copySource {
return objInfo.ETag[len(objInfo.ETag)-32:]
}
objectEncryptionKey, err := decryptObjectInfo(key[:], objInfo.Bucket, objInfo.Name, objInfo.UserDefined)
if err != nil {
return objInfo.ETag
}
return tryDecryptETag(objectEncryptionKey, objInfo.ETag, false)
}
// helper to decrypt Etag given object encryption key and encrypted ETag
func tryDecryptETag(key []byte, encryptedETag string, ssec bool) string {
// ETag for SSE-C encrypted objects need not be content MD5Sum.While encrypted
// md5sum is stored internally, return just the last 32 bytes of hex-encoded and
// encrypted md5sum string for SSE-C
if ssec {
return encryptedETag[len(encryptedETag)-32:]
}
var objectKey crypto.ObjectKey
copy(objectKey[:], key)
encBytes, err := hex.DecodeString(encryptedETag)
if err != nil {
return encryptedETag
}
etagBytes, err := objectKey.UnsealETag(encBytes)
if err != nil {
return encryptedETag
}
return hex.EncodeToString(etagBytes)
}
// GetDecryptedRange - To decrypt the range (off, length) of the
// decrypted object stream, we need to read the range (encOff,
// encLength) of the encrypted object stream to decrypt it, and
// compute skipLen, the number of bytes to skip in the beginning of
// the encrypted range.
//
// In addition we also compute the object part number for where the
// requested range starts, along with the DARE sequence number within
// that part. For single part objects, the partStart will be 0.
func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, skipLen int64, seqNumber uint32, partStart int, err error) {
if !crypto.IsEncrypted(o.UserDefined) {
err = errors.New("Object is not encrypted")
return
}
if rs == nil {
// No range, so offsets refer to the whole object.
return 0, int64(o.Size), 0, 0, 0, nil
}
// Assemble slice of (decrypted) part sizes in `sizes`
var sizes []int64
var decObjSize int64 // decrypted total object size
if isEncryptedMultipart(*o) {
sizes = make([]int64, len(o.Parts))
for i, part := range o.Parts {
var partSize uint64
partSize, err = sio.DecryptedSize(uint64(part.Size))
if err != nil {
err = errObjectTampered
return
}
sizes[i] = int64(partSize)
decObjSize += int64(partSize)
}
} else {
var partSize uint64
partSize, err = sio.DecryptedSize(uint64(o.Size))
if err != nil {
err = errObjectTampered
return
}
sizes = []int64{int64(partSize)}
decObjSize = sizes[0]
}
var off, length int64
off, length, err = rs.GetOffsetLength(decObjSize)
if err != nil {
return
}
// At this point, we have:
//
// 1. the decrypted part sizes in `sizes` (single element for
// single part object) and total decrypted object size `decObjSize`
//
// 2. the (decrypted) start offset `off` and (decrypted)
// length to read `length`
//
// These are the inputs to the rest of the algorithm below.
// Locate the part containing the start of the required range
var partEnd int
var cumulativeSum, encCumulativeSum int64
for i, size := range sizes {
if off < cumulativeSum+size {
partStart = i
break
}
cumulativeSum += size
encPartSize, _ := sio.EncryptedSize(uint64(size))
encCumulativeSum += int64(encPartSize)
}
// partStart is always found in the loop above,
// because off is validated.
sseDAREEncPackageBlockSize := int64(SSEDAREPackageBlockSize + SSEDAREPackageMetaSize)
startPkgNum := (off - cumulativeSum) / SSEDAREPackageBlockSize
// Now we can calculate the number of bytes to skip
skipLen = (off - cumulativeSum) % SSEDAREPackageBlockSize
encOff = encCumulativeSum + startPkgNum*sseDAREEncPackageBlockSize
// Locate the part containing the end of the required range
endOffset := off + length - 1
for i1, size := range sizes[partStart:] {
i := partStart + i1
if endOffset < cumulativeSum+size {
partEnd = i
break
}
cumulativeSum += size
encPartSize, _ := sio.EncryptedSize(uint64(size))
encCumulativeSum += int64(encPartSize)
}
// partEnd is always found in the loop above, because off and
// length are validated.
endPkgNum := (endOffset - cumulativeSum) / SSEDAREPackageBlockSize
// Compute endEncOffset with one additional DARE package (so
// we read the package containing the last desired byte).
endEncOffset := encCumulativeSum + (endPkgNum+1)*sseDAREEncPackageBlockSize
// Check if the DARE package containing the end offset is a
// full sized package (as the last package in the part may be
// smaller)
lastPartSize, _ := sio.EncryptedSize(uint64(sizes[partEnd]))
if endEncOffset > encCumulativeSum+int64(lastPartSize) {
endEncOffset = encCumulativeSum + int64(lastPartSize)
}
encLength = endEncOffset - encOff
// Set the sequence number as the starting package number of
// the requested block
seqNumber = uint32(startPkgNum)
return encOff, encLength, skipLen, seqNumber, partStart, nil
}
// EncryptedSize returns the size of the object after encryption.
// An encrypted object is always larger than a plain object
// except for zero size objects.
func (o *ObjectInfo) EncryptedSize() int64 {
size, err := sio.EncryptedSize(uint64(o.Size))
if err != nil {
// This cannot happen since AWS S3 allows parts to be 5GB at most
// sio max. size is 256 TB
reqInfo := (&logger.ReqInfo{}).AppendTags("size", strconv.FormatUint(size, 10))
ctx := logger.SetReqInfo(GlobalContext, reqInfo)
logger.CriticalIf(ctx, err)
}
return int64(size)
}
// DecryptCopyObjectInfo tries to decrypt the provided object if it is encrypted.
// It fails if the object is encrypted and the HTTP headers don't contain
// SSE-C headers or the object is not encrypted but SSE-C headers are provided. (AWS behavior)
// DecryptObjectInfo returns 'ErrNone' if the object is not encrypted or the
// decryption succeeded.
//
// DecryptCopyObjectInfo also returns whether the object is encrypted or not.
func DecryptCopyObjectInfo(info *ObjectInfo, headers http.Header) (errCode APIErrorCode, encrypted bool) {
// Directories are never encrypted.
if info.IsDir {
return ErrNone, false
}
if errCode, encrypted = ErrNone, crypto.IsEncrypted(info.UserDefined); !encrypted && crypto.SSECopy.IsRequested(headers) {
errCode = ErrInvalidEncryptionParameters
} else if encrypted {
if (!crypto.SSECopy.IsRequested(headers) && crypto.SSEC.IsEncrypted(info.UserDefined)) ||
(crypto.SSECopy.IsRequested(headers) && crypto.S3.IsEncrypted(info.UserDefined)) {
errCode = ErrSSEEncryptedObject
return
}
var err error
if info.Size, err = info.DecryptedSize(); err != nil {
errCode = toAPIErrorCode(GlobalContext, err)
}
}
return
}
// DecryptObjectInfo tries to decrypt the provided object if it is encrypted.
// It fails if the object is encrypted and the HTTP headers don't contain
// SSE-C headers or the object is not encrypted but SSE-C headers are provided. (AWS behavior)
// DecryptObjectInfo returns 'ErrNone' if the object is not encrypted or the
// decryption succeeded.
//
// DecryptObjectInfo also returns whether the object is encrypted or not.
func DecryptObjectInfo(info *ObjectInfo, headers http.Header) (encrypted bool, err error) {
// Directories are never encrypted.
if info.IsDir {
return false, nil
}
// disallow X-Amz-Server-Side-Encryption header on HEAD and GET
if crypto.S3.IsRequested(headers) {
err = errInvalidEncryptionParameters
return
}
if err, encrypted = nil, crypto.IsEncrypted(info.UserDefined); !encrypted && crypto.SSEC.IsRequested(headers) {
err = errInvalidEncryptionParameters
} else if encrypted {
if (crypto.SSEC.IsEncrypted(info.UserDefined) && !crypto.SSEC.IsRequested(headers)) ||
(crypto.S3.IsEncrypted(info.UserDefined) && crypto.SSEC.IsRequested(headers)) {
err = errEncryptedObject
return
}
_, err = info.DecryptedSize()
if crypto.IsEncrypted(info.UserDefined) && !crypto.IsMultiPart(info.UserDefined) {
info.ETag = getDecryptedETag(headers, *info, false)
}
}
return
}
// The customer key in the header is used by the gateway for encryption in the case of
// s3 gateway double encryption. A new client key is derived from the customer provided
// key to be sent to the s3 backend for encryption at the backend.
func deriveClientKey(clientKey [32]byte, bucket, object string) [32]byte {
var key [32]byte
mac := hmac.New(sha256.New, clientKey[:])
mac.Write([]byte(crypto.SSEC.String()))
mac.Write([]byte(path.Join(bucket, object)))
mac.Sum(key[:0])
return key
}
// set encryption options for pass through to backend in the case of gateway and UserDefined metadata
func getDefaultOpts(header http.Header, copySource bool, metadata map[string]string) (opts ObjectOptions, err error) {
var clientKey [32]byte
var sse encrypt.ServerSide
opts = ObjectOptions{UserDefined: metadata}
if copySource {
if crypto.SSECopy.IsRequested(header) {
clientKey, err = crypto.SSECopy.ParseHTTP(header)
if err != nil {
return
}
if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil {
return
}
opts.ServerSideEncryption = encrypt.SSECopy(sse)
return
}
return
}
if crypto.SSEC.IsRequested(header) {
clientKey, err = crypto.SSEC.ParseHTTP(header)
if err != nil {
return
}
if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil {
return
}
opts.ServerSideEncryption = sse
return
}
if crypto.S3.IsRequested(header) || (metadata != nil && crypto.S3.IsEncrypted(metadata)) {
opts.ServerSideEncryption = encrypt.NewSSE()
}
return
}
// get ObjectOptions for GET calls from encryption headers
func getOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
var (
encryption encrypt.ServerSide
opts ObjectOptions
)
var partNumber int
var err error
if pn := r.URL.Query().Get("partNumber"); pn != "" {
partNumber, err = strconv.Atoi(pn)
if err != nil {
return opts, err
}
if partNumber <= 0 {
return opts, errInvalidArgument
}
}
vid := strings.TrimSpace(r.URL.Query().Get("versionId"))
if vid != "" && vid != nullVersionID {
_, err := uuid.Parse(vid)
if err != nil {
logger.LogIf(ctx, err)
return opts, VersionNotFound{
Bucket: bucket,
Object: object,
VersionID: vid,
}
}
}
if GlobalGatewaySSE.SSEC() && crypto.SSEC.IsRequested(r.Header) {
key, err := crypto.SSEC.ParseHTTP(r.Header)
if err != nil {
return opts, err
}
derivedKey := deriveClientKey(key, bucket, object)
encryption, err = encrypt.NewSSEC(derivedKey[:])
logger.CriticalIf(ctx, err)
return ObjectOptions{
ServerSideEncryption: encryption,
VersionID: vid,
PartNumber: partNumber,
}, nil
}
// default case of passing encryption headers to backend
opts, err = getDefaultOpts(r.Header, false, nil)
if err != nil {
return opts, err
}
opts.PartNumber = partNumber
opts.VersionID = vid
return opts, nil
}
func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts ObjectOptions, err error) {
versioned := globalBucketVersioningSys.Enabled(bucket)
opts, err = getOpts(ctx, r, bucket, object)
if err != nil {
return opts, err
}
opts.Versioned = versioned
return opts, nil
}
// get ObjectOptions for PUT calls from encryption headers and metadata
func putOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
versioned := globalBucketVersioningSys.Enabled(bucket)
vid := strings.TrimSpace(r.URL.Query().Get("versionId"))
if vid != "" && vid != nullVersionID {
_, err := uuid.Parse(vid)
if err != nil {
logger.LogIf(ctx, err)
return opts, VersionNotFound{
Bucket: bucket,
Object: object,
VersionID: vid,
}
}
}
// In the case of multipart custom format, the metadata needs to be checked in addition to header to see if it
// is SSE-S3 encrypted, primarily because S3 protocol does not require SSE-S3 headers in PutObjectPart calls
if GlobalGatewaySSE.SSES3() && (crypto.S3.IsRequested(r.Header) || crypto.S3.IsEncrypted(metadata)) {
return ObjectOptions{
ServerSideEncryption: encrypt.NewSSE(),
UserDefined: metadata,
VersionID: vid,
Versioned: versioned,
}, nil
}
if GlobalGatewaySSE.SSEC() && crypto.SSEC.IsRequested(r.Header) {
opts, err = getOpts(ctx, r, bucket, object)
opts.VersionID = vid
opts.Versioned = versioned
opts.UserDefined = metadata
return
}
if crypto.S3KMS.IsRequested(r.Header) {
keyID, context, err := crypto.S3KMS.ParseHTTP(r.Header)
if err != nil {
return ObjectOptions{}, err
}
sseKms, err := encrypt.NewSSEKMS(keyID, context)
if err != nil {
return ObjectOptions{}, err
}
return ObjectOptions{
ServerSideEncryption: sseKms,
UserDefined: metadata,
VersionID: vid,
Versioned: versioned,
}, nil
}
// default case of passing encryption headers and UserDefined metadata to backend
opts, err = getDefaultOpts(r.Header, false, metadata)
if err != nil {
return opts, err
}
opts.VersionID = vid
opts.Versioned = versioned
return opts, nil
}
// get ObjectOptions for Copy calls with encryption headers provided on the target side and source side metadata
func copyDstOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
return putOpts(ctx, r, bucket, object, metadata)
}
// get ObjectOptions for Copy calls with encryption headers provided on the source side
func copySrcOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
var (
ssec encrypt.ServerSide
opts ObjectOptions
)
if GlobalGatewaySSE.SSEC() && crypto.SSECopy.IsRequested(r.Header) {
key, err := crypto.SSECopy.ParseHTTP(r.Header)
if err != nil {
return opts, err
}
derivedKey := deriveClientKey(key, bucket, object)
ssec, err = encrypt.NewSSEC(derivedKey[:])
if err != nil {
return opts, err
}
return ObjectOptions{ServerSideEncryption: encrypt.SSECopy(ssec)}, nil
}
// default case of passing encryption headers to backend
opts, err := getDefaultOpts(r.Header, false, nil)
if err != nil {
return opts, err
}
return opts, nil
}