2016-04-22 19:24:04 +02:00
|
|
|
/*
|
|
|
|
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
2016-04-20 17:32:47 +02:00
|
|
|
|
2016-08-19 01:23:42 +02:00
|
|
|
package cmd
|
2016-04-20 17:32:47 +02:00
|
|
|
|
2016-06-01 05:23:31 +02:00
|
|
|
import (
|
2016-06-22 21:55:23 +02:00
|
|
|
"bytes"
|
2016-06-24 11:06:23 +02:00
|
|
|
"errors"
|
2016-06-01 05:23:31 +02:00
|
|
|
"hash"
|
|
|
|
"io"
|
2016-12-09 05:35:07 +01:00
|
|
|
"sync"
|
2016-06-01 05:23:31 +02:00
|
|
|
|
|
|
|
"github.com/klauspost/reedsolomon"
|
2016-12-22 17:25:03 +01:00
|
|
|
"github.com/minio/sha256-simd"
|
2016-12-21 23:20:01 +01:00
|
|
|
"golang.org/x/crypto/blake2b"
|
2016-06-01 05:23:31 +02:00
|
|
|
)
|
|
|
|
|
2016-06-02 01:43:31 +02:00
|
|
|
// newHashWriters - inititialize a slice of hashes for the disk count.
|
2016-07-28 11:20:34 +02:00
|
|
|
func newHashWriters(diskCount int, algo string) []hash.Hash {
|
2016-06-02 01:43:31 +02:00
|
|
|
hashWriters := make([]hash.Hash, diskCount)
|
|
|
|
for index := range hashWriters {
|
2016-07-28 11:20:34 +02:00
|
|
|
hashWriters[index] = newHash(algo)
|
2016-06-02 01:43:31 +02:00
|
|
|
}
|
|
|
|
return hashWriters
|
|
|
|
}
|
|
|
|
|
2016-06-01 05:23:31 +02:00
|
|
|
// newHash - gives you a newly allocated hash depending on the input algorithm.
|
2016-12-22 17:25:03 +01:00
|
|
|
func newHash(algo string) (h hash.Hash) {
|
2016-06-01 05:23:31 +02:00
|
|
|
switch algo {
|
2016-12-22 17:25:03 +01:00
|
|
|
case "sha256":
|
|
|
|
// sha256 checksum specially on ARM64 platforms or whenever
|
|
|
|
// requested as dictated by `xl.json` entry.
|
|
|
|
h = sha256.New()
|
2016-06-23 02:13:26 +02:00
|
|
|
case "blake2b":
|
2016-12-21 23:20:01 +01:00
|
|
|
// ignore the error, because New512 without a key never fails
|
|
|
|
// New512 only returns a non-nil error, if the length of the passed
|
2016-12-22 19:47:10 +01:00
|
|
|
// key > 64 bytes - but we use blake2b as hash function (no key)
|
2016-12-22 17:25:03 +01:00
|
|
|
h, _ = blake2b.New512(nil)
|
2016-06-01 05:23:31 +02:00
|
|
|
// Add new hashes here.
|
|
|
|
default:
|
2016-06-23 02:13:26 +02:00
|
|
|
// Default to blake2b.
|
2016-12-21 23:20:01 +01:00
|
|
|
// ignore the error, because New512 without a key never fails
|
|
|
|
// New512 only returns a non-nil error, if the length of the passed
|
2016-12-22 19:47:10 +01:00
|
|
|
// key > 64 bytes - but we use blake2b as hash function (no key)
|
2016-12-22 17:25:03 +01:00
|
|
|
h, _ = blake2b.New512(nil)
|
2016-06-01 05:23:31 +02:00
|
|
|
}
|
2016-12-22 17:25:03 +01:00
|
|
|
return h
|
2016-06-01 05:23:31 +02:00
|
|
|
}
|
|
|
|
|
2016-12-09 05:35:07 +01:00
|
|
|
// Hash buffer pool is a pool of reusable
|
|
|
|
// buffers used while checksumming a stream.
|
|
|
|
var hashBufferPool = sync.Pool{
|
|
|
|
New: func() interface{} {
|
|
|
|
b := make([]byte, readSizeV1)
|
|
|
|
return &b
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
2016-06-24 11:06:23 +02:00
|
|
|
// hashSum calculates the hash of the entire path and returns.
|
2016-06-01 05:23:31 +02:00
|
|
|
func hashSum(disk StorageAPI, volume, path string, writer hash.Hash) ([]byte, error) {
|
2016-12-09 05:35:07 +01:00
|
|
|
// Fetch staging a new staging buffer from the pool.
|
|
|
|
bufp := hashBufferPool.Get().(*[]byte)
|
|
|
|
defer hashBufferPool.Put(bufp)
|
2016-06-24 11:06:23 +02:00
|
|
|
|
|
|
|
// Copy entire buffer to writer.
|
2016-12-09 05:35:07 +01:00
|
|
|
if err := copyBuffer(writer, disk, volume, path, *bufp); err != nil {
|
2016-06-24 11:06:23 +02:00
|
|
|
return nil, err
|
2016-06-01 05:23:31 +02:00
|
|
|
}
|
2016-06-24 11:06:23 +02:00
|
|
|
|
|
|
|
// Return the final hash sum.
|
2016-06-01 05:23:31 +02:00
|
|
|
return writer.Sum(nil), nil
|
|
|
|
}
|
2016-05-30 00:38:14 +02:00
|
|
|
|
2016-06-22 21:55:23 +02:00
|
|
|
// getDataBlockLen - get length of data blocks from encoded blocks.
|
|
|
|
func getDataBlockLen(enBlocks [][]byte, dataBlocks int) int {
|
2016-05-30 00:38:14 +02:00
|
|
|
size := 0
|
2016-06-22 21:55:23 +02:00
|
|
|
// Figure out the data block length.
|
|
|
|
for _, block := range enBlocks[:dataBlocks] {
|
2016-05-30 00:38:14 +02:00
|
|
|
size += len(block)
|
|
|
|
}
|
2016-06-22 21:55:23 +02:00
|
|
|
return size
|
|
|
|
}
|
|
|
|
|
|
|
|
// Writes all the data blocks from encoded blocks until requested
|
|
|
|
// outSize length. Provides a way to skip bytes until the offset.
|
2016-07-20 10:30:30 +02:00
|
|
|
func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset int64, length int64) (int64, error) {
|
2016-07-07 10:30:34 +02:00
|
|
|
// Offset and out size cannot be negative.
|
2016-07-20 10:30:30 +02:00
|
|
|
if offset < 0 || length < 0 {
|
2016-08-25 18:39:01 +02:00
|
|
|
return 0, traceError(errUnexpected)
|
2016-07-07 10:30:34 +02:00
|
|
|
}
|
|
|
|
|
2016-06-22 21:55:23 +02:00
|
|
|
// Do we have enough blocks?
|
|
|
|
if len(enBlocks) < dataBlocks {
|
2016-08-25 18:39:01 +02:00
|
|
|
return 0, traceError(reedsolomon.ErrTooFewShards)
|
2016-05-30 00:38:14 +02:00
|
|
|
}
|
2016-06-01 05:23:31 +02:00
|
|
|
|
2016-06-22 21:55:23 +02:00
|
|
|
// Do we have enough data?
|
2016-07-20 10:30:30 +02:00
|
|
|
if int64(getDataBlockLen(enBlocks, dataBlocks)) < length {
|
2016-08-25 18:39:01 +02:00
|
|
|
return 0, traceError(reedsolomon.ErrShortData)
|
2016-06-22 21:55:23 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Counter to decrement total left to write.
|
2016-07-20 10:30:30 +02:00
|
|
|
write := length
|
2016-06-22 21:55:23 +02:00
|
|
|
|
|
|
|
// Counter to increment total written.
|
|
|
|
totalWritten := int64(0)
|
|
|
|
|
|
|
|
// Write all data blocks to dst.
|
|
|
|
for _, block := range enBlocks[:dataBlocks] {
|
|
|
|
// Skip blocks until we have reached our offset.
|
2016-07-20 10:30:30 +02:00
|
|
|
if offset >= int64(len(block)) {
|
2016-06-22 21:55:23 +02:00
|
|
|
// Decrement offset.
|
2016-07-20 10:30:30 +02:00
|
|
|
offset -= int64(len(block))
|
2016-06-22 21:55:23 +02:00
|
|
|
continue
|
|
|
|
} else {
|
|
|
|
// Skip until offset.
|
2016-07-20 10:30:30 +02:00
|
|
|
block = block[offset:]
|
2016-06-22 21:55:23 +02:00
|
|
|
|
|
|
|
// Reset the offset for next iteration to read everything
|
|
|
|
// from subsequent blocks.
|
2016-07-20 10:30:30 +02:00
|
|
|
offset = 0
|
2016-06-22 21:55:23 +02:00
|
|
|
}
|
|
|
|
// We have written all the blocks, write the last remaining block.
|
|
|
|
if write < int64(len(block)) {
|
|
|
|
n, err := io.Copy(dst, bytes.NewReader(block[:write]))
|
|
|
|
if err != nil {
|
2016-08-25 18:39:01 +02:00
|
|
|
return 0, traceError(err)
|
2016-06-22 21:55:23 +02:00
|
|
|
}
|
|
|
|
totalWritten += n
|
|
|
|
break
|
|
|
|
}
|
|
|
|
// Copy the block.
|
|
|
|
n, err := io.Copy(dst, bytes.NewReader(block))
|
|
|
|
if err != nil {
|
2016-08-25 18:39:01 +02:00
|
|
|
return 0, traceError(err)
|
2016-05-30 00:38:14 +02:00
|
|
|
}
|
2016-06-22 21:55:23 +02:00
|
|
|
|
|
|
|
// Decrement output size.
|
|
|
|
write -= n
|
|
|
|
|
|
|
|
// Increment written.
|
|
|
|
totalWritten += n
|
2016-05-07 01:25:08 +02:00
|
|
|
}
|
2016-06-22 21:55:23 +02:00
|
|
|
|
|
|
|
// Success.
|
|
|
|
return totalWritten, nil
|
2016-05-07 01:25:08 +02:00
|
|
|
}
|
|
|
|
|
2016-07-20 10:30:30 +02:00
|
|
|
// chunkSize is roughly BlockSize/DataBlocks.
|
|
|
|
// chunkSize is calculated such that chunkSize*DataBlocks accommodates BlockSize bytes.
|
|
|
|
// So chunkSize*DataBlocks can be slightly larger than BlockSize if BlockSize is not divisible by
|
|
|
|
// DataBlocks. The extra space will have 0-padding.
|
|
|
|
func getChunkSize(blockSize int64, dataBlocks int) int64 {
|
|
|
|
return (blockSize + int64(dataBlocks) - 1) / int64(dataBlocks)
|
2016-06-21 23:34:11 +02:00
|
|
|
}
|
2016-06-24 11:06:23 +02:00
|
|
|
|
|
|
|
// copyBuffer - copies from disk, volume, path to input writer until either EOF
|
|
|
|
// is reached at volume, path or an error occurs. A success copyBuffer returns
|
|
|
|
// err == nil, not err == EOF. Because copyBuffer is defined to read from path
|
|
|
|
// until EOF. It does not treat an EOF from ReadFile an error to be reported.
|
|
|
|
// Additionally copyBuffer stages through the provided buffer; otherwise if it
|
|
|
|
// has zero length, returns error.
|
|
|
|
func copyBuffer(writer io.Writer, disk StorageAPI, volume string, path string, buf []byte) error {
|
|
|
|
// Error condition of zero length buffer.
|
|
|
|
if buf != nil && len(buf) == 0 {
|
|
|
|
return errors.New("empty buffer in readBuffer")
|
|
|
|
}
|
|
|
|
|
|
|
|
// Starting offset for Reading the file.
|
|
|
|
startOffset := int64(0)
|
|
|
|
|
|
|
|
// Read until io.EOF.
|
|
|
|
for {
|
|
|
|
n, err := disk.ReadFile(volume, path, startOffset, buf)
|
|
|
|
if n > 0 {
|
2016-07-30 22:36:43 +02:00
|
|
|
m, wErr := writer.Write(buf[:n])
|
|
|
|
if wErr != nil {
|
|
|
|
return wErr
|
2016-06-24 11:06:23 +02:00
|
|
|
}
|
|
|
|
if int64(m) != n {
|
|
|
|
return io.ErrShortWrite
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
// Progress the offset.
|
|
|
|
startOffset += n
|
|
|
|
}
|
|
|
|
|
|
|
|
// Success.
|
|
|
|
return nil
|
|
|
|
}
|