2016-08-17 20:36:33 +02:00
|
|
|
/*
|
|
|
|
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2016-08-19 01:23:42 +02:00
|
|
|
package cmd
|
2016-08-17 20:36:33 +02:00
|
|
|
|
2017-09-20 18:50:27 +02:00
|
|
|
import (
|
2017-09-29 00:57:19 +02:00
|
|
|
"fmt"
|
2017-09-20 18:50:27 +02:00
|
|
|
"hash"
|
2017-09-29 00:57:19 +02:00
|
|
|
"strings"
|
2017-09-20 18:50:27 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
// HealFile tries to reconstruct an erasure-coded file spread over all
|
|
|
|
// available disks. HealFile will read the valid parts of the file,
|
|
|
|
// reconstruct the missing data and write the reconstructed parts back
|
2017-09-29 00:57:19 +02:00
|
|
|
// to `staleDisks` at the destination `dstVol/dstPath/`. Parts are
|
|
|
|
// verified against the given BitrotAlgorithm and checksums.
|
2017-09-20 18:50:27 +02:00
|
|
|
//
|
|
|
|
// `staleDisks` is a slice of disks where each non-nil entry has stale
|
|
|
|
// or no data, and so will be healed.
|
|
|
|
//
|
|
|
|
// It is required that `s.disks` have a (read-quorum) majority of
|
|
|
|
// disks with valid data for healing to work.
|
|
|
|
//
|
|
|
|
// In addition, `staleDisks` and `s.disks` must have the same ordering
|
|
|
|
// of disks w.r.t. erasure coding of the object.
|
|
|
|
//
|
2017-09-29 00:57:19 +02:00
|
|
|
// Errors when writing to `staleDisks` are not propagated as long as
|
|
|
|
// writes succeed for at least one disk. This allows partial healing
|
|
|
|
// despite stale disks being faulty.
|
2017-09-20 18:50:27 +02:00
|
|
|
//
|
2017-09-29 00:57:19 +02:00
|
|
|
// It returns bitrot checksums for the non-nil staleDisks on which
|
|
|
|
// healing succeeded.
|
|
|
|
func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, blocksize int64,
|
|
|
|
dstVol, dstPath string, size int64, alg BitrotAlgorithm, checksums [][]byte) (
|
|
|
|
f ErasureFileInfo, err error) {
|
2016-08-17 20:36:33 +02:00
|
|
|
|
2017-09-29 00:57:19 +02:00
|
|
|
if !alg.Available() {
|
2017-08-15 03:08:42 +02:00
|
|
|
return f, traceError(errBitrotHashAlgoInvalid)
|
|
|
|
}
|
2017-09-20 18:50:27 +02:00
|
|
|
|
|
|
|
// Initialization
|
2017-08-15 03:08:42 +02:00
|
|
|
f.Checksums = make([][]byte, len(s.disks))
|
2017-09-20 18:50:27 +02:00
|
|
|
hashers := make([]hash.Hash, len(s.disks))
|
|
|
|
verifiers := make([]*BitrotVerifier, len(s.disks))
|
2017-08-15 03:08:42 +02:00
|
|
|
for i, disk := range s.disks {
|
2017-09-20 18:50:27 +02:00
|
|
|
switch {
|
|
|
|
case staleDisks[i] != nil:
|
2017-09-29 00:57:19 +02:00
|
|
|
hashers[i] = alg.New()
|
2017-09-20 18:50:27 +02:00
|
|
|
case disk == nil:
|
|
|
|
// disregard unavailable disk
|
|
|
|
continue
|
|
|
|
default:
|
2017-09-29 00:57:19 +02:00
|
|
|
verifiers[i] = NewBitrotVerifier(alg, checksums[i])
|
2016-08-17 20:36:33 +02:00
|
|
|
}
|
2017-08-15 03:08:42 +02:00
|
|
|
}
|
2017-09-29 00:57:19 +02:00
|
|
|
writeErrors := make([]error, len(s.disks))
|
2017-09-20 18:50:27 +02:00
|
|
|
|
|
|
|
// Scan part files on disk, block-by-block reconstruct it and
|
|
|
|
// write to stale disks.
|
2017-08-15 03:08:42 +02:00
|
|
|
chunksize := getChunkSize(blocksize, s.dataBlocks)
|
2017-09-27 19:29:42 +02:00
|
|
|
blocks := make([][]byte, len(s.disks))
|
|
|
|
for i := range blocks {
|
|
|
|
blocks[i] = make([]byte, chunksize)
|
|
|
|
}
|
2017-09-20 18:50:27 +02:00
|
|
|
var chunkOffset, blockOffset int64
|
|
|
|
for ; blockOffset < size; blockOffset += blocksize {
|
|
|
|
// last iteration may have less than blocksize data
|
|
|
|
// left, so chunksize needs to be recomputed.
|
|
|
|
if size < blockOffset+blocksize {
|
|
|
|
blocksize = size - blockOffset
|
2017-08-15 03:08:42 +02:00
|
|
|
chunksize = getChunkSize(blocksize, s.dataBlocks)
|
2017-09-27 19:29:42 +02:00
|
|
|
for i := range blocks {
|
|
|
|
blocks[i] = blocks[i][:chunksize]
|
|
|
|
}
|
2017-08-15 03:08:42 +02:00
|
|
|
}
|
2017-09-20 18:50:27 +02:00
|
|
|
// read a chunk from each disk, until we have
|
|
|
|
// `s.dataBlocks` number of chunks set to non-nil in
|
|
|
|
// `blocks`
|
2017-08-15 03:08:42 +02:00
|
|
|
numReads := 0
|
|
|
|
for i, disk := range s.disks {
|
2017-09-20 18:50:27 +02:00
|
|
|
// skip reading from unavailable or stale disks
|
|
|
|
if disk == nil || staleDisks[i] != nil {
|
2017-09-27 19:29:42 +02:00
|
|
|
blocks[i] = blocks[i][:0] // mark shard as missing
|
2017-09-20 18:50:27 +02:00
|
|
|
continue
|
|
|
|
}
|
2017-09-27 19:29:42 +02:00
|
|
|
_, err = disk.ReadFile(volume, path, chunkOffset, blocks[i], verifiers[i])
|
2017-09-20 18:50:27 +02:00
|
|
|
if err != nil {
|
|
|
|
// LOG FIXME: add a conditional log
|
|
|
|
// for read failures, once per-disk
|
|
|
|
// per-function-invocation.
|
2017-09-27 19:29:42 +02:00
|
|
|
blocks[i] = blocks[i][:0] // mark shard as missing
|
2017-09-20 18:50:27 +02:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
numReads++
|
|
|
|
if numReads == s.dataBlocks {
|
|
|
|
// we have enough data to reconstruct
|
2017-09-27 19:29:42 +02:00
|
|
|
// mark all other blocks as missing
|
|
|
|
for j := i + 1; j < len(blocks); j++ {
|
|
|
|
blocks[j] = blocks[j][:0] // mark shard as missing
|
|
|
|
}
|
2017-09-20 18:50:27 +02:00
|
|
|
break
|
2016-08-17 20:36:33 +02:00
|
|
|
}
|
|
|
|
}
|
2017-09-20 18:50:27 +02:00
|
|
|
|
|
|
|
// advance the chunk offset to prepare for next loop
|
|
|
|
// iteration
|
|
|
|
chunkOffset += chunksize
|
|
|
|
|
|
|
|
// reconstruct data - this computes all data and parity shards
|
2017-08-15 03:08:42 +02:00
|
|
|
if err = s.ErasureDecodeDataAndParityBlocks(blocks); err != nil {
|
|
|
|
return f, err
|
2016-08-17 20:36:33 +02:00
|
|
|
}
|
2017-09-20 18:50:27 +02:00
|
|
|
|
|
|
|
// write computed shards as chunks on file in each
|
|
|
|
// stale disk
|
2017-09-29 00:57:19 +02:00
|
|
|
writeSucceeded := false
|
2017-09-20 18:50:27 +02:00
|
|
|
for i, disk := range staleDisks {
|
2017-09-29 00:57:19 +02:00
|
|
|
// skip nil disk or disk that had error on
|
|
|
|
// previous write
|
|
|
|
if disk == nil || writeErrors[i] != nil {
|
2016-08-17 20:36:33 +02:00
|
|
|
continue
|
|
|
|
}
|
2017-09-20 18:50:27 +02:00
|
|
|
|
2017-09-29 00:57:19 +02:00
|
|
|
writeErrors[i] = disk.AppendFile(dstVol, dstPath, blocks[i])
|
|
|
|
if writeErrors[i] == nil {
|
|
|
|
hashers[i].Write(blocks[i])
|
|
|
|
writeSucceeded = true
|
2016-08-17 20:36:33 +02:00
|
|
|
}
|
2017-09-29 00:57:19 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// If all disks had write errors we quit.
|
|
|
|
if !writeSucceeded {
|
|
|
|
// build error from all write errors
|
|
|
|
return f, traceError(joinWriteErrors(writeErrors))
|
2016-08-17 20:36:33 +02:00
|
|
|
}
|
|
|
|
}
|
2017-09-20 18:50:27 +02:00
|
|
|
|
|
|
|
// copy computed file hashes into output variable
|
2017-08-15 03:08:42 +02:00
|
|
|
f.Size = size
|
2017-09-29 00:57:19 +02:00
|
|
|
f.Algorithm = alg
|
2017-09-20 18:50:27 +02:00
|
|
|
for i, disk := range staleDisks {
|
2017-09-29 00:57:19 +02:00
|
|
|
if disk == nil || writeErrors[i] != nil {
|
2016-08-17 20:36:33 +02:00
|
|
|
continue
|
|
|
|
}
|
2017-08-15 03:08:42 +02:00
|
|
|
f.Checksums[i] = hashers[i].Sum(nil)
|
2016-08-17 20:36:33 +02:00
|
|
|
}
|
2017-08-15 03:08:42 +02:00
|
|
|
return f, nil
|
2016-08-17 20:36:33 +02:00
|
|
|
}
|
2017-09-29 00:57:19 +02:00
|
|
|
|
|
|
|
func joinWriteErrors(errs []error) error {
|
|
|
|
msgs := []string{}
|
|
|
|
for i, err := range errs {
|
|
|
|
if err == nil {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
msgs = append(msgs, fmt.Sprintf("disk %d: %v", i+1, err))
|
|
|
|
}
|
|
|
|
return fmt.Errorf("all stale disks had write errors during healing: %s",
|
|
|
|
strings.Join(msgs, ", "))
|
|
|
|
}
|