e082f26e10
io.EOF is okay since io.ReadFull will not have read any bytes at all. Also making error channel receive only for go routine.
249 lines
7.5 KiB
Go
249 lines
7.5 KiB
Go
/*
|
|
* Minio Cloud Storage, (C) 2014 Minio, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package erasure
|
|
|
|
// #cgo CFLAGS: -O0
|
|
// #include <stdlib.h>
|
|
// #include "ec_isal-l.h"
|
|
// #include "ec_minio_common.h"
|
|
import "C"
|
|
import (
|
|
"errors"
|
|
"io"
|
|
"unsafe"
|
|
)
|
|
|
|
// Technique - type of matrix type used in encoding
|
|
type Technique uint8
|
|
|
|
// Different types of supported matrix types
|
|
const (
|
|
Vandermonde Technique = iota
|
|
Cauchy
|
|
None
|
|
)
|
|
|
|
// Default Data and Parity blocks
|
|
const (
|
|
K = 10
|
|
M = 3
|
|
)
|
|
|
|
// Block alignment
|
|
const (
|
|
SIMDAlign = 32
|
|
)
|
|
|
|
// Params is a configuration set for building an encoder. It is created using ValidateParams().
|
|
type Params struct {
|
|
K uint8
|
|
M uint8
|
|
Technique Technique // cauchy or vandermonde matrix (RS)
|
|
}
|
|
|
|
// Erasure is an object used to encode and decode data.
|
|
type Erasure struct {
|
|
params *Params
|
|
encodeMatrix, encodeTbls *C.uchar
|
|
decodeMatrix, decodeTbls *C.uchar
|
|
decodeIndex *C.uint32_t
|
|
}
|
|
|
|
// ValidateParams creates an Params object.
|
|
//
|
|
// k and m represent the matrix size, which corresponds to the protection level
|
|
// technique is the matrix type. Valid inputs are Cauchy (recommended) or Vandermonde.
|
|
//
|
|
func ValidateParams(k, m uint8, technique Technique) (*Params, error) {
|
|
if k < 1 {
|
|
return nil, errors.New("k cannot be zero")
|
|
}
|
|
|
|
if m < 1 {
|
|
return nil, errors.New("m cannot be zero")
|
|
}
|
|
|
|
if k+m > 255 {
|
|
return nil, errors.New("(k + m) cannot be bigger than Galois field GF(2^8) - 1")
|
|
}
|
|
|
|
switch technique {
|
|
case Vandermonde:
|
|
break
|
|
case Cauchy:
|
|
break
|
|
default:
|
|
return nil, errors.New("Technique can be either vandermonde or cauchy")
|
|
}
|
|
|
|
return &Params{
|
|
K: k,
|
|
M: m,
|
|
Technique: technique,
|
|
}, nil
|
|
}
|
|
|
|
// NewErasure creates an encoder object with a given set of parameters.
|
|
func NewErasure(ep *Params) *Erasure {
|
|
var k = C.int(ep.K)
|
|
var m = C.int(ep.M)
|
|
|
|
var encodeMatrix *C.uchar
|
|
var encodeTbls *C.uchar
|
|
|
|
C.minio_init_encoder(C.int(ep.Technique), k, m, &encodeMatrix,
|
|
&encodeTbls)
|
|
|
|
return &Erasure{
|
|
params: ep,
|
|
encodeMatrix: encodeMatrix,
|
|
encodeTbls: encodeTbls,
|
|
decodeMatrix: nil,
|
|
decodeTbls: nil,
|
|
decodeIndex: nil,
|
|
}
|
|
}
|
|
|
|
// GetEncodedBlocksLen - total length of all encoded blocks
|
|
func GetEncodedBlocksLen(inputLen int, k, m uint8) (outputLen int) {
|
|
outputLen = GetEncodedBlockLen(inputLen, k) * int(k+m)
|
|
return outputLen
|
|
}
|
|
|
|
// GetEncodedBlockLen - length per block of encoded blocks
|
|
func GetEncodedBlockLen(inputLen int, k uint8) (encodedOutputLen int) {
|
|
alignment := int(k) * SIMDAlign
|
|
remainder := inputLen % alignment
|
|
|
|
paddedInputLen := inputLen
|
|
if remainder != 0 {
|
|
paddedInputLen = inputLen + (alignment - remainder)
|
|
}
|
|
encodedOutputLen = paddedInputLen / int(k)
|
|
return encodedOutputLen
|
|
}
|
|
|
|
// Encode erasure codes a block of data in "k" data blocks and "m" parity blocks.
|
|
// Output is [k+m][]blocks of data and parity slices.
|
|
func (e *Erasure) Encode(inputData []byte) (encodedBlocks [][]byte, err error) {
|
|
k := int(e.params.K) // "k" data blocks
|
|
m := int(e.params.M) // "m" parity blocks
|
|
n := k + m // "n" total encoded blocks
|
|
|
|
// Length of a single encoded chunk.
|
|
// Total number of encoded chunks = "k" data + "m" parity blocks
|
|
encodedBlockLen := GetEncodedBlockLen(len(inputData), uint8(k))
|
|
|
|
// Length of total number of "k" data chunks
|
|
encodedDataBlocksLen := encodedBlockLen * k
|
|
|
|
// Length of extra padding required for the data blocks.
|
|
encodedDataBlocksPadLen := encodedDataBlocksLen - len(inputData)
|
|
|
|
// Extend inputData buffer to accommodate coded data blocks if necesssary
|
|
if encodedDataBlocksPadLen > 0 {
|
|
padding := make([]byte, encodedDataBlocksPadLen)
|
|
// Expand with new padded blocks to the byte array
|
|
inputData = append(inputData, padding...)
|
|
}
|
|
|
|
// Extend inputData buffer to accommodate coded parity blocks
|
|
{ // Local Scope
|
|
encodedParityBlocksLen := encodedBlockLen * m
|
|
parityBlocks := make([]byte, encodedParityBlocksLen)
|
|
inputData = append(inputData, parityBlocks...)
|
|
}
|
|
|
|
// Allocate memory to the "encoded blocks" return buffer
|
|
encodedBlocks = make([][]byte, n) // Return buffer
|
|
|
|
// Neccessary to bridge Go to the C world. C requires 2D arry of pointers to
|
|
// byte array. "encodedBlocks" is a 2D slice.
|
|
pointersToEncodedBlock := make([]*byte, n) // Pointers to encoded blocks.
|
|
|
|
// Copy data block slices to encoded block buffer
|
|
for i := 0; i < k; i++ {
|
|
encodedBlocks[i] = inputData[i*encodedBlockLen : (i+1)*encodedBlockLen]
|
|
pointersToEncodedBlock[i] = &encodedBlocks[i][0]
|
|
}
|
|
|
|
// Copy erasure block slices to encoded block buffer
|
|
for i := k; i < n; i++ {
|
|
encodedBlocks[i] = make([]byte, encodedBlockLen)
|
|
pointersToEncodedBlock[i] = &encodedBlocks[i][0]
|
|
}
|
|
|
|
// Erasure code the data into K data blocks and M parity
|
|
// blocks. Only the parity blocks are filled. Data blocks remain
|
|
// intact.
|
|
C.ec_encode_data(C.int(encodedBlockLen), C.int(k), C.int(m), e.encodeTbls,
|
|
(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[:k][0])), // Pointers to data blocks
|
|
(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[k:][0]))) // Pointers to parity blocks
|
|
|
|
return encodedBlocks, nil
|
|
}
|
|
|
|
// EncodeStream erasure codes a block of data in "k" data blocks and "m" parity blocks.
|
|
// Output is [k+m][]blocks of data and parity slices.
|
|
func (e *Erasure) EncodeStream(data io.Reader, size int64) ([][]byte, []byte, error) {
|
|
k := int(e.params.K) // "k" data blocks
|
|
m := int(e.params.M) // "m" parity blocks
|
|
n := k + m // "n" total encoded blocks
|
|
|
|
// Length of a single encoded chunk.
|
|
// Total number of encoded chunks = "k" data + "m" parity blocks
|
|
encodedBlockLen := GetEncodedBlockLen(int(size), uint8(k))
|
|
|
|
// Length of total number of "n" data chunks
|
|
encodedDataBlocksLen := encodedBlockLen * n
|
|
|
|
// allocate byte array for encodedBlock length
|
|
inputData := make([]byte, size, encodedDataBlocksLen)
|
|
|
|
_, err := io.ReadFull(data, inputData)
|
|
if err != nil {
|
|
// do not check for io.ErrUnexpectedEOF, we know the right amount of size
|
|
// to be read if its a short read we need to throw error since reader could
|
|
// have been prematurely closed.
|
|
if err != io.EOF {
|
|
return nil, nil, err
|
|
}
|
|
}
|
|
|
|
// Allocate memory to the "encoded blocks" return buffer
|
|
encodedBlocks := make([][]byte, n) // Return buffer
|
|
|
|
// Neccessary to bridge Go to the C world. C requires 2D arry of pointers to
|
|
// byte array. "encodedBlocks" is a 2D slice.
|
|
pointersToEncodedBlock := make([]*byte, n) // Pointers to encoded blocks.
|
|
|
|
// Copy data block slices to encoded block buffer
|
|
for i := 0; i < n; i++ {
|
|
encodedBlocks[i] = inputData[i*encodedBlockLen : (i+1)*encodedBlockLen]
|
|
pointersToEncodedBlock[i] = &encodedBlocks[i][0]
|
|
}
|
|
|
|
// Erasure code the data into K data blocks and M parity
|
|
// blocks. Only the parity blocks are filled. Data blocks remain
|
|
// intact.
|
|
C.ec_encode_data(C.int(encodedBlockLen), C.int(k), C.int(m), e.encodeTbls,
|
|
(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[:k][0])), // Pointers to data blocks
|
|
(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[k:][0]))) // Pointers to parity blocks
|
|
|
|
return encodedBlocks, inputData[0:size], nil
|
|
}
|