maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

More simplification to erasure also add two more test functions

Browse source
This commit is contained in:
Harshavardhana 2015-03-29 14:43:35 -07:00
parent 4c6677fd2c
commit b748cf359a
6 changed files with 83 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
pkg/encoding/erasure/cauchy_test.go
View file

@ -34,19 +34,33 @@ const (
m = 5
)
func (s *MySuite) TestCauchyDecode(c *C) {
ep, _ := ParseEncoderParams(k, m, Cauchy)
func (s *MySuite) TestCauchyEncodeDecodeFailure(c *C) {
ep, _ := ValidateParams(k, m, Cauchy)
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
e := NewEncoder(ep)
chunks, _ := e.Encode(data)
e := NewErasure(ep)
chunks, err := e.Encode(data)
c.Assert(err, IsNil)
chunks[0] = nil
chunks[3] = nil
chunks[5] = nil
chunks[9] = nil
chunks[13] = nil
errorIndex := []int{0, 3, 5, 9, 11, 13}
chunks = corruptChunks(chunks, errorIndex)
_, err = e.Decode(chunks, len(data))
c.Assert(err, Not(IsNil))
}
func (s *MySuite) TestCauchyEncodeDecodeSuccess(c *C) {
ep, _ := ValidateParams(k, m, Cauchy)
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
e := NewErasure(ep)
chunks, err := e.Encode(data)
c.Assert(err, IsNil)
errorIndex := []int{0, 3, 5, 9, 13}
chunks = corruptChunks(chunks, errorIndex)
recoveredData, err := e.Decode(chunks, len(data))
c.Assert(err, IsNil)

12
pkg/encoding/erasure/doc.go
View file

@ -28,8 +28,8 @@
// 2. Create a new encoder
// 3. Decode data
//
// Encoder parameters contain three configurable elements:
// ParseEncoderParams(k, m, technique int) (EncoderParams, error)
// Erasure parameters contain three configurable elements:
// ValidateParams(k, m, technique int) (ErasureParams, error)
// k - Number of rows in matrix
// m - Number of colums in matrix
// technique - Matrix type, can be either Cauchy (recommended) or Vandermonde
@ -53,15 +53,15 @@
//
// Creating and using an encoder
// var bytes []byte
// params := erasure.ParseEncoderParams(10, 5, erasure.Cauchy)
// encoder := erasure.NewEncoder(params)
// params := erasure.ValidateParams(10, 5, erasure.Cauchy)
// encoder := erasure.NewErasure(params)
// encodedData, length := encoder.Encode(bytes)
//
// Creating and using a decoder
// var encodedData [][]byte
// var length int
// params := erasure.ParseEncoderParams(10, 5, erasure.Cauchy)
// encoder := erasure.NewEncoder(params)
// params := erasure.ValidateParams(10, 5, erasure.Cauchy)
// encoder := erasure.NewErasure(params)
// originalData, err := encoder.Decode(encodedData, length)
//
package erasure

2
pkg/encoding/erasure/erasure_decode.go
View file

@ -36,7 +36,7 @@ import (
// are set to "nil". There must be at least "K" number of data|parity
// blocks.
// "dataLen" is the length of original source data
func (e *Encoder) Decode(encodedDataBlocks [][]byte, dataLen int) (decodedData []byte, err error) {
func (e *Erasure) Decode(encodedDataBlocks [][]byte, dataLen int) (decodedData []byte, err error) {
var source, target **C.uint8_t
k := int(e.params.K)

24
pkg/encoding/erasure/erasure_encode.go
View file

@ -47,27 +47,27 @@ const (
SIMDAlign = 32
)
// EncoderParams is a configuration set for building an encoder. It is created using ValidateParams.
type EncoderParams struct {
// ErasureParams is a configuration set for building an encoder. It is created using ValidateParams().
type ErasureParams struct {
K uint8
M uint8
Technique Technique // cauchy or vandermonde matrix (RS)
}
// Encoder is an object used to encode and decode data.
type Encoder struct {
params *EncoderParams
// Erasure is an object used to encode and decode data.
type Erasure struct {
params *ErasureParams
encodeMatrix, encodeTbls *C.uint8_t
decodeMatrix, decodeTbls *C.uint8_t
decodeIndex *C.uint32_t
}
// ParseEncoderParams creates an EncoderParams object.
// ValidateParams creates an ErasureParams object.
//
// k and m represent the matrix size, which corresponds to the protection level
// technique is the matrix type. Valid inputs are Cauchy (recommended) or Vandermonde.
//
func ParseEncoderParams(k, m uint8, technique Technique) (*EncoderParams, error) {
func ValidateParams(k, m uint8, technique Technique) (*ErasureParams, error) {
if k < 1 {
return nil, errors.New("k cannot be zero")
}
@ -89,15 +89,15 @@ func ParseEncoderParams(k, m uint8, technique Technique) (*EncoderParams, error)
return nil, errors.New("Technique can be either vandermonde or cauchy")
}
return &EncoderParams{
return &ErasureParams{
K: k,
M: m,
Technique: technique,
}, nil
}
// NewEncoder creates an encoder object with a given set of parameters.
func NewEncoder(ep *EncoderParams) *Encoder {
// NewErasure creates an encoder object with a given set of parameters.
func NewErasure(ep *ErasureParams) *Erasure {
var k = C.int(ep.K)
var m = C.int(ep.M)
@ -107,7 +107,7 @@ func NewEncoder(ep *EncoderParams) *Encoder {
C.minio_init_encoder(C.int(ep.Technique), k, m, &encodeMatrix,
&encodeTbls)
return &Encoder{
return &Erasure{
params: ep,
encodeMatrix: encodeMatrix,
encodeTbls: encodeTbls,
@ -138,7 +138,7 @@ func GetEncodedBlockLen(inputLen int, k uint8) (encodedOutputLen int) {
// Encode erasure codes a block of data in "k" data blocks and "m" parity blocks.
// Output is [k+m][]blocks of data and parity slices.
func (e *Encoder) Encode(inputData []byte) (encodedBlocks [][]byte, err error) {
func (e *Erasure) Encode(inputData []byte) (encodedBlocks [][]byte, err error) {
k := int(e.params.K) // "k" data blocks
m := int(e.params.M) // "m" parity blocks
n := k + m // "n" total encoded blocks

39
pkg/encoding/erasure/vandermonde_test.go
View file

@ -22,21 +22,40 @@ import (
. "gopkg.in/check.v1"
)
func (s *MySuite) TestVanderMondeDecode(c *C) {
ep, _ := ParseEncoderParams(k, m, Vandermonde)
func corruptChunks(chunks [][]byte, errorIndex []int) [][]byte {
for _, err := range errorIndex {
chunks[err] = nil
}
return chunks
}
func (s *MySuite) TestVanderMondeEncodeDecodeFailure(c *C) {
ep, _ := ValidateParams(k, m, Vandermonde)
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
e := NewEncoder(ep)
e := NewErasure(ep)
chunks, err := e.Encode(data)
c.Logf("chunks length: %d", len(chunks))
c.Logf("length: %d", len(data))
c.Assert(err, IsNil)
chunks[0] = nil
chunks[3] = nil
chunks[5] = nil
chunks[9] = nil
chunks[13] = nil
errorIndex := []int{0, 3, 5, 9, 11, 13}
chunks = corruptChunks(chunks, errorIndex)
_, err = e.Decode(chunks, len(data))
c.Assert(err, Not(IsNil))
}
func (s *MySuite) TestVanderMondeEncodeDecodeSuccess(c *C) {
ep, _ := ValidateParams(k, m, Vandermonde)
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
e := NewErasure(ep)
chunks, err := e.Encode(data)
c.Assert(err, IsNil)
errorIndex := []int{0, 3, 5, 9, 13}
chunks = corruptChunks(chunks, errorIndex)
recoveredData, err := e.Decode(chunks, len(data))
c.Assert(err, IsNil)

26
pkg/storage/donut/erasure.go
View file

@ -3,6 +3,7 @@ package donut
import (
"bytes"
"errors"
"hash"
"io"
"strconv"
"strings"
@ -12,21 +13,19 @@ import (
"encoding/hex"
"github.com/minio-io/iodine"
"github.com/minio-io/minio/pkg/encoding/erasure"
encoding "github.com/minio-io/minio/pkg/encoding/erasure"
"github.com/minio-io/minio/pkg/utils/split"
"hash"
"log"
)
// getErasureTechnique - convert technique string into Technique type
func getErasureTechnique(technique string) (erasure.Technique, error) {
func getErasureTechnique(technique string) (encoding.Technique, error) {
switch true {
case technique == "Cauchy":
return erasure.Cauchy, nil
return encoding.Cauchy, nil
case technique == "Vandermonde":
return erasure.Cauchy, nil
return encoding.Cauchy, nil
default:
return erasure.None, iodine.New(errors.New("Invalid erasure technique: "+technique), nil)
return encoding.None, iodine.New(errors.New("Invalid erasure technique: "+technique), nil)
}
}
@ -70,13 +69,12 @@ func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, wri
return
}
hasher := md5.New()
params, err := erasure.ParseEncoderParams(k, m, technique)
params, err := encoding.ValidateParams(k, m, technique)
if err != nil {
writer.CloseWithError(iodine.New(err, donutMetadata))
}
encoder := erasure.NewEncoder(params)
encoder := encoding.NewErasure(params)
for i := 0; i < totalChunks; i++ {
log.Println(i)
totalLeft, err = decodeChunk(writer, readers, encoder, hasher, k, m, totalLeft, blockSize)
if err != nil {
errParams := map[string]string{
@ -97,7 +95,7 @@ func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, wri
return
}
func decodeChunk(writer *io.PipeWriter, readers []io.ReadCloser, encoder *erasure.Encoder, hasher hash.Hash, k, m uint8, totalLeft int64, blockSize int) (int64, error) {
func decodeChunk(writer *io.PipeWriter, readers []io.ReadCloser, encoder *encoding.Erasure, hasher hash.Hash, k, m uint8, totalLeft int64, blockSize int) (int64, error) {
curBlockSize := 0
if int64(blockSize) < totalLeft {
curBlockSize = blockSize
@ -105,7 +103,7 @@ func decodeChunk(writer *io.PipeWriter, readers []io.ReadCloser, encoder *erasur
curBlockSize = int(totalLeft) // cast is safe, blockSize in if protects
}
curChunkSize := erasure.GetEncodedBlockLen(curBlockSize, uint8(k))
curChunkSize := encoding.GetEncodedBlockLen(curBlockSize, uint8(k))
encodedBytes := make([][]byte, 16)
for i, reader := range readers {
var bytesBuffer bytes.Buffer
@ -167,8 +165,8 @@ func newErasureWriter(writers []Writer) ObjectWriter {
func erasureGoroutine(r *io.PipeReader, eWriter erasureWriter, isClosed chan<- bool) {
chunks := split.Stream(r, 10*1024*1024)
params, _ := erasure.ParseEncoderParams(8, 8, erasure.Cauchy)
encoder := erasure.NewEncoder(params)
params, _ := encoding.ValidateParams(8, 8, encoding.Cauchy)
encoder := encoding.NewErasure(params)
chunkCount := 0
totalLength := 0
summer := md5.New()