maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge pull request #55 from harshavardhana/pr_out_decoding_bug_fixes_42

Browse source 
Decoding bug fixes #42
This commit is contained in:
Harshavardhana 2014-11-29 17:24:40 -08:00
parent a6e75f42dd 4d7c1e3b68
commit 980ad3b373
8 changed files with 116 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
pkgs/erasure/common.h Normal file
View file

@ -0,0 +1,45 @@
/*
* Mini Object Storage, (C) 2014 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef __COMMON_H__
#define __COMMON_H__
#include <stdint.h>
#define SIMD_ALIGN 32
int32_t minio_init_encoder (int technique, int k, int m,
unsigned char **encode_matrix,
unsigned char **encode_tbls);
uint32_t minio_calc_chunk_size (int k,
uint32_t split_len);
int32_t minio_init_decoder (int *src_err_list,
unsigned char *encoding_matrix,
unsigned char **decode_matrix,
unsigned char **decode_tbls,
int k, int n, int errs);
int32_t minio_src_in_err (int r, int *src_err_list);
int32_t minio_get_source_target(int *src_err_list,
int errs, int k, int m,
unsigned char **data,
unsigned char **coding,
unsigned char ***source,
unsigned char ***target);
#endif /* __COMMON_H__ */

52
pkgs/erasure/decode.c
View file

@ -19,13 +19,14 @@
#include <string.h>
#include <erasure-code.h>
#include "decode.h"
#include "common.h"
static int src_in_err (int r, int *src_err_list)
int32_t minio_src_in_err (int r, int *src_err_list)
{
int i;
for (i = 0; src_err_list[i] != -1; i++) {
if (src_err_list[i] == r) {
// true
return 1;
}
}
@ -33,6 +34,51 @@ static int src_in_err (int r, int *src_err_list)
return 0;
}
int32_t minio_get_source_target(int *src_err_list,
int errs, int k, int m,
unsigned char **data,
unsigned char **coding,
unsigned char ***source,
unsigned char ***target)
{
int i, j, l;
unsigned char *tmp_source[k];
unsigned char *tmp_target[m];
// Fill zeroes
memset (tmp_source, 0, sizeof(tmp_source));
memset (tmp_target, 0, sizeof(tmp_target));
// Separate out source and target buffers from input data/coding chunks
// This separation needs to happen at error chunks from input chunks
for (i = 0, j = 0, l = 0;
((l < k) || (j < errs)) && (i < (k + m)); i++) {
if (!minio_src_in_err(i, src_err_list)) {
if (l < k) {
if (i < k)
tmp_source[l] =
(unsigned char *) data[i];
else
tmp_source[l] =
(unsigned char *) coding[i - k];
l++;
}
} else {
if (j < m) {
if (i < k)
tmp_target[j] =
(unsigned char *) data[i];
else
tmp_target[j] =
(unsigned char *) coding[i - k];
j++;
}
}
}
*source = tmp_source;
*target = tmp_target;
}
/*
Generate decode matrix during the decoding phase
*/
@ -58,7 +104,7 @@ int minio_init_decoder (int *src_err_list,
return -1;
for (i = 0, r = 0; i < k; i++, r++) {
while (src_in_err(r, src_err_list))
while (minio_src_in_err(r, src_err_list))
r++;
for (j = 0; j < k; j++) {
input_matrix[k * i + j] = encode_matrix[k * r + j];

21
pkgs/erasure/decode.go
View file

@ -24,8 +24,7 @@ package erasure
// #include <erasure-code.h>
// #include <stdlib.h>
//
// #include "decode.h"
// #include "encode.h"
// #include "common.h"
import "C"
import (
"errors"
@ -36,6 +35,7 @@ import (
func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
var decode_matrix *C.uchar
var decode_tbls *C.uchar
var source, target **C.uchar
k := int(e.p.k)
n := int(e.p.k + e.p.m)
@ -44,7 +44,7 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
return nil, errors.New(fmt.Sprintf("chunks length must be %d", n))
}
chunk_size := int(C.calc_chunk_size(e.k, C.uint(length)))
chunk_size := int(C.minio_calc_chunk_size(e.k, C.uint32_t(length)))
src_err_list := make([]int, n+1)
var err_count int = 0
@ -84,21 +84,14 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
pointers[i] = &chunks[i][0]
}
/*
// Pack recovery array as list of valid sources
// Its order must be the same as the order
// to generate matrix b in gf_gen_decode_matrix
var i int
for i = 0; i < e.p.k; i++ {
recov[i] = buffs[decode_index[i]]
}
*/
data := (**C.uchar)(unsafe.Pointer(&pointers[:k][0]))
coding := (**C.uchar)(unsafe.Pointer(&pointers[k:][0]))
C.minio_get_source_target(src_err_list_ptr, C.int(err_count-1),
e.k, e.m, data, coding, &source, &target)
C.ec_encode_data(C.int(chunk_size), e.k, C.int(err_count-1), decode_tbls,
data, coding)
source, target)
recovered_output := make([]byte, 0, chunk_size*k)
for i := 0; i < k; i++ {

25
pkgs/erasure/decode.h
View file

@ -1,25 +0,0 @@
/*
* Mini Object Storage, (C) 2014 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef __DECODE_H__
#define __DECODE_H__
int minio_init_decoder (int *src_err_list,
unsigned char *encoding_matrix,
unsigned char **decode_matrix,
unsigned char **decode_tbls,
int k, int n, int errs);
#endif /* __DECODE_H__ */

13
pkgs/erasure/doc.go
View file

@ -13,37 +13,36 @@
//
// Encoding data may be performed in 3 steps.
//
// 1. Create a validated set of encoder parameters
// 1. Create a parse set of encoder parameters
// 2. Create a new encoder
// 3. Encode data
//
// Decoding data is also performed in 3 steps.
//
// 1. Create a validated set of encoder parameters
// 1. Create a parse set of encoder parameters for validation
// 2. Create a new encoder
// 3. Decode data
//
// Encoder parameters contain four configurable elements:
// ValidateParams(k, m, w, technique int) (EncoderParams, error)
// ParseEncoderParams(k, m, technique int) (EncoderParams, error)
// k - Number of rows in matrix
// m - Number of colums in matrix
// w - Word size, typically 8 or aligned by 8.
// technique - Matrix type, can be either CAUCHY (recommended) or VANDERMONDE
// constraints: k + m < 256
// constraints: k + m < Galois Field (2^8)
//
// Example
//
//
// Creating and using an encoder
// var bytes []byte
// params := erasure.ValidateParams(10,5, 8, erasure.CAUCHY)
// params := erasure.ParseEncoderParams(10, 5, erasure.CAUCHY)
// encoder := erasure.NewEncoder(params)
// encodedData, length := encoder.Encode(bytes)
//
// Creating and using a decoder
// var encodedData [][]byte
// var length int
// params := erasure.ValidateParams(10,5, 8, erasure.CAUCHY)
// params := erasure.ParseEncoderParams(10, 5, erasure.CAUCHY)
// encoder := erasure.NewEncoder(params)
// originalData, err := encoder.Decode(encodedData, length)
package erasure

16
pkgs/erasure/encode.c
View file

@ -18,11 +18,11 @@
#include <stdio.h>
#include <erasure-code.h>
#include "encode.h"
#include "common.h"
void minio_init_encoder (int technique, int k, int m,
unsigned char **encode_matrix,
unsigned char **encode_tbls)
int32_t minio_init_encoder (int technique, int k, int m,
unsigned char **encode_matrix,
unsigned char **encode_tbls)
{
size_t encode_matrix_size;
size_t encode_tbls_size;
@ -52,9 +52,11 @@ void minio_init_encoder (int technique, int k, int m,
*encode_matrix = tmp_matrix;
*encode_tbls = tmp_tbls;
return 0;
}
unsigned int calc_chunk_size (int k, unsigned int split_len)
uint32_t minio_calc_chunk_size (int k, uint32_t split_len)
{
int alignment;
int remainder;
@ -69,7 +71,3 @@ unsigned int calc_chunk_size (int k, unsigned int split_len)
}
return padded_len / k;
}
/*
void minio_encode (int k, int m, )
{
*/

4
pkgs/erasure/encode.go
View file

@ -24,7 +24,7 @@ package erasure
// #include <erasure-code.h>
// #include <stdlib.h>
//
// #include "encode.h"
// #include "common.h"
import "C"
import (
"errors"
@ -123,7 +123,7 @@ func NewEncoder(ep *EncoderParams) *Encoder {
func (e *Encoder) Encode(block []byte) ([][]byte, int) {
var block_len = len(block)
chunk_size := int(C.calc_chunk_size(e.k, C.uint(block_len)))
chunk_size := int(C.minio_calc_chunk_size(e.k, C.uint32_t(block_len)))
chunk_len := chunk_size * e.p.k
pad_len := chunk_len - block_len

25
pkgs/erasure/encode.h
View file

@ -1,25 +0,0 @@
/*
* Mini Object Storage, (C) 2014 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef __ENCODE_H__
#define __ENCODE_H__
#define SIMD_ALIGN 32
void minio_init_encoder (int technique, int k, int m,
unsigned char **encode_matrix,
unsigned char **encode_tbls);
unsigned int calc_chunk_size (int k, unsigned int split_len);
#endif /* __ENCODE_H__ */