From 1e7515a7df7cfbead60fb9c7afb8660f2bf255b4 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 5 Nov 2014 03:09:40 -0800 Subject: [PATCH] Add erasure coding and decoding using Intel Storage Acceleration library - move contrib/erasure --> contrib/isal - bring in low level 'isal' package for Go for exposing C functions - Implement Erasure 'encoding' Supports - Reed Solomon Codes, Cauchy Codes - Implement Erasure 'decoding' Supports - Reed Solomon Codes, Cauchy Codes - Renames Minios -> Minio at all the references --- .gitignore | 1 + NOTICE | 9 +- erasure/Makefile | 16 ++ erasure/cauchy_test.go | 63 ++++++ erasure/cpufeatures.c | 59 +++++ erasure/cpufeatures.h | 24 ++ erasure/ctypes.go | 58 +++++ erasure/decode.go | 112 ++++++++++ erasure/encode.go | 205 ++++++++++++++++++ {contrib/erasure => erasure/isal}/.gitignore | 0 {contrib/erasure => erasure/isal}/LICENSE | 0 .../erasure => erasure/isal}/LICENSE.INTEL | 0 {contrib/erasure => erasure/isal}/Makefile | 0 .../erasure => erasure/isal}/Makefile.nmake | 0 {contrib/erasure => erasure/isal}/README.md | 0 .../isal}/RELEASE-NOTES.INTEL | 0 .../isal}/docs/isa-l_open_src_2.10.pdf | 0 .../isal}/include/ec-base.h | 0 .../isal}/include/erasure-code.h | 0 .../isal}/include/erasure/tests.h | 0 .../isal}/include/erasure/types.h | 0 .../isal}/include/gf-vect-mul.h | 0 .../isal}/include/reg-sizes.asm | 0 {contrib/erasure => erasure/isal}/make.inc | 0 .../erasure => erasure/isal}/src/Makefile | 0 .../erasure => erasure/isal}/src/ec-base.c | 0 .../isal}/src/ec-highlevel-func.c | 0 .../isal}/src/ec-multibinary.asm | 0 .../isal}/src/erasure-code-base-perf.c | 0 .../isal}/src/erasure-code-base-test.c | 0 .../isal}/src/erasure-code-perf.c | 0 .../isal}/src/erasure-code-sse-perf.c | 0 .../isal}/src/erasure-code-sse-test.c | 0 .../isal}/src/erasure-code-test.c | 0 .../isal}/src/gf-2vect-dot-prod-avx.asm | 0 .../isal}/src/gf-2vect-dot-prod-avx2.asm | 0 .../isal}/src/gf-2vect-dot-prod-sse-perf.c | 0 .../isal}/src/gf-2vect-dot-prod-sse-test.c | 0 .../isal}/src/gf-2vect-dot-prod-sse.asm | 0 .../isal}/src/gf-3vect-dot-prod-avx.asm | 0 .../isal}/src/gf-3vect-dot-prod-avx2.asm | 0 .../isal}/src/gf-3vect-dot-prod-sse-perf.c | 0 .../isal}/src/gf-3vect-dot-prod-sse-test.c | 0 .../isal}/src/gf-3vect-dot-prod-sse.asm | 0 .../isal}/src/gf-4vect-dot-prod-avx.asm | 0 .../isal}/src/gf-4vect-dot-prod-avx2.asm | 0 .../isal}/src/gf-4vect-dot-prod-sse-perf.c | 0 .../isal}/src/gf-4vect-dot-prod-sse-test.c | 0 .../isal}/src/gf-4vect-dot-prod-sse.asm | 0 .../isal}/src/gf-5vect-dot-prod-avx.asm | 0 .../isal}/src/gf-5vect-dot-prod-avx2.asm | 0 .../isal}/src/gf-5vect-dot-prod-sse-perf.c | 0 .../isal}/src/gf-5vect-dot-prod-sse-test.c | 0 .../isal}/src/gf-5vect-dot-prod-sse.asm | 0 .../isal}/src/gf-6vect-dot-prod-avx.asm | 0 .../isal}/src/gf-6vect-dot-prod-avx2.asm | 0 .../isal}/src/gf-6vect-dot-prod-sse-perf.c | 0 .../isal}/src/gf-6vect-dot-prod-sse-test.c | 0 .../isal}/src/gf-6vect-dot-prod-sse.asm | 0 .../isal}/src/gf-inverse-test.c | 0 .../isal}/src/gf-vect-dot-prod-1tbl.c | 0 .../isal}/src/gf-vect-dot-prod-avx-perf.c | 0 .../isal}/src/gf-vect-dot-prod-avx-test.c | 0 .../isal}/src/gf-vect-dot-prod-avx.asm | 0 .../isal}/src/gf-vect-dot-prod-avx2.asm | 0 .../isal}/src/gf-vect-dot-prod-base-test.c | 0 .../isal}/src/gf-vect-dot-prod-perf.c | 0 .../isal}/src/gf-vect-dot-prod-sse-perf.c | 0 .../isal}/src/gf-vect-dot-prod-sse-test.c | 0 .../isal}/src/gf-vect-dot-prod-sse.asm | 0 .../isal}/src/gf-vect-dot-prod-test.c | 0 .../isal}/src/gf-vect-mul-avx-perf.c | 0 .../isal}/src/gf-vect-mul-avx-test.c | 0 .../isal}/src/gf-vect-mul-avx.asm | 0 .../isal}/src/gf-vect-mul-base-test.c | 0 .../isal}/src/gf-vect-mul-perf.c | 0 .../isal}/src/gf-vect-mul-sse-perf.c | 0 .../isal}/src/gf-vect-mul-sse-test.c | 0 .../isal}/src/gf-vect-mul-sse.asm | 0 .../isal}/src/gf-vect-mul-test.c | 0 erasure/lru.go | 64 ++++++ erasure/matrix_decode.c | 98 +++++++++ erasure/matrix_decode.h | 24 ++ erasure/stdint.go | 48 ++++ erasure/vandermonde_test.go | 63 ++++++ 85 files changed, 839 insertions(+), 5 deletions(-) create mode 100644 erasure/Makefile create mode 100644 erasure/cauchy_test.go create mode 100644 erasure/cpufeatures.c create mode 100644 erasure/cpufeatures.h create mode 100644 erasure/ctypes.go create mode 100644 erasure/decode.go create mode 100644 erasure/encode.go rename {contrib/erasure => erasure/isal}/.gitignore (100%) rename {contrib/erasure => erasure/isal}/LICENSE (100%) rename {contrib/erasure => erasure/isal}/LICENSE.INTEL (100%) rename {contrib/erasure => erasure/isal}/Makefile (100%) rename {contrib/erasure => erasure/isal}/Makefile.nmake (100%) rename {contrib/erasure => erasure/isal}/README.md (100%) rename {contrib/erasure => erasure/isal}/RELEASE-NOTES.INTEL (100%) rename {contrib/erasure => erasure/isal}/docs/isa-l_open_src_2.10.pdf (100%) rename {contrib/erasure => erasure/isal}/include/ec-base.h (100%) rename {contrib/erasure => erasure/isal}/include/erasure-code.h (100%) rename {contrib/erasure => erasure/isal}/include/erasure/tests.h (100%) rename {contrib/erasure => erasure/isal}/include/erasure/types.h (100%) rename {contrib/erasure => erasure/isal}/include/gf-vect-mul.h (100%) rename {contrib/erasure => erasure/isal}/include/reg-sizes.asm (100%) rename {contrib/erasure => erasure/isal}/make.inc (100%) rename {contrib/erasure => erasure/isal}/src/Makefile (100%) rename {contrib/erasure => erasure/isal}/src/ec-base.c (100%) rename {contrib/erasure => erasure/isal}/src/ec-highlevel-func.c (100%) rename {contrib/erasure => erasure/isal}/src/ec-multibinary.asm (100%) rename {contrib/erasure => erasure/isal}/src/erasure-code-base-perf.c (100%) rename {contrib/erasure => erasure/isal}/src/erasure-code-base-test.c (100%) rename {contrib/erasure => erasure/isal}/src/erasure-code-perf.c (100%) rename {contrib/erasure => erasure/isal}/src/erasure-code-sse-perf.c (100%) rename {contrib/erasure => erasure/isal}/src/erasure-code-sse-test.c (100%) rename {contrib/erasure => erasure/isal}/src/erasure-code-test.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-2vect-dot-prod-avx.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-2vect-dot-prod-avx2.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-2vect-dot-prod-sse-perf.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-2vect-dot-prod-sse-test.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-2vect-dot-prod-sse.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-3vect-dot-prod-avx.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-3vect-dot-prod-avx2.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-3vect-dot-prod-sse-perf.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-3vect-dot-prod-sse-test.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-3vect-dot-prod-sse.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-4vect-dot-prod-avx.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-4vect-dot-prod-avx2.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-4vect-dot-prod-sse-perf.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-4vect-dot-prod-sse-test.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-4vect-dot-prod-sse.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-5vect-dot-prod-avx.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-5vect-dot-prod-avx2.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-5vect-dot-prod-sse-perf.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-5vect-dot-prod-sse-test.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-5vect-dot-prod-sse.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-6vect-dot-prod-avx.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-6vect-dot-prod-avx2.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-6vect-dot-prod-sse-perf.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-6vect-dot-prod-sse-test.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-6vect-dot-prod-sse.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-inverse-test.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-dot-prod-1tbl.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-dot-prod-avx-perf.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-dot-prod-avx-test.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-dot-prod-avx.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-dot-prod-avx2.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-dot-prod-base-test.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-dot-prod-perf.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-dot-prod-sse-perf.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-dot-prod-sse-test.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-dot-prod-sse.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-dot-prod-test.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-mul-avx-perf.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-mul-avx-test.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-mul-avx.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-mul-base-test.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-mul-perf.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-mul-sse-perf.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-mul-sse-test.c (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-mul-sse.asm (100%) rename {contrib/erasure => erasure/isal}/src/gf-vect-mul-test.c (100%) create mode 100644 erasure/lru.go create mode 100644 erasure/matrix_decode.c create mode 100644 erasure/matrix_decode.h create mode 100644 erasure/stdint.go create mode 100644 erasure/vandermonde_test.go diff --git a/.gitignore b/.gitignore index 2ac51d598..58c670ab2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ **/*.swp /cover/ +*~ diff --git a/NOTICE b/NOTICE index 080edd126..d5699a39f 100644 --- a/NOTICE +++ b/NOTICE @@ -4,8 +4,7 @@ Copyright 2014 Minio, Inc. This product includes software developed at Minio, Inc. (http://minio.io/). -The Minio project contains unmodified subcomponents under the contrib -folder with separate copyright notices and license terms. Your use of -the source code for the these subcomponents is subject to the terms -and conditions of the following licenses. - +The Minio project contains unmodified/modified subcomponents too with +separate copyright notices and license terms. Your use of the source +code for the these subcomponents is subject to the terms and conditions +of the following licenses. diff --git a/erasure/Makefile b/erasure/Makefile new file mode 100644 index 000000000..2e152c899 --- /dev/null +++ b/erasure/Makefile @@ -0,0 +1,16 @@ +all: build test +.PHONY: all + +test: cauchy vandermonde + +cauchy: + @go test -test.run="TestCauchy*" + +vandermonde: + @go test -test.run="TestVanderMonde*" + +isal/isal-l.so: + @$(MAKE) --quiet -C isal + +build: isal/isal-l.so + @go build diff --git a/erasure/cauchy_test.go b/erasure/cauchy_test.go new file mode 100644 index 000000000..6da038911 --- /dev/null +++ b/erasure/cauchy_test.go @@ -0,0 +1,63 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package erasure + +import ( + "bytes" + "testing" +) + +func TestCachyEncode(t *testing.T) { + ep, _ := ValidateParams(10, 5, 8, CAUCHY) + p := NewEncoder(ep) + + data := make([]byte, 1000) + chunks, length := p.Encode(data) + + t.Logf("chunks length: %d;\nlength: %d\n", len(chunks), length) + if length != len(data) { + t.Fatal() + } +} + +func TestCauchyDecode(t *testing.T) { + ep, _ := ValidateParams(10, 5, 8, CAUCHY) + + data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.") + + p := NewEncoder(ep) + chunks, length := p.Encode(data) + t.Logf("chunks length: %d;\nlength: %d\n", len(chunks), length) + if length != len(data) { + t.Fatal() + } + + chunks[0] = nil + chunks[3] = nil + chunks[5] = nil + chunks[9] = nil + chunks[13] = nil + + recovered_data, err := p.Decode(chunks, length) + if err != nil { + t.Fatalf("Error: %s", err) + } + + if i := bytes.Compare(recovered_data, data); i < 0 { + t.Fatalf("Error: recovered data is less than original data") + } +} diff --git a/erasure/cpufeatures.c b/erasure/cpufeatures.c new file mode 100644 index 000000000..b4f9832c0 --- /dev/null +++ b/erasure/cpufeatures.c @@ -0,0 +1,59 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +static void cpuid(int cpuinfo[4] ,int infotype) { + __asm__ __volatile__ ( + "cpuid": + "=a" (cpuinfo[0]), + "=b" (cpuinfo[1]), + "=c" (cpuinfo[2]), + "=d" (cpuinfo[3]) : + "a" (infotype), "c" (0) + ); +} + +/* + SSE41 : return true + no SSE41 : return false +*/ +int has_sse41 (void) { + int info[4]; + cpuid(info, 0x00000001); + return ((info[2] & ((int)1 << 19)) != 0); +} + +/* + AVX : return true + no AVX : return false +*/ +int has_avx (void) { + int info[4]; + cpuid(info, 0x00000001); + return ((info[2] & ((int)1 << 28)) != 0); +} + +/* + AVX2 : return true + no AVX2 : return false +*/ +int has_avx2 (void) { + int info[4]; + cpuid(info, 0x00000007); + return ((info[1] & ((int)1 << 5)) != 0); +} diff --git a/erasure/cpufeatures.h b/erasure/cpufeatures.h new file mode 100644 index 000000000..5eb27e13c --- /dev/null +++ b/erasure/cpufeatures.h @@ -0,0 +1,24 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __CPU_H__ +#define __CPU_H__ + +int has_sse41 (void); +int has_avx (void); +int has_avx2 (void); + +#endif /* __CPU_H__ */ diff --git a/erasure/ctypes.go b/erasure/ctypes.go new file mode 100644 index 000000000..a95816791 --- /dev/null +++ b/erasure/ctypes.go @@ -0,0 +1,58 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package erasure + +import "C" +import ( + "fmt" + "unsafe" +) + +// Integer to Int conversion +func Int2cInt(src_err_list []int) *C.int { + SrcErrInt := int(unsafe.Sizeof(src_err_list[0])) + switch SrcErrInt { + case SizeInt: + return (*C.int)(unsafe.Pointer(&src_err_list[0])) + case SizeInt8: + Int8Array := make([]int8, len(src_err_list)) + for i, v := range src_err_list { + Int8Array[i] = int8(v) + } + return (*C.int)(unsafe.Pointer(&Int8Array[0])) + case SizeInt16: + Int16Array := make([]int16, len(src_err_list)) + for i, v := range src_err_list { + Int16Array[i] = int16(v) + } + return (*C.int)(unsafe.Pointer(&Int16Array[0])) + case SizeInt32: + Int32Array := make([]int32, len(src_err_list)) + for i, v := range src_err_list { + Int32Array[i] = int32(v) + } + return (*C.int)(unsafe.Pointer(&Int32Array[0])) + case SizeInt64: + Int64Array := make([]int64, len(src_err_list)) + for i, v := range src_err_list { + Int64Array[i] = int64(v) + } + return (*C.int)(unsafe.Pointer(&Int64Array[0])) + default: + panic(fmt.Sprintf("Unsupported: %d", SizeInt)) + } +} diff --git a/erasure/decode.go b/erasure/decode.go new file mode 100644 index 000000000..0f3a268d1 --- /dev/null +++ b/erasure/decode.go @@ -0,0 +1,112 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// +build linux +// amd64 + +package erasure + +// #cgo CPPFLAGS: -Iisal/include +// #cgo LDFLAGS: isal/isa-l.so +// #include +// #include +// #include +// +// #include "matrix_decode.h" +import "C" +import ( + "errors" + "fmt" + "unsafe" +) + +func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) { + var decode_matrix *C.uchar + var decode_tbls *C.uchar + + var matrix_size C.size_t + var decode_tbls_size C.size_t + + k := int(e.p.k) + n := int(e.p.n) + + if len(chunks) != n { + return nil, errors.New(fmt.Sprintf("chunks length must be %d", n)) + } + + var chunk_size int = e.CalcChunkSize(length) + + src_err_list := make([]int, n+1) + var err_count int = 0 + + for i := range chunks { + // Check of chunks are really null + if chunks[i] == nil || len(chunks[i]) == 0 { + src_err_list[err_count] = i + err_count++ + } + } + src_err_list[err_count] = -1 + err_count++ + + // Too many missing chunks, cannot be more than parity `m` + if err_count-1 > e.p.m { + return nil, errors.New("too many erasures requested, can't decode") + } + + src_err_list_ptr := Int2cInt(src_err_list[:err_count]) + + for i := range chunks { + if chunks[i] == nil || len(chunks[i]) == 0 { + chunks[i] = make([]byte, chunk_size) + } + } + + matrix_size = C.size_t(k * n) + decode_matrix = (*C.uchar)(unsafe.Pointer(C.malloc(matrix_size))) + defer C.free(unsafe.Pointer(decode_matrix)) + + decode_tbls_size = C.size_t(k * n * 32) + decode_tbls = (*C.uchar)(unsafe.Pointer(C.malloc(decode_tbls_size))) + defer C.free(unsafe.Pointer(decode_tbls)) + + C.gf_gen_decode_matrix(src_err_list_ptr, e.encode_matrix, + decode_matrix, e.k, e.k+e.m, C.int(err_count-1), matrix_size) + + C.ec_init_tables(e.k, C.int(err_count-1), decode_matrix, decode_tbls) + e.decode_matrix = decode_matrix + e.decode_tbls = decode_tbls + + pointers := make([]*byte, n) + for i := range chunks { + pointers[i] = &chunks[i][0] + } + data := (**C.uchar)(unsafe.Pointer(&pointers[:k][0])) + coding := (**C.uchar)(unsafe.Pointer(&pointers[k:][0])) + + C.ec_encode_data(C.int(matrix_size), e.k, C.int(err_count-1), e.decode_tbls, + data, coding) + + recovered_output := make([]byte, 0, chunk_size*k) + for i := 0; i < k; i++ { + recovered_output = append(recovered_output, chunks[i]...) + } + return recovered_output[:length], nil +} + +func Decode(chunks [][]byte, ep EncoderParams, length int) (block []byte, err error) { + return GetEncoder(ep).Decode(chunks, length) +} diff --git a/erasure/encode.go b/erasure/encode.go new file mode 100644 index 000000000..1a2243bd6 --- /dev/null +++ b/erasure/encode.go @@ -0,0 +1,205 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// +build linux +// amd64 + +package erasure + +// #cgo CPPFLAGS: -Iisal/include +// #cgo LDFLAGS: isal/isa-l.so +// #include +// #include +// #include +// +// #include "cpufeatures.h" +import "C" +import ( + "errors" + "unsafe" +) + +const ( + VANDERMONDE = iota + CAUCHY = iota +) + +const ( + DEFAULT_K = 10 + DEFAULT_M = 3 +) + +type EncoderParams struct { + k, + m, + w, + n, + technique int // cauchy or vandermonde matrix (RS) +} + +type Encoder struct { + p *EncoderParams + k, + m, + w C.int + encode_matrix, + encode_tbls, + decode_matrix, + decode_tbls *C.uchar +} + +// Parameter validation +func ValidateParams(k, m, w, technique int) (*EncoderParams, error) { + if k < 1 { + return nil, errors.New("k cannot be zero") + } + + if m < 1 { + return nil, errors.New("m cannot be zero") + } + + if k+m > 255 { + return nil, errors.New("(k + m) cannot be bigger than Galois field GF(2^8) - 1") + } + + if 1< 5. + // -- Intel + C.gf_gen_rs_matrix(encode_matrix, n, k) + } else if ep.technique == CAUCHY { + C.gf_gen_cauchy1_matrix(encode_matrix, n, k) + } + + C.ec_init_tables(k, m, encode_matrix, encode_tbls) + + return &Encoder{ + p: ep, + k: k, + m: m, + w: w, + encode_matrix: encode_matrix, + encode_tbls: encode_tbls, + decode_matrix: nil, + decode_tbls: nil, + } +} + +func (e *Encoder) CalcChunkSize(block_len int) int { + var alignment int = e.p.k * e.p.w + var padding = block_len % alignment + var padded_len int + + if padding > 0 { + padded_len = block_len + (alignment - padding) + } else { + padded_len = block_len + } + return padded_len / e.p.k +} + +func (e *Encoder) Encode(block []byte) ([][]byte, int) { + var block_len = len(block) + + chunk_size := e.CalcChunkSize(block_len) + padded_len := chunk_size * e.p.k + + if (padded_len - block_len) > 0 { + s := make([]byte, (padded_len - block_len)) + // Expand with new padded blocks to the byte array + block = append(block, s...) + } + + coded_len := chunk_size * e.p.m + c := make([]byte, coded_len) + block = append(block, c...) + + // Allocate chunks + chunks := make([][]byte, e.p.n) + pointers := make([]*byte, e.p.n) + + var i int + // Add data and code blocks to chunks + for i = 0; i < e.p.n; i++ { + chunks[i] = block[i*chunk_size : (i+1)*chunk_size] + pointers[i] = &chunks[i][0] + } + + data := (**C.uchar)(unsafe.Pointer(&pointers[:e.p.k][0])) + coding := (**C.uchar)(unsafe.Pointer(&pointers[e.p.k:][0])) + + C.ec_encode_data(C.int(chunk_size), e.k, e.m, e.encode_tbls, data, + coding) + + return chunks, block_len +} + +func GetEncoder(ep EncoderParams) *Encoder { + return DefaultCache.GetC(ep) +} + +func Encode(data []byte, ep EncoderParams) (chunks [][]byte, length int) { + return GetEncoder(ep).Encode(data) +} diff --git a/contrib/erasure/.gitignore b/erasure/isal/.gitignore similarity index 100% rename from contrib/erasure/.gitignore rename to erasure/isal/.gitignore diff --git a/contrib/erasure/LICENSE b/erasure/isal/LICENSE similarity index 100% rename from contrib/erasure/LICENSE rename to erasure/isal/LICENSE diff --git a/contrib/erasure/LICENSE.INTEL b/erasure/isal/LICENSE.INTEL similarity index 100% rename from contrib/erasure/LICENSE.INTEL rename to erasure/isal/LICENSE.INTEL diff --git a/contrib/erasure/Makefile b/erasure/isal/Makefile similarity index 100% rename from contrib/erasure/Makefile rename to erasure/isal/Makefile diff --git a/contrib/erasure/Makefile.nmake b/erasure/isal/Makefile.nmake similarity index 100% rename from contrib/erasure/Makefile.nmake rename to erasure/isal/Makefile.nmake diff --git a/contrib/erasure/README.md b/erasure/isal/README.md similarity index 100% rename from contrib/erasure/README.md rename to erasure/isal/README.md diff --git a/contrib/erasure/RELEASE-NOTES.INTEL b/erasure/isal/RELEASE-NOTES.INTEL similarity index 100% rename from contrib/erasure/RELEASE-NOTES.INTEL rename to erasure/isal/RELEASE-NOTES.INTEL diff --git a/contrib/erasure/docs/isa-l_open_src_2.10.pdf b/erasure/isal/docs/isa-l_open_src_2.10.pdf similarity index 100% rename from contrib/erasure/docs/isa-l_open_src_2.10.pdf rename to erasure/isal/docs/isa-l_open_src_2.10.pdf diff --git a/contrib/erasure/include/ec-base.h b/erasure/isal/include/ec-base.h similarity index 100% rename from contrib/erasure/include/ec-base.h rename to erasure/isal/include/ec-base.h diff --git a/contrib/erasure/include/erasure-code.h b/erasure/isal/include/erasure-code.h similarity index 100% rename from contrib/erasure/include/erasure-code.h rename to erasure/isal/include/erasure-code.h diff --git a/contrib/erasure/include/erasure/tests.h b/erasure/isal/include/erasure/tests.h similarity index 100% rename from contrib/erasure/include/erasure/tests.h rename to erasure/isal/include/erasure/tests.h diff --git a/contrib/erasure/include/erasure/types.h b/erasure/isal/include/erasure/types.h similarity index 100% rename from contrib/erasure/include/erasure/types.h rename to erasure/isal/include/erasure/types.h diff --git a/contrib/erasure/include/gf-vect-mul.h b/erasure/isal/include/gf-vect-mul.h similarity index 100% rename from contrib/erasure/include/gf-vect-mul.h rename to erasure/isal/include/gf-vect-mul.h diff --git a/contrib/erasure/include/reg-sizes.asm b/erasure/isal/include/reg-sizes.asm similarity index 100% rename from contrib/erasure/include/reg-sizes.asm rename to erasure/isal/include/reg-sizes.asm diff --git a/contrib/erasure/make.inc b/erasure/isal/make.inc similarity index 100% rename from contrib/erasure/make.inc rename to erasure/isal/make.inc diff --git a/contrib/erasure/src/Makefile b/erasure/isal/src/Makefile similarity index 100% rename from contrib/erasure/src/Makefile rename to erasure/isal/src/Makefile diff --git a/contrib/erasure/src/ec-base.c b/erasure/isal/src/ec-base.c similarity index 100% rename from contrib/erasure/src/ec-base.c rename to erasure/isal/src/ec-base.c diff --git a/contrib/erasure/src/ec-highlevel-func.c b/erasure/isal/src/ec-highlevel-func.c similarity index 100% rename from contrib/erasure/src/ec-highlevel-func.c rename to erasure/isal/src/ec-highlevel-func.c diff --git a/contrib/erasure/src/ec-multibinary.asm b/erasure/isal/src/ec-multibinary.asm similarity index 100% rename from contrib/erasure/src/ec-multibinary.asm rename to erasure/isal/src/ec-multibinary.asm diff --git a/contrib/erasure/src/erasure-code-base-perf.c b/erasure/isal/src/erasure-code-base-perf.c similarity index 100% rename from contrib/erasure/src/erasure-code-base-perf.c rename to erasure/isal/src/erasure-code-base-perf.c diff --git a/contrib/erasure/src/erasure-code-base-test.c b/erasure/isal/src/erasure-code-base-test.c similarity index 100% rename from contrib/erasure/src/erasure-code-base-test.c rename to erasure/isal/src/erasure-code-base-test.c diff --git a/contrib/erasure/src/erasure-code-perf.c b/erasure/isal/src/erasure-code-perf.c similarity index 100% rename from contrib/erasure/src/erasure-code-perf.c rename to erasure/isal/src/erasure-code-perf.c diff --git a/contrib/erasure/src/erasure-code-sse-perf.c b/erasure/isal/src/erasure-code-sse-perf.c similarity index 100% rename from contrib/erasure/src/erasure-code-sse-perf.c rename to erasure/isal/src/erasure-code-sse-perf.c diff --git a/contrib/erasure/src/erasure-code-sse-test.c b/erasure/isal/src/erasure-code-sse-test.c similarity index 100% rename from contrib/erasure/src/erasure-code-sse-test.c rename to erasure/isal/src/erasure-code-sse-test.c diff --git a/contrib/erasure/src/erasure-code-test.c b/erasure/isal/src/erasure-code-test.c similarity index 100% rename from contrib/erasure/src/erasure-code-test.c rename to erasure/isal/src/erasure-code-test.c diff --git a/contrib/erasure/src/gf-2vect-dot-prod-avx.asm b/erasure/isal/src/gf-2vect-dot-prod-avx.asm similarity index 100% rename from contrib/erasure/src/gf-2vect-dot-prod-avx.asm rename to erasure/isal/src/gf-2vect-dot-prod-avx.asm diff --git a/contrib/erasure/src/gf-2vect-dot-prod-avx2.asm b/erasure/isal/src/gf-2vect-dot-prod-avx2.asm similarity index 100% rename from contrib/erasure/src/gf-2vect-dot-prod-avx2.asm rename to erasure/isal/src/gf-2vect-dot-prod-avx2.asm diff --git a/contrib/erasure/src/gf-2vect-dot-prod-sse-perf.c b/erasure/isal/src/gf-2vect-dot-prod-sse-perf.c similarity index 100% rename from contrib/erasure/src/gf-2vect-dot-prod-sse-perf.c rename to erasure/isal/src/gf-2vect-dot-prod-sse-perf.c diff --git a/contrib/erasure/src/gf-2vect-dot-prod-sse-test.c b/erasure/isal/src/gf-2vect-dot-prod-sse-test.c similarity index 100% rename from contrib/erasure/src/gf-2vect-dot-prod-sse-test.c rename to erasure/isal/src/gf-2vect-dot-prod-sse-test.c diff --git a/contrib/erasure/src/gf-2vect-dot-prod-sse.asm b/erasure/isal/src/gf-2vect-dot-prod-sse.asm similarity index 100% rename from contrib/erasure/src/gf-2vect-dot-prod-sse.asm rename to erasure/isal/src/gf-2vect-dot-prod-sse.asm diff --git a/contrib/erasure/src/gf-3vect-dot-prod-avx.asm b/erasure/isal/src/gf-3vect-dot-prod-avx.asm similarity index 100% rename from contrib/erasure/src/gf-3vect-dot-prod-avx.asm rename to erasure/isal/src/gf-3vect-dot-prod-avx.asm diff --git a/contrib/erasure/src/gf-3vect-dot-prod-avx2.asm b/erasure/isal/src/gf-3vect-dot-prod-avx2.asm similarity index 100% rename from contrib/erasure/src/gf-3vect-dot-prod-avx2.asm rename to erasure/isal/src/gf-3vect-dot-prod-avx2.asm diff --git a/contrib/erasure/src/gf-3vect-dot-prod-sse-perf.c b/erasure/isal/src/gf-3vect-dot-prod-sse-perf.c similarity index 100% rename from contrib/erasure/src/gf-3vect-dot-prod-sse-perf.c rename to erasure/isal/src/gf-3vect-dot-prod-sse-perf.c diff --git a/contrib/erasure/src/gf-3vect-dot-prod-sse-test.c b/erasure/isal/src/gf-3vect-dot-prod-sse-test.c similarity index 100% rename from contrib/erasure/src/gf-3vect-dot-prod-sse-test.c rename to erasure/isal/src/gf-3vect-dot-prod-sse-test.c diff --git a/contrib/erasure/src/gf-3vect-dot-prod-sse.asm b/erasure/isal/src/gf-3vect-dot-prod-sse.asm similarity index 100% rename from contrib/erasure/src/gf-3vect-dot-prod-sse.asm rename to erasure/isal/src/gf-3vect-dot-prod-sse.asm diff --git a/contrib/erasure/src/gf-4vect-dot-prod-avx.asm b/erasure/isal/src/gf-4vect-dot-prod-avx.asm similarity index 100% rename from contrib/erasure/src/gf-4vect-dot-prod-avx.asm rename to erasure/isal/src/gf-4vect-dot-prod-avx.asm diff --git a/contrib/erasure/src/gf-4vect-dot-prod-avx2.asm b/erasure/isal/src/gf-4vect-dot-prod-avx2.asm similarity index 100% rename from contrib/erasure/src/gf-4vect-dot-prod-avx2.asm rename to erasure/isal/src/gf-4vect-dot-prod-avx2.asm diff --git a/contrib/erasure/src/gf-4vect-dot-prod-sse-perf.c b/erasure/isal/src/gf-4vect-dot-prod-sse-perf.c similarity index 100% rename from contrib/erasure/src/gf-4vect-dot-prod-sse-perf.c rename to erasure/isal/src/gf-4vect-dot-prod-sse-perf.c diff --git a/contrib/erasure/src/gf-4vect-dot-prod-sse-test.c b/erasure/isal/src/gf-4vect-dot-prod-sse-test.c similarity index 100% rename from contrib/erasure/src/gf-4vect-dot-prod-sse-test.c rename to erasure/isal/src/gf-4vect-dot-prod-sse-test.c diff --git a/contrib/erasure/src/gf-4vect-dot-prod-sse.asm b/erasure/isal/src/gf-4vect-dot-prod-sse.asm similarity index 100% rename from contrib/erasure/src/gf-4vect-dot-prod-sse.asm rename to erasure/isal/src/gf-4vect-dot-prod-sse.asm diff --git a/contrib/erasure/src/gf-5vect-dot-prod-avx.asm b/erasure/isal/src/gf-5vect-dot-prod-avx.asm similarity index 100% rename from contrib/erasure/src/gf-5vect-dot-prod-avx.asm rename to erasure/isal/src/gf-5vect-dot-prod-avx.asm diff --git a/contrib/erasure/src/gf-5vect-dot-prod-avx2.asm b/erasure/isal/src/gf-5vect-dot-prod-avx2.asm similarity index 100% rename from contrib/erasure/src/gf-5vect-dot-prod-avx2.asm rename to erasure/isal/src/gf-5vect-dot-prod-avx2.asm diff --git a/contrib/erasure/src/gf-5vect-dot-prod-sse-perf.c b/erasure/isal/src/gf-5vect-dot-prod-sse-perf.c similarity index 100% rename from contrib/erasure/src/gf-5vect-dot-prod-sse-perf.c rename to erasure/isal/src/gf-5vect-dot-prod-sse-perf.c diff --git a/contrib/erasure/src/gf-5vect-dot-prod-sse-test.c b/erasure/isal/src/gf-5vect-dot-prod-sse-test.c similarity index 100% rename from contrib/erasure/src/gf-5vect-dot-prod-sse-test.c rename to erasure/isal/src/gf-5vect-dot-prod-sse-test.c diff --git a/contrib/erasure/src/gf-5vect-dot-prod-sse.asm b/erasure/isal/src/gf-5vect-dot-prod-sse.asm similarity index 100% rename from contrib/erasure/src/gf-5vect-dot-prod-sse.asm rename to erasure/isal/src/gf-5vect-dot-prod-sse.asm diff --git a/contrib/erasure/src/gf-6vect-dot-prod-avx.asm b/erasure/isal/src/gf-6vect-dot-prod-avx.asm similarity index 100% rename from contrib/erasure/src/gf-6vect-dot-prod-avx.asm rename to erasure/isal/src/gf-6vect-dot-prod-avx.asm diff --git a/contrib/erasure/src/gf-6vect-dot-prod-avx2.asm b/erasure/isal/src/gf-6vect-dot-prod-avx2.asm similarity index 100% rename from contrib/erasure/src/gf-6vect-dot-prod-avx2.asm rename to erasure/isal/src/gf-6vect-dot-prod-avx2.asm diff --git a/contrib/erasure/src/gf-6vect-dot-prod-sse-perf.c b/erasure/isal/src/gf-6vect-dot-prod-sse-perf.c similarity index 100% rename from contrib/erasure/src/gf-6vect-dot-prod-sse-perf.c rename to erasure/isal/src/gf-6vect-dot-prod-sse-perf.c diff --git a/contrib/erasure/src/gf-6vect-dot-prod-sse-test.c b/erasure/isal/src/gf-6vect-dot-prod-sse-test.c similarity index 100% rename from contrib/erasure/src/gf-6vect-dot-prod-sse-test.c rename to erasure/isal/src/gf-6vect-dot-prod-sse-test.c diff --git a/contrib/erasure/src/gf-6vect-dot-prod-sse.asm b/erasure/isal/src/gf-6vect-dot-prod-sse.asm similarity index 100% rename from contrib/erasure/src/gf-6vect-dot-prod-sse.asm rename to erasure/isal/src/gf-6vect-dot-prod-sse.asm diff --git a/contrib/erasure/src/gf-inverse-test.c b/erasure/isal/src/gf-inverse-test.c similarity index 100% rename from contrib/erasure/src/gf-inverse-test.c rename to erasure/isal/src/gf-inverse-test.c diff --git a/contrib/erasure/src/gf-vect-dot-prod-1tbl.c b/erasure/isal/src/gf-vect-dot-prod-1tbl.c similarity index 100% rename from contrib/erasure/src/gf-vect-dot-prod-1tbl.c rename to erasure/isal/src/gf-vect-dot-prod-1tbl.c diff --git a/contrib/erasure/src/gf-vect-dot-prod-avx-perf.c b/erasure/isal/src/gf-vect-dot-prod-avx-perf.c similarity index 100% rename from contrib/erasure/src/gf-vect-dot-prod-avx-perf.c rename to erasure/isal/src/gf-vect-dot-prod-avx-perf.c diff --git a/contrib/erasure/src/gf-vect-dot-prod-avx-test.c b/erasure/isal/src/gf-vect-dot-prod-avx-test.c similarity index 100% rename from contrib/erasure/src/gf-vect-dot-prod-avx-test.c rename to erasure/isal/src/gf-vect-dot-prod-avx-test.c diff --git a/contrib/erasure/src/gf-vect-dot-prod-avx.asm b/erasure/isal/src/gf-vect-dot-prod-avx.asm similarity index 100% rename from contrib/erasure/src/gf-vect-dot-prod-avx.asm rename to erasure/isal/src/gf-vect-dot-prod-avx.asm diff --git a/contrib/erasure/src/gf-vect-dot-prod-avx2.asm b/erasure/isal/src/gf-vect-dot-prod-avx2.asm similarity index 100% rename from contrib/erasure/src/gf-vect-dot-prod-avx2.asm rename to erasure/isal/src/gf-vect-dot-prod-avx2.asm diff --git a/contrib/erasure/src/gf-vect-dot-prod-base-test.c b/erasure/isal/src/gf-vect-dot-prod-base-test.c similarity index 100% rename from contrib/erasure/src/gf-vect-dot-prod-base-test.c rename to erasure/isal/src/gf-vect-dot-prod-base-test.c diff --git a/contrib/erasure/src/gf-vect-dot-prod-perf.c b/erasure/isal/src/gf-vect-dot-prod-perf.c similarity index 100% rename from contrib/erasure/src/gf-vect-dot-prod-perf.c rename to erasure/isal/src/gf-vect-dot-prod-perf.c diff --git a/contrib/erasure/src/gf-vect-dot-prod-sse-perf.c b/erasure/isal/src/gf-vect-dot-prod-sse-perf.c similarity index 100% rename from contrib/erasure/src/gf-vect-dot-prod-sse-perf.c rename to erasure/isal/src/gf-vect-dot-prod-sse-perf.c diff --git a/contrib/erasure/src/gf-vect-dot-prod-sse-test.c b/erasure/isal/src/gf-vect-dot-prod-sse-test.c similarity index 100% rename from contrib/erasure/src/gf-vect-dot-prod-sse-test.c rename to erasure/isal/src/gf-vect-dot-prod-sse-test.c diff --git a/contrib/erasure/src/gf-vect-dot-prod-sse.asm b/erasure/isal/src/gf-vect-dot-prod-sse.asm similarity index 100% rename from contrib/erasure/src/gf-vect-dot-prod-sse.asm rename to erasure/isal/src/gf-vect-dot-prod-sse.asm diff --git a/contrib/erasure/src/gf-vect-dot-prod-test.c b/erasure/isal/src/gf-vect-dot-prod-test.c similarity index 100% rename from contrib/erasure/src/gf-vect-dot-prod-test.c rename to erasure/isal/src/gf-vect-dot-prod-test.c diff --git a/contrib/erasure/src/gf-vect-mul-avx-perf.c b/erasure/isal/src/gf-vect-mul-avx-perf.c similarity index 100% rename from contrib/erasure/src/gf-vect-mul-avx-perf.c rename to erasure/isal/src/gf-vect-mul-avx-perf.c diff --git a/contrib/erasure/src/gf-vect-mul-avx-test.c b/erasure/isal/src/gf-vect-mul-avx-test.c similarity index 100% rename from contrib/erasure/src/gf-vect-mul-avx-test.c rename to erasure/isal/src/gf-vect-mul-avx-test.c diff --git a/contrib/erasure/src/gf-vect-mul-avx.asm b/erasure/isal/src/gf-vect-mul-avx.asm similarity index 100% rename from contrib/erasure/src/gf-vect-mul-avx.asm rename to erasure/isal/src/gf-vect-mul-avx.asm diff --git a/contrib/erasure/src/gf-vect-mul-base-test.c b/erasure/isal/src/gf-vect-mul-base-test.c similarity index 100% rename from contrib/erasure/src/gf-vect-mul-base-test.c rename to erasure/isal/src/gf-vect-mul-base-test.c diff --git a/contrib/erasure/src/gf-vect-mul-perf.c b/erasure/isal/src/gf-vect-mul-perf.c similarity index 100% rename from contrib/erasure/src/gf-vect-mul-perf.c rename to erasure/isal/src/gf-vect-mul-perf.c diff --git a/contrib/erasure/src/gf-vect-mul-sse-perf.c b/erasure/isal/src/gf-vect-mul-sse-perf.c similarity index 100% rename from contrib/erasure/src/gf-vect-mul-sse-perf.c rename to erasure/isal/src/gf-vect-mul-sse-perf.c diff --git a/contrib/erasure/src/gf-vect-mul-sse-test.c b/erasure/isal/src/gf-vect-mul-sse-test.c similarity index 100% rename from contrib/erasure/src/gf-vect-mul-sse-test.c rename to erasure/isal/src/gf-vect-mul-sse-test.c diff --git a/contrib/erasure/src/gf-vect-mul-sse.asm b/erasure/isal/src/gf-vect-mul-sse.asm similarity index 100% rename from contrib/erasure/src/gf-vect-mul-sse.asm rename to erasure/isal/src/gf-vect-mul-sse.asm diff --git a/contrib/erasure/src/gf-vect-mul-test.c b/erasure/isal/src/gf-vect-mul-test.c similarity index 100% rename from contrib/erasure/src/gf-vect-mul-test.c rename to erasure/isal/src/gf-vect-mul-test.c diff --git a/erasure/lru.go b/erasure/lru.go new file mode 100644 index 000000000..23934b995 --- /dev/null +++ b/erasure/lru.go @@ -0,0 +1,64 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package erasure + +import ( + "github.com/golang/groupcache/lru" + "sync" +) + +// thread-safe LRU cache from GroupCache +type Cache struct { + mutex sync.RWMutex + cache *lru.Cache +} + +var DefaultCache *Cache = GetCache(0) + +// Allocate ``Cache`` LRU +func GetCache(capacity int) *Cache { + return &Cache{ + cache: lru.New(capacity), + } +} + +// ``GetC()`` -- Grab encoder from LRU +func (c *Cache) GetC(ep EncoderParams) *Encoder { + if encoder, ret := c._Get(ep); ret { + return encoder + } + encoder := NewEncoder(&ep) + c._Put(ep, encoder) + return encoder +} + +// ``_Get()`` -- Get key from existing LRU +func (c *Cache) _Get(ep EncoderParams) (*Encoder, bool) { + c.mutex.RLock() + defer c.mutex.RUnlock() + if encoder, ret := c.cache.Get(ep); ret { + return encoder.(*Encoder), ret + } + return nil, false +} + +// ``_Put()`` -- Add key to existing LRU +func (c *Cache) _Put(ep EncoderParams, encoder *Encoder) { + c.mutex.Lock() + defer c.mutex.Unlock() + c.cache.Add(ep, encoder) +} diff --git a/erasure/matrix_decode.c b/erasure/matrix_decode.c new file mode 100644 index 000000000..ad0526b7e --- /dev/null +++ b/erasure/matrix_decode.c @@ -0,0 +1,98 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include "matrix_decode.h" + +static int src_in_err (int r, int *src_err_list) +{ + + int i; + for (i = 0; src_err_list[i] != -1; i++) { + if (src_err_list[i] == r) { + return 1; + } + } + // false + return 0; +} + +/* + Generate decode matrix during the decoding phase +*/ + +int gf_gen_decode_matrix (int *src_err_list, + unsigned char *encode_matrix, + unsigned char *decode_matrix, + int k, int n, int errs, + size_t matrix_size) +{ + int i, j, r, s, l, z; + unsigned char *input_matrix = NULL; + unsigned char *inverse_matrix = NULL; + + input_matrix = malloc(k * n); + if (!input_matrix) { + return -1; + } + + inverse_matrix = malloc(matrix_size); + if (!inverse_matrix) { + return -1; + } + + for (i = 0, r = 0; i < k; i++, r++) { + while (src_in_err(r, src_err_list)) + r++; + for (j = 0; j < k; j++) { + input_matrix[k * i + j] = encode_matrix[k * r + j]; + } + } + + // Not all Vandermonde matrix can be inverted + if (gf_invert_matrix(input_matrix, inverse_matrix, k) < 0) { + return -1; + } + + for (l = 0; l < errs; l++) { + if (src_err_list[l] < k) { + // decoding matrix elements for data chunks + for (j = 0; j < k; j++) { + decode_matrix[k * l + j] = + inverse_matrix[k * + src_err_list[l] + j]; + } + } else { + int s = 0; + // decoding matrix element for coding chunks + for (i = 0; i < k; i++) { + s = 0; + for (j = 0; j < k; j++) { + s ^= gf_mul(inverse_matrix[j * k + i], + encode_matrix[k * + src_err_list[l] + j]); + } + decode_matrix[k * l + i] = s; + } + } + } + free(input_matrix); + free(inverse_matrix); + return 0; +} diff --git a/erasure/matrix_decode.h b/erasure/matrix_decode.h new file mode 100644 index 000000000..f15081b9c --- /dev/null +++ b/erasure/matrix_decode.h @@ -0,0 +1,24 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __MATRIX_DECODE_H__ +#define __MATRIX_DECODE_H__ + +int gf_gen_decode_matrix (int *src_err_list, + unsigned char *encoding_matrix, + unsigned char *decode_matrix, int k, int n, + int errs, size_t matrix_size); +#endif /* __MATRIX_DECODE_H__ */ diff --git a/erasure/stdint.go b/erasure/stdint.go new file mode 100644 index 000000000..834757037 --- /dev/null +++ b/erasure/stdint.go @@ -0,0 +1,48 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package erasure + +// +// int SizeInt() +// { +// return sizeof(int); +// } +import "C" +import "unsafe" + +var ( + // See http://golang.org/ref/spec#Numeric_types + + // SizeUint8 is the byte size of a uint8. + SizeUint8 = int(unsafe.Sizeof(uint8(0))) + // SizeUint16 is the byte size of a uint16. + SizeUint16 = int(unsafe.Sizeof(uint16(0))) + // SizeUint32 is the byte size of a uint32. + SizeUint32 = int(unsafe.Sizeof(uint32(0))) + // SizeUint64 is the byte size of a uint64. + SizeUint64 = int(unsafe.Sizeof(uint64(0))) + + SizeInt = int(C.SizeInt()) + // SizeInt8 is the byte size of a int8. + SizeInt8 = int(unsafe.Sizeof(int8(0))) + // SizeInt16 is the byte size of a int16. + SizeInt16 = int(unsafe.Sizeof(int16(0))) + // SizeInt32 is the byte size of a int32. + SizeInt32 = int(unsafe.Sizeof(int32(0))) + // SizeInt64 is the byte size of a int64. + SizeInt64 = int(unsafe.Sizeof(int64(0))) +) diff --git a/erasure/vandermonde_test.go b/erasure/vandermonde_test.go new file mode 100644 index 000000000..8e78c65a6 --- /dev/null +++ b/erasure/vandermonde_test.go @@ -0,0 +1,63 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package erasure + +import ( + "bytes" + "testing" +) + +func TestVanderMondeEncode(t *testing.T) { + ep, _ := ValidateParams(10, 5, 8, VANDERMONDE) + p := NewEncoder(ep) + + data := make([]byte, 1000) + chunks, length := p.Encode(data) + + t.Logf("chunks length: %d;\nlength: %d\n", len(chunks), length) + if length != len(data) { + t.Fatal() + } +} + +func TestVanderMondeDecode(t *testing.T) { + ep, _ := ValidateParams(10, 5, 8, VANDERMONDE) + p := NewEncoder(ep) + + data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.") + + chunks, length := p.Encode(data) + t.Logf("chunks length: %d;\nlength: %d\n", len(chunks), length) + if length != len(data) { + t.Fatal() + } + + chunks[0] = nil + chunks[3] = nil + chunks[5] = nil + chunks[9] = nil + chunks[13] = nil + + recovered_data, err := p.Decode(chunks, length) + if err != nil { + t.Fatalf("Error: %s", err) + } + + if i := bytes.Compare(recovered_data, data); i < 0 { + t.Fatalf("Error: recovered data is less than original data") + } +}