maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge pull request #74 from fkautz/pr_out_adding_chunking_by_block_to_erasure_demo_via_block_size_parameter

Browse source 
Adding chunking by block to erasure-demo via --block-size parameter
This commit is contained in:
Harshavardhana 2014-12-01 21:45:17 -08:00
parent dac14cd7a1 46b08681a4
commit 39beabbbb6
3 changed files with 108 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

88
cmd/erasure-demo/decode.go
View file

@ -24,36 +24,68 @@ func decode(c *cli.Context) {
k := config.k
m := config.m
// get chunks
chunks := make([][]byte, k+m)
for i := 0; i < k+m; i++ {
chunks[i], _ = ioutil.ReadFile(config.input + "." + strconv.Itoa(i))
// check if output file exists, fail if so
if _, err := os.Stat(config.output); !os.IsNotExist(err) {
log.Fatal("Output file exists")
}
// get length
lengthBytes, err := ioutil.ReadFile(config.input + ".length")
if err != nil {
log.Fatal(err)
}
lengthString := string(lengthBytes)
length, err := strconv.Atoi(lengthString)
if err != nil {
log.Fatal(err)
}
// set up encoder
erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY)
// decode data
decodedData, err := erasure.Decode(chunks, erasureParameters, length)
if err != nil {
log.Fatal(err)
}
// write decode data out
if _, err := os.Stat(config.output); os.IsNotExist(err) {
ioutil.WriteFile(config.output, decodedData, 0600)
// get list of files
var inputFiles []string
if _, err := os.Stat(config.input + ".length"); os.IsNotExist(err) {
err = nil
chunkCount := 0
for !os.IsNotExist(err) {
_, err = os.Stat(config.input + "." + strconv.Itoa(chunkCount) + ".length")
chunkCount += 1
}
chunkCount = chunkCount - 1
inputFiles = make([]string, chunkCount)
for i := 0; i < chunkCount; i++ {
inputFiles[i] = config.input + "." + strconv.Itoa(i)
}
} else {
log.Fatal("Output file already exists")
inputFiles = []string{config.input}
}
// open file to write
outputFile, err := os.OpenFile(config.output, os.O_CREATE|os.O_WRONLY, 0600)
defer outputFile.Close()
if err != nil {
log.Fatal(err)
}
for _, inputFile := range inputFiles {
// get chunks
chunks := make([][]byte, k+m)
for i := 0; i < k+m; i++ {
chunks[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i))
}
// get length
lengthBytes, err := ioutil.ReadFile(inputFile + ".length")
if err != nil {
log.Fatal(err)
}
lengthString := string(lengthBytes)
length, err := strconv.Atoi(lengthString)
if err != nil {
log.Fatal(err)
}
// set up encoder
erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY)
// decode data
decodedData, err := erasure.Decode(chunks, erasureParameters, length)
if err != nil {
log.Fatal(err)
}
// append decoded data
length, err = outputFile.Write(decodedData)
if err != nil {
log.Fatal(err)
}
}
}

31
cmd/erasure-demo/encode.go
View file

@ -1,6 +1,7 @@
package main
import (
"bytes"
"io/ioutil"
"log"
"os"
@ -8,6 +9,7 @@ import (
"github.com/codegangsta/cli"
"github.com/minio-io/minio/pkgs/erasure"
"github.com/minio-io/minio/pkgs/split"
)
func encode(c *cli.Context) {
@ -24,6 +26,7 @@ func encode(c *cli.Context) {
// get file
inputFile, err := os.Open(config.input)
defer inputFile.Close()
if err != nil {
log.Fatal(err)
}
@ -37,11 +40,27 @@ func encode(c *cli.Context) {
// set up encoder
erasureParameters, _ := erasure.ParseEncoderParams(config.k, config.m, erasure.CAUCHY)
// encode data
encodedData, length := erasure.Encode(input, erasureParameters)
// write encoded data out
for key, data := range encodedData {
ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600)
if config.blockSize == 0 {
encodedData, length := erasure.Encode(input, erasureParameters)
for key, data := range encodedData {
ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600)
ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600)
}
} else {
chunkCount := 0
splitChannel := make(chan split.ByteMessage)
inputReader := bytes.NewReader(input)
go split.SplitStream(inputReader, config.blockSize, splitChannel)
for chunk := range splitChannel {
if chunk.Err != nil {
log.Fatal(chunk.Err)
}
encodedData, length := erasure.Encode(chunk.Data, erasureParameters)
for key, data := range encodedData {
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key), data, 0600)
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+".length", []byte(strconv.Itoa(length)), 0600)
}
chunkCount += 1
}
}
ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600)
}

28
cmd/erasure-demo/main.go
View file

@ -7,6 +7,7 @@ import (
"strings"
"github.com/codegangsta/cli"
"github.com/minio-io/minio/pkgs/strbyteconv"
)
func main() {
@ -29,6 +30,11 @@ func main() {
Value: "10,6",
Usage: "data,parity",
},
cli.StringFlag{
Name: "block-size",
Value: "1M",
Usage: "Size of blocks. Examples: 1K, 1M, full",
},
},
},
{
@ -58,7 +64,7 @@ type inputConfig struct {
output string
k int
m int
blockSize int
blockSize uint64
}
// parses input and returns an inputConfig with parsed input
@ -88,10 +94,22 @@ func parseInput(c *cli.Context) (inputConfig, error) {
return inputConfig{}, err
}
var blockSize uint64
blockSize = 0
if c.String("block-size") != "" {
if c.String("block-size") != "full" {
blockSize, err = strbyteconv.StringToBytes(c.String("block-size"))
if err != nil {
return inputConfig{}, err
}
}
}
return inputConfig{
input: inputFilePath,
output: outputFilePath,
k: k,
m: m,
input: inputFilePath,
output: outputFilePath,
k: k,
m: m,
blockSize: blockSize,
}, nil
}