01c0e45b5c
Now upon conditions - if checksum '.cksum' file is missing - or corrupted chunk (mismatching chunk) - or missing chunk entirely Decoding works consistently
83 lines
2.1 KiB
Go
83 lines
2.1 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"io/ioutil"
|
|
"log"
|
|
"os"
|
|
"strconv"
|
|
|
|
"github.com/codegangsta/cli"
|
|
"github.com/minio-io/minio/pkgs/checksum/crc32c"
|
|
"github.com/minio-io/minio/pkgs/erasure"
|
|
"github.com/minio-io/minio/pkgs/split"
|
|
)
|
|
|
|
func encode(c *cli.Context) {
|
|
// check if minio-encode called without parameters
|
|
if len(c.Args()) != 1 {
|
|
cli.ShowCommandHelp(c, "encode")
|
|
return
|
|
}
|
|
|
|
config, err := parseInput(c)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
// get file
|
|
inputFile, err := os.Open(config.input)
|
|
defer inputFile.Close()
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
// read file
|
|
input, err := ioutil.ReadAll(inputFile)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
// set up encoder
|
|
erasureParameters, _ := erasure.ParseEncoderParams(config.k, config.m, erasure.CAUCHY)
|
|
|
|
// Init new encoder
|
|
encoder := erasure.NewEncoder(erasureParameters)
|
|
|
|
// encode data
|
|
if config.blockSize == 0 {
|
|
encodedData, length := encoder.Encode(input)
|
|
for key, data := range encodedData {
|
|
crc, err := crc32c.Crc32c(data)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
crcString := strconv.Itoa(int(crc))
|
|
ioutil.WriteFile(config.output+"."+strconv.Itoa(key)+".cksum", []byte(crcString), 0600)
|
|
ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600)
|
|
ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600)
|
|
}
|
|
} else {
|
|
chunkCount := 0
|
|
splitChannel := make(chan split.SplitMessage)
|
|
inputReader := bytes.NewReader(input)
|
|
go split.SplitStream(inputReader, config.blockSize, splitChannel)
|
|
for chunk := range splitChannel {
|
|
if chunk.Err != nil {
|
|
log.Fatal(chunk.Err)
|
|
}
|
|
encodedData, length := encoder.Encode(chunk.Data)
|
|
for key, data := range encodedData {
|
|
crc, err := crc32c.Crc32c(data)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
crcString := strconv.Itoa(int(crc))
|
|
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key)+".cksum", []byte(crcString), 0600)
|
|
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key), data, 0600)
|
|
}
|
|
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+".length", []byte(strconv.Itoa(length)), 0600)
|
|
chunkCount += 1
|
|
}
|
|
}
|
|
}
|