maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

xl: Moved to minio/minio - fixes #1112

Browse source
This commit is contained in:
Harshavardhana 2016-02-10 16:40:09 -08:00
parent 33bd97d581
commit 62f6ffb6db
137 changed files with 9408 additions and 515 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -11,3 +11,5 @@ site/
**/access.log
ui-assets.go
ui-assets.asc
build
isa-l

14
.travis.yml
View file

@ -1,10 +1,21 @@
sudo: false
sudo: required
dist: trusty
language: go
os:
- linux
- osx
before_install:
- git clone https://github.com/yasm/yasm
- cd yasm
- git checkout v1.3.0
- "./autogen.sh"
- "./configure"
- make
- export PATH=$PATH:`pwd`
- cd ..
osx_image: xcode7.2
env:
@ -16,7 +27,6 @@ script:
- make test GOFLAGS="-race"
go:
- 1.5.2
- 1.5.3
notifications:

30
Makefile
View file

@ -48,6 +48,15 @@ fmt:
@GO15VENDOREXPERIMENT=1 gofmt -s -l *.go
@GO15VENDOREXPERIMENT=1 gofmt -s -l pkg
## Configure Intel library.
isa-l:
@echo "Configuring $@:"
@git clone -q https://github.com/minio/isa-l.git
@mkdir -p build
@cd build; $(PWD)/isa-l/configure --prefix $(PWD)/build/lib --sysconfdir $(PWD)/build/lib --includedir $(PWD)/build/lib --libdir $(PWD)/build/lib >/dev/null
@make -C build >/dev/null
@make -C build install >/dev/null
lint:
@echo "Running $@:"
@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/golint *.go
@ -58,8 +67,8 @@ cyclo:
@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 *.go
@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 pkg
build: getdeps verifiers $(UI_ASSETS)
@echo "Installing minio:"
build: getdeps verifiers $(UI_ASSETS) isa-l
@GO15VENDOREXPERIMENT=1 go generate ./...
deadcode:
@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/deadcode
@ -69,12 +78,13 @@ spelling:
@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/misspell pkg/**/*
test: build
@echo "Running all testing:"
@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) .
@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) github.com/minio/minio/pkg...
@echo "Running all minio testing:"
@CGO_CPPFLAGS="-I$(PWD)/build/lib" CGO_LDFLAGS="$(PWD)/build/lib/libisal.a" GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) .
@CGO_CPPFLAGS="-I$(PWD)/build/lib" CGO_LDFLAGS="$(PWD)/build/lib/libisal.a" GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) github.com/minio/minio/pkg...
gomake-all: build
@GO15VENDOREXPERIMENT=1 go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
@echo "Installing minio:"
@CGO_CPPFLAGS="-I$(PWD)/build/lib" CGO_LDFLAGS="$(PWD)/build/lib/libisal.a" GO15VENDOREXPERIMENT=1 go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
pkg-add:
@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor add $(PKG)
@ -100,9 +110,9 @@ release:
clean:
@echo "Cleaning up all the generated files:"
@rm -fv cover.out
@rm -fv minio
@rm -fv minio.test
@rm -fv pkg/fs/fs.test
@rm -fv minio minio.test cover.out
@find . -name '*.test' | xargs rm -fv
@rm -fv ui-assets.go
@rm -fv ui-assets.asc
@rm -rf isa-l
@rm -rf build

2
accesslog-handler.go
View file

@ -23,7 +23,7 @@ import (
"os"
"time"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/probe"
)
type accessLogHandler struct {

2
api-auth-utils.go
View file

@ -21,7 +21,7 @@ import (
"encoding/base64"
"regexp"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/probe"
)
const (

18
api-signature.go
View file

@ -26,8 +26,8 @@ import (
"strings"
"time"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/fs"
"github.com/minio/minio/pkg/probe"
v4 "github.com/minio/minio/pkg/signature"
)
const (
@ -125,7 +125,7 @@ func stripAccessKeyID(authHeaderValue string) (string, *probe.Error) {
}
// initSignatureV4 initializing signature verification.
func initSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) {
func initSignatureV4(req *http.Request) (*v4.Signature, *probe.Error) {
// strip auth from authorization header.
authHeaderValue := req.Header.Get("Authorization")
@ -156,7 +156,7 @@ func initSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) {
return nil, err.Trace(authHeaderValue)
}
if config.Credentials.AccessKeyID == accessKeyID {
signature := &fs.Signature{
signature := &v4.Signature{
AccessKeyID: config.Credentials.AccessKeyID,
SecretAccessKey: config.Credentials.SecretAccessKey,
Region: region,
@ -203,7 +203,7 @@ func applyPolicy(formValues map[string]string) *probe.Error {
if e != nil {
return probe.NewError(e)
}
postPolicyForm, err := fs.ParsePostPolicyForm(string(policyBytes))
postPolicyForm, err := v4.ParsePostPolicyForm(string(policyBytes))
if err != nil {
return err.Trace()
}
@ -244,7 +244,7 @@ func applyPolicy(formValues map[string]string) *probe.Error {
}
// initPostPresignedPolicyV4 initializing post policy signature verification
func initPostPresignedPolicyV4(formValues map[string]string) (*fs.Signature, *probe.Error) {
func initPostPresignedPolicyV4(formValues map[string]string) (*v4.Signature, *probe.Error) {
credentialElements := strings.Split(strings.TrimSpace(formValues["X-Amz-Credential"]), "/")
if len(credentialElements) != 5 {
return nil, probe.NewError(errCredentialTagMalformed)
@ -259,7 +259,7 @@ func initPostPresignedPolicyV4(formValues map[string]string) (*fs.Signature, *pr
}
region := credentialElements[2]
if config.Credentials.AccessKeyID == accessKeyID {
signature := &fs.Signature{
signature := &v4.Signature{
AccessKeyID: config.Credentials.AccessKeyID,
SecretAccessKey: config.Credentials.SecretAccessKey,
Region: region,
@ -272,7 +272,7 @@ func initPostPresignedPolicyV4(formValues map[string]string) (*fs.Signature, *pr
}
// initPresignedSignatureV4 initializing presigned signature verification
func initPresignedSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) {
func initPresignedSignatureV4(req *http.Request) (*v4.Signature, *probe.Error) {
credentialElements := strings.Split(strings.TrimSpace(req.URL.Query().Get("X-Amz-Credential")), "/")
if len(credentialElements) != 5 {
return nil, probe.NewError(errCredentialTagMalformed)
@ -289,7 +289,7 @@ func initPresignedSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) {
signedHeaders := strings.Split(strings.TrimSpace(req.URL.Query().Get("X-Amz-SignedHeaders")), ";")
signature := strings.TrimSpace(req.URL.Query().Get("X-Amz-Signature"))
if config.Credentials.AccessKeyID == accessKeyID {
signature := &fs.Signature{
signature := &v4.Signature{
AccessKeyID: config.Credentials.AccessKeyID,
SecretAccessKey: config.Credentials.SecretAccessKey,
Region: region,

28
appveyor.yml
View file

@ -1,30 +1,46 @@
# version format
version: "{build}"
# Operating system (build VM template)
os: Windows Server 2012 R2
os: Visual Studio 2015
platform: x64
clone_folder: c:\gopath\src\github.com\minio\minio
# environment variables
environment:
GOPATH: c:\gopath
GO_EXTLINK_ENABLED: 0
GO15VENDOREXPERIMENT: 1
UI_ASSETS: ui-assets.go
UI_ASSETS_ARMOR: ui-assets.asc
# scripts that run after cloning repository
install:
- '"C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64'
- curl -fsSL -o c:\go\bin\yasm.exe http://www.tortall.net/projects/yasm/releases/yasm-1.3.0-win64.exe
- set PATH=%GOPATH%\bin;c:\go\bin;%PATH%
- curl -fsSL -o mingw-w64.7z http://downloads.sourceforge.net/project/mingw-w64-dgn/mingw-w64/mingw-w64-bin-x86_64-20151206.7z
- 7z x -oC:\ mingw-w64.7z > NUL
- set PATH=C:\mingw64\bin;%PATH%
- x86_64-w64-mingw32-gcc --version
- go version
- go env
- git clone https://github.com/minio/isa-l
- cd isa-l
- make -f Makefile.unx arch=mingw
- mv include isa-l
- set CGO_CPPFLAGS=-Ic:/gopath/src/github.com/minio/minio/isa-l
- set CGO_LDFLAGS=c:/gopath/src/github.com/minio/minio/isa-l/isa-l.a
- set CC=x86_64-w64-mingw32-gcc
- set CXX=x86_64-w64-mingw32-g++
- cd %GOPATH%\src\github.com\minio\minio
# to run your custom scripts instead of automatic MSBuild
build_script:
- curl -o ui-assets.go -L https://dl.minio.io/assets/server/ui/%UI_ASSETS%
- curl -o ui-assets.asc -L https://dl.minio.io/assets/server/ui/%UI_ASSETS_ARMOR%
- curl -fsSL -o ui-assets.go https://dl.minio.io/assets/server/ui/%UI_ASSETS%
- curl -fsSL -o ui-assets.asc https://dl.minio.io/assets/server/ui/%UI_ASSETS_ARMOR%
- gpg --batch --no-tty --yes --keyserver pgp.mit.edu --recv-keys F9AAC728
- gpg --batch --no-tty --verify %UI_ASSETS_ARMOR% %UI_ASSETS%
- go generate ./...
- go test .
- go test -race .
- go test github.com/minio/minio/pkg...

9
bucket-handlers.go
View file

@ -22,9 +22,10 @@ import (
"net/http"
"github.com/gorilla/mux"
"github.com/minio/minio-xl/pkg/crypto/sha256"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/crypto/sha256"
"github.com/minio/minio/pkg/fs"
"github.com/minio/minio/pkg/probe"
v4 "github.com/minio/minio/pkg/signature"
)
// GetBucketLocationHandler - GET Bucket location.
@ -204,7 +205,7 @@ func (api CloudStorageAPI) PutBucketHandler(w http.ResponseWriter, req *http.Req
return
}
var signature *fs.Signature
var signature *v4.Signature
// Init signature V4 verification
if isRequestSignatureV4(req) {
var err *probe.Error
@ -340,7 +341,7 @@ func (api CloudStorageAPI) PostPolicyBucketHandler(w http.ResponseWriter, req *h
writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
case fs.BadDigest:
writeErrorResponse(w, req, BadDigest, req.URL.Path)
case fs.SignatureDoesNotMatch:
case v4.SigDoesNotMatch:
writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
case fs.IncompleteBody:
writeErrorResponse(w, req, IncompleteBody, req.URL.Path)

2
config-logger-main.go
View file

@ -20,7 +20,7 @@ import (
"runtime"
"github.com/minio/cli"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/probe"
)
// Configure logger

66
damerau-levenshtein.go Normal file
View file

@ -0,0 +1,66 @@
/*
* Minio Client (C) 2014-2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"math"
)
// Returns the minimum value of a slice of integers
func minimum(integers []int) (minVal int) {
minVal = math.MaxInt32
for _, v := range integers {
if v < minVal {
minVal = v
}
}
return
}
// DamerauLevenshteinDistance calculates distance between two strings using an algorithm
// described in https://en.wikipedia.org/wiki/Damerau-Levenshtein_distance
func DamerauLevenshteinDistance(a string, b string) int {
d := make([][]int, len(a)+1)
for i := 1; i <= len(a)+1; i++ {
d[i-1] = make([]int, len(b)+1)
}
for i := 0; i <= len(a); i++ {
d[i][0] = i
}
for j := 0; j <= len(b); j++ {
d[0][j] = j
}
for i := 1; i <= len(a); i++ {
for j := 1; j <= len(b); j++ {
cost := 0
if a[i-1] == b[j-1] {
cost = 0
} else {
cost = 1
}
d[i][j] = minimum([]int{
d[i-1][j] + 1,
d[i][j-1] + 1,
d[i-1][j-1] + cost,
})
if i > 1 && j > 1 && a[i-1] == b[j-2] && a[i-2] == b[j-1] {
d[i][j] = minimum([]int{d[i][j], d[i-2][j-2] + cost}) // transposition
}
}
}
return d[len(a)][len(b)]
}

2
httprange.go
View file

@ -22,8 +22,8 @@ import (
"strconv"
"strings"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/fs"
"github.com/minio/minio/pkg/probe"
)
const (

2
jwt.go
View file

@ -22,7 +22,7 @@ import (
"time"
jwtgo "github.com/dgrijalva/jwt-go"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/probe"
"golang.org/x/crypto/bcrypt"
)

2
logger-file-hook.go
View file

@ -21,7 +21,7 @@ import (
"os"
"github.com/Sirupsen/logrus"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/probe"
)
type localFile struct {

2
logger-mongo-hook.go
View file

@ -20,7 +20,7 @@ import (
"fmt"
"github.com/Sirupsen/logrus"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/probe"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
)

2
logger-syslog-hook_nix.go
View file

@ -23,7 +23,7 @@ import (
"log/syslog"
"github.com/Sirupsen/logrus"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/probe"
)
// syslogHook to send logs via syslog.

2
logger-syslog-hook_windows.go
View file

@ -18,7 +18,7 @@
package main
import "github.com/minio/minio-xl/pkg/probe"
import "github.com/minio/minio/pkg/probe"
func log2Syslog(network, raddr string) *probe.Error {
return probe.NewError(errSysLogNotSupported)

2
logger.go
View file

@ -21,7 +21,7 @@ import (
"reflect"
"github.com/Sirupsen/logrus"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/probe"
)
type fields map[string]interface{}

2
logger_test.go
View file

@ -22,7 +22,7 @@ import (
"errors"
"github.com/Sirupsen/logrus"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/probe"
. "gopkg.in/check.v1"
)

16
main.go
View file

@ -20,11 +20,12 @@ import (
"fmt"
"os"
"runtime"
"sort"
"strconv"
"github.com/dustin/go-humanize"
"github.com/minio/cli"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/probe"
)
// Help template for minio.
@ -101,6 +102,19 @@ func findClosestCommands(command string) []string {
for _, value := range commandsTree.PrefixMatch(command) {
closestCommands = append(closestCommands, value.(string))
}
sort.Strings(closestCommands)
// Suggest other close commands - allow missed, wrongly added and
// even transposed characters
for _, value := range commandsTree.walk(commandsTree.root) {
if sort.SearchStrings(closestCommands, value.(string)) < len(closestCommands) {
continue
}
// 2 is arbitrary and represents the max
// allowed number of typed errors
if DamerauLevenshteinDistance(command, value.(string)) < 2 {
closestCommands = append(closestCommands, value.(string))
}
}
return closestCommands
}

2
notifier.go
View file

@ -23,7 +23,7 @@ import (
"strings"
"github.com/fatih/color"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/probe"
"github.com/olekukonko/ts"
)

15
object-handlers.go
View file

@ -22,8 +22,9 @@ import (
"strconv"
"github.com/gorilla/mux"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/fs"
"github.com/minio/minio/pkg/probe"
v4 "github.com/minio/minio/pkg/signature"
)
const (
@ -172,7 +173,7 @@ func (api CloudStorageAPI) PutObjectHandler(w http.ResponseWriter, req *http.Req
return
}
var signature *fs.Signature
var signature *v4.Signature
if isRequestSignatureV4(req) {
// Init signature V4 verification
var err *probe.Error
@ -209,7 +210,7 @@ func (api CloudStorageAPI) PutObjectHandler(w http.ResponseWriter, req *http.Req
writeErrorResponse(w, req, BadDigest, req.URL.Path)
case fs.MissingDateHeader:
writeErrorResponse(w, req, RequestTimeTooSkewed, req.URL.Path)
case fs.SignatureDoesNotMatch:
case v4.SigDoesNotMatch:
writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
case fs.IncompleteBody:
writeErrorResponse(w, req, IncompleteBody, req.URL.Path)
@ -320,7 +321,7 @@ func (api CloudStorageAPI) PutObjectPartHandler(w http.ResponseWriter, req *http
}
}
var signature *fs.Signature
var signature *v4.Signature
if isRequestSignatureV4(req) {
// Init signature V4 verification
var err *probe.Error
@ -353,7 +354,7 @@ func (api CloudStorageAPI) PutObjectPartHandler(w http.ResponseWriter, req *http
writeErrorResponse(w, req, NoSuchUpload, req.URL.Path)
case fs.BadDigest:
writeErrorResponse(w, req, BadDigest, req.URL.Path)
case fs.SignatureDoesNotMatch:
case v4.SigDoesNotMatch:
writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
case fs.IncompleteBody:
writeErrorResponse(w, req, IncompleteBody, req.URL.Path)
@ -475,7 +476,7 @@ func (api CloudStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter,
}
objectResourcesMetadata := getObjectResources(req.URL.Query())
var signature *fs.Signature
var signature *v4.Signature
if isRequestSignatureV4(req) {
// Init signature V4 verification
var err *probe.Error
@ -516,7 +517,7 @@ func (api CloudStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter,
writeErrorResponse(w, req, InvalidPart, req.URL.Path)
case fs.InvalidPartOrder:
writeErrorResponse(w, req, InvalidPartOrder, req.URL.Path)
case fs.SignatureDoesNotMatch:
case v4.SigDoesNotMatch:
writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
case fs.IncompleteBody:
writeErrorResponse(w, req, IncompleteBody, req.URL.Path)

2
vendor/github.com/minio/minio-xl/pkg/atomic/atomic.go → pkg/atomic/atomic.go
View file

@ -1,5 +1,5 @@
/*
* Minio Client (C) 2015 Minio, Inc.
* Minio Cloud Storage (C) 2015-2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
vendor/github.com/minio/minio-xl/pkg/atomic/atomic_test.go → pkg/atomic/atomic_test.go
View file

@ -34,7 +34,7 @@ type MySuite struct {
var _ = Suite(&MySuite{})
func (s *MySuite) SetUpSuite(c *C) {
root, err := ioutil.TempDir("/tmp", "atomic-")
root, err := ioutil.TempDir(os.TempDir(), "atomic-")
c.Assert(err, IsNil)
s.root = root
}

0
vendor/github.com/minio/minio-xl/pkg/cpu/cpu_amd64.go → pkg/cpu/cpu_amd64.go
View file

0
vendor/github.com/minio/minio-xl/pkg/cpu/cpu_amd64.s → pkg/cpu/cpu_amd64.s
View file

0
vendor/github.com/minio/minio-xl/pkg/cpu/cpu_arm.go → pkg/cpu/cpu_arm.go
View file

2
vendor/github.com/minio/minio-xl/pkg/cpu/cpu_test.go → pkg/cpu/cpu_test.go
View file

@ -23,7 +23,7 @@ import (
"strings"
"testing"
"github.com/minio/minio-xl/pkg/cpu"
"github.com/minio/minio/pkg/cpu"
. "gopkg.in/check.v1"
)

0
vendor/github.com/minio/minio-xl/pkg/cpu/doc.go → pkg/cpu/doc.go
View file

1
pkg/crypto/sha1/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
*.syso

0
vendor/github.com/minio/minio-xl/pkg/crypto/sha256/LICENSE → pkg/crypto/sha1/LICENSE
View file

150
pkg/crypto/sha1/sha1.go Normal file
View file

@ -0,0 +1,150 @@
/*
* Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file of
// Golang project:
// https://github.com/golang/go/blob/master/LICENSE
// Using this part of Minio codebase under the license
// Apache License Version 2.0 with modifications
// Package sha1 implements the SHA1 hash algorithm as defined in RFC 3174.
package sha1
import "hash"
// Size - The size of a SHA1 checksum in bytes.
const Size = 20
// BlockSize - The blocksize of SHA1 in bytes.
const BlockSize = 64
const (
chunk = 64
init0 = 0x67452301
init1 = 0xEFCDAB89
init2 = 0x98BADCFE
init3 = 0x10325476
init4 = 0xC3D2E1F0
)
// digest represents the partial evaluation of a checksum.
type digest struct {
h [5]uint32
x [chunk]byte
nx int
len uint64
}
// Reset digest
func (d *digest) Reset() {
d.h[0] = init0
d.h[1] = init1
d.h[2] = init2
d.h[3] = init3
d.h[4] = init4
d.nx = 0
d.len = 0
}
// New returns a new hash.Hash computing the SHA1 checksum.
func New() hash.Hash {
d := new(digest)
d.Reset()
return d
}
// Return output size
func (d *digest) Size() int { return Size }
// Return checksum blocksize
func (d *digest) BlockSize() int { return BlockSize }
// Write to digest
func (d *digest) Write(p []byte) (nn int, err error) {
nn = len(p)
d.len += uint64(nn)
if d.nx > 0 {
n := copy(d.x[d.nx:], p)
d.nx += n
if d.nx == chunk {
block(d, d.x[:])
d.nx = 0
}
p = p[n:]
}
if len(p) >= chunk {
n := len(p) &^ (chunk - 1)
block(d, p[:n])
p = p[n:]
}
if len(p) > 0 {
d.nx = copy(d.x[:], p)
}
return
}
// Return checksum bytes
func (d *digest) Sum(in []byte) []byte {
// Make a copy of d0 so that caller can keep writing and summing.
d0 := *d
hash := d0.checkSum()
return append(in, hash[:]...)
}
// Intermediate checksum function
func (d *digest) checkSum() [Size]byte {
len := d.len
// Padding. Add a 1 bit and 0 bits until 56 bytes mod 64.
var tmp [64]byte
tmp[0] = 0x80
if len%64 < 56 {
d.Write(tmp[0 : 56-len%64])
} else {
d.Write(tmp[0 : 64+56-len%64])
}
// Length in bits.
len <<= 3
for i := uint(0); i < 8; i++ {
tmp[i] = byte(len >> (56 - 8*i))
}
d.Write(tmp[0:8])
if d.nx != 0 {
panic("d.nx != 0")
}
var digest [Size]byte
for i, s := range d.h {
digest[i*4] = byte(s >> 24)
digest[i*4+1] = byte(s >> 16)
digest[i*4+2] = byte(s >> 8)
digest[i*4+3] = byte(s)
}
return digest
}
// Sum - single caller sha1 helper
func Sum(data []byte) [Size]byte {
var d digest
d.Reset()
d.Write(data)
return d.checkSum()
}

967
pkg/crypto/sha1/sha1_linux.S Normal file
View file

@ -0,0 +1,967 @@
/*
* Implement fast SHA-1 with AVX2 instructions. (x86_64)
*
* This file is provided under a dual BSD/GPLv2 license. When using or
* redistributing this file, you may do so under either license.
*
* GPL LICENSE SUMMARY
*
* Copyright(c) 2014 Intel Corporation.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of version 2 of the GNU General Public License as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* Contact Information:
* Ilya Albrekht <ilya.albrekht@intel.com>
* Maxim Locktyukhin <maxim.locktyukhin@intel.com>
* Ronen Zohar <ronen.zohar@intel.com>
* Chandramouli Narayanan <mouli@linux.intel.com>
*
* BSD LICENSE
*
* Copyright(c) 2014 Intel Corporation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
/*
* SHA-1 implementation with Intel(R) AVX2 instruction set extensions.
*
*This implementation is based on the previous SSSE3 release:
* https://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1
*
*Updates 20-byte SHA-1 record in 'hash' for even number of
*'num_blocks' consecutive 64-byte blocks
*
*/
/*
* Using this part of Minio codebase under the license
* Apache License Version 2.0 with modifications
*
*/
#ifdef HAS_AVX2
#ifndef ENTRY
#define ENTRY(name) \
.globl name ; \
.align 4,0x90 ; \
name:
#endif
#ifndef END
#define END(name) \
.size name, .-name
#endif
#ifndef ENDPROC
#define ENDPROC(name) \
.type name, @function ; \
END(name)
#endif
#define NUM_INVALID 100
#define TYPE_R32 0
#define TYPE_R64 1
#define TYPE_XMM 2
#define TYPE_INVALID 100
.macro R32_NUM opd r32
\opd = NUM_INVALID
.ifc \r32,%eax
\opd = 0
.endif
.ifc \r32,%ecx
\opd = 1
.endif
.ifc \r32,%edx
\opd = 2
.endif
.ifc \r32,%ebx
\opd = 3
.endif
.ifc \r32,%esp
\opd = 4
.endif
.ifc \r32,%ebp
\opd = 5
.endif
.ifc \r32,%esi
\opd = 6
.endif
.ifc \r32,%edi
\opd = 7
.endif
#ifdef X86_64
.ifc \r32,%r8d
\opd = 8
.endif
.ifc \r32,%r9d
\opd = 9
.endif
.ifc \r32,%r10d
\opd = 10
.endif
.ifc \r32,%r11d
\opd = 11
.endif
.ifc \r32,%r12d
\opd = 12
.endif
.ifc \r32,%r13d
\opd = 13
.endif
.ifc \r32,%r14d
\opd = 14
.endif
.ifc \r32,%r15d
\opd = 15
.endif
#endif
.endm
.macro R64_NUM opd r64
\opd = NUM_INVALID
#ifdef X86_64
.ifc \r64,%rax
\opd = 0
.endif
.ifc \r64,%rcx
\opd = 1
.endif
.ifc \r64,%rdx
\opd = 2
.endif
.ifc \r64,%rbx
\opd = 3
.endif
.ifc \r64,%rsp
\opd = 4
.endif
.ifc \r64,%rbp
\opd = 5
.endif
.ifc \r64,%rsi
\opd = 6
.endif
.ifc \r64,%rdi
\opd = 7
.endif
.ifc \r64,%r8
\opd = 8
.endif
.ifc \r64,%r9
\opd = 9
.endif
.ifc \r64,%r10
\opd = 10
.endif
.ifc \r64,%r11
\opd = 11
.endif
.ifc \r64,%r12
\opd = 12
.endif
.ifc \r64,%r13
\opd = 13
.endif
.ifc \r64,%r14
\opd = 14
.endif
.ifc \r64,%r15
\opd = 15
.endif
#endif
.endm
.macro XMM_NUM opd xmm
\opd = NUM_INVALID
.ifc \xmm,%xmm0
\opd = 0
.endif
.ifc \xmm,%xmm1
\opd = 1
.endif
.ifc \xmm,%xmm2
\opd = 2
.endif
.ifc \xmm,%xmm3
\opd = 3
.endif
.ifc \xmm,%xmm4
\opd = 4
.endif
.ifc \xmm,%xmm5
\opd = 5
.endif
.ifc \xmm,%xmm6
\opd = 6
.endif
.ifc \xmm,%xmm7
\opd = 7
.endif
.ifc \xmm,%xmm8
\opd = 8
.endif
.ifc \xmm,%xmm9
\opd = 9
.endif
.ifc \xmm,%xmm10
\opd = 10
.endif
.ifc \xmm,%xmm11
\opd = 11
.endif
.ifc \xmm,%xmm12
\opd = 12
.endif
.ifc \xmm,%xmm13
\opd = 13
.endif
.ifc \xmm,%xmm14
\opd = 14
.endif
.ifc \xmm,%xmm15
\opd = 15
.endif
.endm
.macro TYPE type reg
R32_NUM reg_type_r32 \reg
R64_NUM reg_type_r64 \reg
XMM_NUM reg_type_xmm \reg
.if reg_type_r64 <> NUM_INVALID
\type = TYPE_R64
.elseif reg_type_r32 <> NUM_INVALID
\type = TYPE_R32
.elseif reg_type_xmm <> NUM_INVALID
\type = TYPE_XMM
.else
\type = TYPE_INVALID
.endif
.endm
.macro PFX_OPD_SIZE
.byte 0x66
.endm
.macro PFX_REX opd1 opd2 W=0
.if ((\opd1 | \opd2) & 8) || \W
.byte 0x40 | ((\opd1 & 8) >> 3) | ((\opd2 & 8) >> 1) | (\W << 3)
.endif
.endm
.macro MODRM mod opd1 opd2
.byte \mod | (\opd1 & 7) | ((\opd2 & 7) << 3)
.endm
.macro PSHUFB_XMM xmm1 xmm2
XMM_NUM pshufb_opd1 \xmm1
XMM_NUM pshufb_opd2 \xmm2
PFX_OPD_SIZE
PFX_REX pshufb_opd1 pshufb_opd2
.byte 0x0f, 0x38, 0x00
MODRM 0xc0 pshufb_opd1 pshufb_opd2
.endm
.macro PCLMULQDQ imm8 xmm1 xmm2
XMM_NUM clmul_opd1 \xmm1
XMM_NUM clmul_opd2 \xmm2
PFX_OPD_SIZE
PFX_REX clmul_opd1 clmul_opd2
.byte 0x0f, 0x3a, 0x44
MODRM 0xc0 clmul_opd1 clmul_opd2
.byte \imm8
.endm
.macro PEXTRD imm8 xmm gpr
R32_NUM extrd_opd1 \gpr
XMM_NUM extrd_opd2 \xmm
PFX_OPD_SIZE
PFX_REX extrd_opd1 extrd_opd2
.byte 0x0f, 0x3a, 0x16
MODRM 0xc0 extrd_opd1 extrd_opd2
.byte \imm8
.endm
.macro MOVQ_R64_XMM opd1 opd2
TYPE movq_r64_xmm_opd1_type \opd1
.if movq_r64_xmm_opd1_type == TYPE_XMM
XMM_NUM movq_r64_xmm_opd1 \opd1
R64_NUM movq_r64_xmm_opd2 \opd2
.else
R64_NUM movq_r64_xmm_opd1 \opd1
XMM_NUM movq_r64_xmm_opd2 \opd2
.endif
PFX_OPD_SIZE
PFX_REX movq_r64_xmm_opd1 movq_r64_xmm_opd2 1
.if movq_r64_xmm_opd1_type == TYPE_XMM
.byte 0x0f, 0x7e
.else
.byte 0x0f, 0x6e
.endif
MODRM 0xc0 movq_r64_xmm_opd1 movq_r64_xmm_opd2
.endm
#define CTX %rdi /* arg1 */
#define BUF %rsi /* arg2 */
#define CNT %rdx /* arg3 */
#define REG_A %ecx
#define REG_B %esi
#define REG_C %edi
#define REG_D %eax
#define REG_E %edx
#define REG_TB %ebx
#define REG_TA %r12d
#define REG_RA %rcx
#define REG_RB %rsi
#define REG_RC %rdi
#define REG_RD %rax
#define REG_RE %rdx
#define REG_RTA %r12
#define REG_RTB %rbx
#define REG_T1 %ebp
#define xmm_mov vmovups
#define avx2_zeroupper vzeroupper
#define RND_F1 1
#define RND_F2 2
#define RND_F3 3
.macro REGALLOC
.set A, REG_A
.set B, REG_B
.set C, REG_C
.set D, REG_D
.set E, REG_E
.set TB, REG_TB
.set TA, REG_TA
.set RA, REG_RA
.set RB, REG_RB
.set RC, REG_RC
.set RD, REG_RD
.set RE, REG_RE
.set RTA, REG_RTA
.set RTB, REG_RTB
.set T1, REG_T1
.endm
#define K_BASE %r8
#define HASH_PTR %r9
#define BUFFER_PTR %r10
#define BUFFER_PTR2 %r13
#define BUFFER_END %r11
#define PRECALC_BUF %r14
#define WK_BUF %r15
#define W_TMP %xmm0
#define WY_TMP %ymm0
#define WY_TMP2 %ymm9
# AVX2 variables
#define WY0 %ymm3
#define WY4 %ymm5
#define WY08 %ymm7
#define WY12 %ymm8
#define WY16 %ymm12
#define WY20 %ymm13
#define WY24 %ymm14
#define WY28 %ymm15
#define YMM_SHUFB_BSWAP %ymm10
/*
* Keep 2 iterations precalculated at a time:
* - 80 DWORDs per iteration * 2
*/
#define W_SIZE (80*2*2 +16)
#define WK(t) ((((t) % 80) / 4)*32 + ( (t) % 4)*4 + ((t)/80)*16 )(WK_BUF)
#define PRECALC_WK(t) ((t)*2*2)(PRECALC_BUF)
.macro UPDATE_HASH hash, val
add \hash, \val
mov \val, \hash
.endm
.macro PRECALC_RESET_WY
.set WY_00, WY0
.set WY_04, WY4
.set WY_08, WY08
.set WY_12, WY12
.set WY_16, WY16
.set WY_20, WY20
.set WY_24, WY24
.set WY_28, WY28
.set WY_32, WY_00
.endm
.macro PRECALC_ROTATE_WY
/* Rotate macros */
.set WY_32, WY_28
.set WY_28, WY_24
.set WY_24, WY_20
.set WY_20, WY_16
.set WY_16, WY_12
.set WY_12, WY_08
.set WY_08, WY_04
.set WY_04, WY_00
.set WY_00, WY_32
/* Define register aliases */
.set WY, WY_00
.set WY_minus_04, WY_04
.set WY_minus_08, WY_08
.set WY_minus_12, WY_12
.set WY_minus_16, WY_16
.set WY_minus_20, WY_20
.set WY_minus_24, WY_24
.set WY_minus_28, WY_28
.set WY_minus_32, WY
.endm
.macro PRECALC_00_15
.if (i == 0) # Initialize and rotate registers
PRECALC_RESET_WY
PRECALC_ROTATE_WY
.endif
/* message scheduling pre-compute for rounds 0-15 */
.if ((i & 7) == 0)
/*
* blended AVX2 and ALU instruction scheduling
* 1 vector iteration per 8 rounds
*/
vmovdqu ((i * 2) + PRECALC_OFFSET)(BUFFER_PTR), W_TMP
.elseif ((i & 7) == 1)
vinsertf128 $1, (((i-1) * 2)+PRECALC_OFFSET)(BUFFER_PTR2),\
WY_TMP, WY_TMP
.elseif ((i & 7) == 2)
vpshufb YMM_SHUFB_BSWAP, WY_TMP, WY
.elseif ((i & 7) == 4)
vpaddd K_XMM(K_BASE), WY, WY_TMP
.elseif ((i & 7) == 7)
vmovdqu WY_TMP, PRECALC_WK(i&~7)
PRECALC_ROTATE_WY
.endif
.endm
.macro PRECALC_16_31
/*
* message scheduling pre-compute for rounds 16-31
* calculating last 32 w[i] values in 8 XMM registers
* pre-calculate K+w[i] values and store to mem
* for later load by ALU add instruction
*
* "brute force" vectorization for rounds 16-31 only
* due to w[i]->w[i-3] dependency
*/
.if ((i & 7) == 0)
/*
* blended AVX2 and ALU instruction scheduling
* 1 vector iteration per 8 rounds
*/
/* w[i-14] */
vpalignr $8, WY_minus_16, WY_minus_12, WY
vpsrldq $4, WY_minus_04, WY_TMP /* w[i-3] */
.elseif ((i & 7) == 1)
vpxor WY_minus_08, WY, WY
vpxor WY_minus_16, WY_TMP, WY_TMP
.elseif ((i & 7) == 2)
vpxor WY_TMP, WY, WY
vpslldq $12, WY, WY_TMP2
.elseif ((i & 7) == 3)
vpslld $1, WY, WY_TMP
vpsrld $31, WY, WY
.elseif ((i & 7) == 4)
vpor WY, WY_TMP, WY_TMP
vpslld $2, WY_TMP2, WY
.elseif ((i & 7) == 5)
vpsrld $30, WY_TMP2, WY_TMP2
vpxor WY, WY_TMP, WY_TMP
.elseif ((i & 7) == 7)
vpxor WY_TMP2, WY_TMP, WY
vpaddd K_XMM(K_BASE), WY, WY_TMP
vmovdqu WY_TMP, PRECALC_WK(i&~7)
PRECALC_ROTATE_WY
.endif
.endm
.macro PRECALC_32_79
/*
* in SHA-1 specification:
* w[i] = (w[i-3] ^ w[i-8] ^ w[i-14] ^ w[i-16]) rol 1
* instead we do equal:
* w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2
* allows more efficient vectorization
* since w[i]=>w[i-3] dependency is broken
*/
.if ((i & 7) == 0)
/*
* blended AVX2 and ALU instruction scheduling
* 1 vector iteration per 8 rounds
*/
vpalignr $8, WY_minus_08, WY_minus_04, WY_TMP
.elseif ((i & 7) == 1)
/* W is W_minus_32 before xor */
vpxor WY_minus_28, WY, WY
.elseif ((i & 7) == 2)
vpxor WY_minus_16, WY_TMP, WY_TMP
.elseif ((i & 7) == 3)
vpxor WY_TMP, WY, WY
.elseif ((i & 7) == 4)
vpslld $2, WY, WY_TMP
.elseif ((i & 7) == 5)
vpsrld $30, WY, WY
vpor WY, WY_TMP, WY
.elseif ((i & 7) == 7)
vpaddd K_XMM(K_BASE), WY, WY_TMP
vmovdqu WY_TMP, PRECALC_WK(i&~7)
PRECALC_ROTATE_WY
.endif
.endm
.macro PRECALC r, s
.set i, \r
.if (i < 40)
.set K_XMM, 32*0
.elseif (i < 80)
.set K_XMM, 32*1
.elseif (i < 120)
.set K_XMM, 32*2
.else
.set K_XMM, 32*3
.endif
.if (i<32)
PRECALC_00_15 \s
.elseif (i<64)
PRECALC_16_31 \s
.elseif (i < 160)
PRECALC_32_79 \s
.endif
.endm
.macro ROTATE_STATE
.set T_REG, E
.set E, D
.set D, C
.set C, B
.set B, TB
.set TB, A
.set A, T_REG
.set T_REG, RE
.set RE, RD
.set RD, RC
.set RC, RB
.set RB, RTB
.set RTB, RA
.set RA, T_REG
.endm
/* Macro relies on saved ROUND_Fx */
.macro RND_FUN f, r
.if (\f == RND_F1)
ROUND_F1 \r
.elseif (\f == RND_F2)
ROUND_F2 \r
.elseif (\f == RND_F3)
ROUND_F3 \r
.endif
.endm
.macro RR r
.set round_id, (\r % 80)
.if (round_id == 0) /* Precalculate F for first round */
.set ROUND_FUNC, RND_F1
mov B, TB
rorx $(32-30), B, B /* b>>>2 */
andn D, TB, T1
and C, TB
xor T1, TB
.endif
RND_FUN ROUND_FUNC, \r
ROTATE_STATE
.if (round_id == 18)
.set ROUND_FUNC, RND_F2
.elseif (round_id == 38)
.set ROUND_FUNC, RND_F3
.elseif (round_id == 58)
.set ROUND_FUNC, RND_F2
.endif
.set round_id, ( (\r+1) % 80)
RND_FUN ROUND_FUNC, (\r+1)
ROTATE_STATE
.endm
.macro ROUND_F1 r
add WK(\r), E
andn C, A, T1 /* ~b&d */
lea (RE,RTB), E /* Add F from the previous round */
rorx $(32-5), A, TA /* T2 = A >>> 5 */
rorx $(32-30),A, TB /* b>>>2 for next round */
PRECALC (\r) /* msg scheduling for next 2 blocks */
/*
* Calculate F for the next round
* (b & c) ^ andn[b, d]
*/
and B, A /* b&c */
xor T1, A /* F1 = (b&c) ^ (~b&d) */
lea (RE,RTA), E /* E += A >>> 5 */
.endm
.macro ROUND_F2 r
add WK(\r), E
lea (RE,RTB), E /* Add F from the previous round */
/* Calculate F for the next round */
rorx $(32-5), A, TA /* T2 = A >>> 5 */
.if ((round_id) < 79)
rorx $(32-30), A, TB /* b>>>2 for next round */
.endif
PRECALC (\r) /* msg scheduling for next 2 blocks */
.if ((round_id) < 79)
xor B, A
.endif
add TA, E /* E += A >>> 5 */
.if ((round_id) < 79)
xor C, A
.endif
.endm
.macro ROUND_F3 r
add WK(\r), E
PRECALC (\r) /* msg scheduling for next 2 blocks */
lea (RE,RTB), E /* Add F from the previous round */
mov B, T1
or A, T1
rorx $(32-5), A, TA /* T2 = A >>> 5 */
rorx $(32-30), A, TB /* b>>>2 for next round */
/* Calculate F for the next round
* (b and c) or (d and (b or c))
*/
and C, T1
and B, A
or T1, A
add TA, E /* E += A >>> 5 */
.endm
/*
* macro implements 80 rounds of SHA-1, for multiple blocks with s/w pipelining
*/
.macro SHA1_PIPELINED_MAIN_BODY
REGALLOC
mov (HASH_PTR), A
mov 4(HASH_PTR), B
mov 8(HASH_PTR), C
mov 12(HASH_PTR), D
mov 16(HASH_PTR), E
mov %rsp, PRECALC_BUF
lea (2*4*80+32)(%rsp), WK_BUF
# Precalc WK for first 2 blocks
PRECALC_OFFSET = 0
.set i, 0
.rept 160
PRECALC i
.set i, i + 1
.endr
PRECALC_OFFSET = 128
xchg WK_BUF, PRECALC_BUF
.align 32
_loop:
/*
* code loops through more than one block
* we use K_BASE value as a signal of a last block,
* it is set below by: cmovae BUFFER_PTR, K_BASE
*/
cmp K_BASE, BUFFER_PTR
jne _begin
.align 32
jmp _end
.align 32
_begin:
/*
* Do first block
* rounds: 0,2,4,6,8
*/
.set j, 0
.rept 5
RR j
.set j, j+2
.endr
jmp _loop0
_loop0:
/*
* rounds:
* 10,12,14,16,18
* 20,22,24,26,28
* 30,32,34,36,38
* 40,42,44,46,48
* 50,52,54,56,58
*/
.rept 25
RR j
.set j, j+2
.endr
add $(2*64), BUFFER_PTR /* move to next odd-64-byte block */
cmp BUFFER_END, BUFFER_PTR /* is current block the last one? */
cmovae K_BASE, BUFFER_PTR /* signal the last iteration smartly */
/*
* rounds
* 60,62,64,66,68
* 70,72,74,76,78
*/
.rept 10
RR j
.set j, j+2
.endr
UPDATE_HASH (HASH_PTR), A
UPDATE_HASH 4(HASH_PTR), TB
UPDATE_HASH 8(HASH_PTR), C
UPDATE_HASH 12(HASH_PTR), D
UPDATE_HASH 16(HASH_PTR), E
cmp K_BASE, BUFFER_PTR /* is current block the last one? */
je _loop
mov TB, B
/* Process second block */
/*
* rounds
* 0+80, 2+80, 4+80, 6+80, 8+80
* 10+80,12+80,14+80,16+80,18+80
*/
.set j, 0
.rept 10
RR j+80
.set j, j+2
.endr
jmp _loop1
_loop1:
/*
* rounds
* 20+80,22+80,24+80,26+80,28+80
* 30+80,32+80,34+80,36+80,38+80
*/
.rept 10
RR j+80
.set j, j+2
.endr
jmp _loop2
_loop2:
/*
* rounds
* 40+80,42+80,44+80,46+80,48+80
* 50+80,52+80,54+80,56+80,58+80
*/
.rept 10
RR j+80
.set j, j+2
.endr
add $(2*64), BUFFER_PTR2 /* move to next even-64-byte block */
cmp BUFFER_END, BUFFER_PTR2 /* is current block the last one */
cmovae K_BASE, BUFFER_PTR /* signal the last iteration smartly */
jmp _loop3
_loop3:
/*
* rounds
* 60+80,62+80,64+80,66+80,68+80
* 70+80,72+80,74+80,76+80,78+80
*/
.rept 10
RR j+80
.set j, j+2
.endr
UPDATE_HASH (HASH_PTR), A
UPDATE_HASH 4(HASH_PTR), TB
UPDATE_HASH 8(HASH_PTR), C
UPDATE_HASH 12(HASH_PTR), D
UPDATE_HASH 16(HASH_PTR), E
/* Reset state for AVX2 reg permutation */
mov A, TA
mov TB, A
mov C, TB
mov E, C
mov D, B
mov TA, D
REGALLOC
xchg WK_BUF, PRECALC_BUF
jmp _loop
.align 32
_end:
.endm
.section .rodata
#define K1 0x5a827999
#define K2 0x6ed9eba1
#define K3 0x8f1bbcdc
#define K4 0xca62c1d6
.align 128
K_XMM_AR:
.long K1, K1, K1, K1
.long K1, K1, K1, K1
.long K2, K2, K2, K2
.long K2, K2, K2, K2
.long K3, K3, K3, K3
.long K3, K3, K3, K3
.long K4, K4, K4, K4
.long K4, K4, K4, K4
BSWAP_SHUFB_CTL:
.long 0x00010203
.long 0x04050607
.long 0x08090a0b
.long 0x0c0d0e0f
.long 0x00010203
.long 0x04050607
.long 0x08090a0b
.long 0x0c0d0e0f
# void sha1_transform(int32_t *hash, const char* input, size_t num_blocks) ;
.text
ENTRY(sha1_transform)
push %rbx
push %rbp
push %r12
push %r13
push %r14
push %r15
RESERVE_STACK = (W_SIZE*4 + 8+24)
/* Align stack */
mov %rsp, %rbx
and $~(0x20-1), %rsp
push %rbx
sub $RESERVE_STACK, %rsp
avx2_zeroupper
lea K_XMM_AR(%rip), K_BASE
mov CTX, HASH_PTR
mov BUF, BUFFER_PTR
lea 64(BUF), BUFFER_PTR2
shl $6, CNT /* mul by 64 */
add BUF, CNT
add $64, CNT
mov CNT, BUFFER_END
cmp BUFFER_END, BUFFER_PTR2
cmovae K_BASE, BUFFER_PTR2
xmm_mov BSWAP_SHUFB_CTL(%rip), YMM_SHUFB_BSWAP
SHA1_PIPELINED_MAIN_BODY
avx2_zeroupper
add $RESERVE_STACK, %rsp
pop %rsp
pop %r15
pop %r14
pop %r13
pop %r12
pop %rbp
pop %rbx
ret
ENDPROC(sha1_transform)
#endif

579
pkg/crypto/sha1/sha1_sse3_amd64.asm Normal file
View file

@ -0,0 +1,579 @@
;---------------------
; https://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1
;
; License information:
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; This implementation notably advances the performance of SHA-1 algorithm compared to existing
; implementations. We are encouraging all projects utilizing SHA-1 to integrate this new fast
; implementation and are ready to help if issues or concerns arise (you are welcome to leave
; a comment or write an email to the authors). It is provided 'as is' and free for either
; commercial or non-commercial use.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;
; This code implements two interfaces of SHA-1 update function: 1) working on a single
; 64-byte block and 2) working on a buffer of multiple 64-bit blocks. Multiple blocks
; version of code is software pipelined and faster overall, it is a default. Assemble
; with -DINTEL_SHA1_SINGLEBLOCK to select single 64-byte block function interface.
;
; C++ prototypes of implemented functions are below:
;
; #ifndef INTEL_SHA1_SINGLEBLOCK
; // Updates 20-byte SHA-1 record in 'hash' for 'num_blocks' consequtive 64-byte blocks
; extern "C" void sha1_update_intel(int *hash, const char* input, size_t num_blocks );
; #else
; // Updates 20-byte SHA-1 record in 'hash' for one 64-byte block pointed by 'input'
; extern "C" void sha1_update_intel(int *hash, const char* input);
; #endif
;
; Function name 'sha1_update_intel' can be changed in the source or via macro:
; -DINTEL_SHA1_UPDATE_FUNCNAME=my_sha1_update_func_name
;
; It implements both UNIX(default) and Windows ABIs, use -DWIN_ABI on Windows
;
; Code checks CPU for SSSE3 support via CPUID feature flag (CPUID.1.ECX.SSSE3[bit 9]==1),
; and performs dispatch. Since in most cases the functionality on non-SSSE3 supporting CPUs
; is also required, the default (e.g. one being replaced) function can be provided for
; dispatch on such CPUs, the name of old function can be changed in the source or via macro:
; -DINTEL_SHA1_UPDATE_DEFAULT_DISPATCH=default_sha1_update_function_name
;
; Authors: Maxim Locktyukhin and Ronen Zohar at Intel.com
;
%ifndef INTEL_SHA1_UPDATE_DEFAULT_DISPATCH
;; can be replaced with a default SHA-1 update function name
%define INTEL_SHA1_UPDATE_DEFAULT_DISPATCH sha1_intel_non_ssse3_cpu_stub_
%else
extern INTEL_SHA1_UPDATE_DEFAULT_DISPATCH
%endif
;; provide alternative SHA-1 update function's name here
%ifndef INTEL_SHA1_UPDATE_FUNCNAME
%define INTEL_SHA1_UPDATE_FUNCNAME sha1_update_intel
%endif
global INTEL_SHA1_UPDATE_FUNCNAME
%ifndef INTEL_SHA1_SINGLEBLOCK
%assign multiblock 1
%else
%assign multiblock 0
%endif
bits 64
default rel
%ifdef WIN_ABI
%xdefine arg1 rcx
%xdefine arg2 rdx
%xdefine arg3 r8
%else
%xdefine arg1 rdi
%xdefine arg2 rsi
%xdefine arg3 rdx
%endif
%xdefine ctx arg1
%xdefine buf arg2
%xdefine cnt arg3
%macro REGALLOC 0
%xdefine A ecx
%xdefine B esi
%xdefine C edi
%xdefine D ebp
%xdefine E edx
%xdefine T1 eax
%xdefine T2 ebx
%endmacro
%xdefine K_BASE r8
%xdefine HASH_PTR r9
%xdefine BUFFER_PTR r10
%xdefine BUFFER_END r11
%xdefine W_TMP xmm0
%xdefine W_TMP2 xmm9
%xdefine W0 xmm1
%xdefine W4 xmm2
%xdefine W8 xmm3
%xdefine W12 xmm4
%xdefine W16 xmm5
%xdefine W20 xmm6
%xdefine W24 xmm7
%xdefine W28 xmm8
%xdefine XMM_SHUFB_BSWAP xmm10
;; we keep window of 64 w[i]+K pre-calculated values in a circular buffer
%xdefine WK(t) (rsp + (t & 15)*4)
;------------------------------------------------------------------------------
;
; macro implements SHA-1 function's body for single or several 64-byte blocks
; first param: function's name
; second param: =0 - function implements single 64-byte block hash
; =1 - function implements multiple64-byte blocks hash
; 3rd function's argument is a number, greater 0, of 64-byte blocks to calc hash for
;
%macro SHA1_VECTOR_ASM 2
align 4096
%1:
push rbx
push rbp
%ifdef WIN_ABI
push rdi
push rsi
%xdefine stack_size (16*4 + 16*5 + 8)
%else
%xdefine stack_size (16*4 + 8)
%endif
sub rsp, stack_size
%ifdef WIN_ABI
%xdefine xmm_save_base (rsp + 16*4)
xmm_mov [xmm_save_base + 0*16], xmm6
xmm_mov [xmm_save_base + 1*16], xmm7
xmm_mov [xmm_save_base + 2*16], xmm8
xmm_mov [xmm_save_base + 3*16], xmm9
xmm_mov [xmm_save_base + 4*16], xmm10
%endif
mov HASH_PTR, ctx
mov BUFFER_PTR, buf
%if (%2 == 1)
shl cnt, 6 ;; mul by 64
add cnt, buf
mov BUFFER_END, cnt
%endif
lea K_BASE, [K_XMM_AR]
xmm_mov XMM_SHUFB_BSWAP, [bswap_shufb_ctl]
SHA1_PIPELINED_MAIN_BODY %2
%ifdef WIN_ABI
xmm_mov xmm6, [xmm_save_base + 0*16]
xmm_mov xmm7, [xmm_save_base + 1*16]
xmm_mov xmm8, [xmm_save_base + 2*16]
xmm_mov xmm9, [xmm_save_base + 3*16]
xmm_mov xmm10,[xmm_save_base + 4*16]
%endif
add rsp, stack_size
%ifdef WIN_ABI
pop rsi
pop rdi
%endif
pop rbp
pop rbx
ret
%endmacro
;--------------------------------------------
; macro implements 80 rounds of SHA-1, for one 64-byte block or multiple blocks with s/w pipelining
; macro param: =0 - process single 64-byte block
; =1 - multiple blocks
;
%macro SHA1_PIPELINED_MAIN_BODY 1
REGALLOC
mov A, [HASH_PTR ]
mov B, [HASH_PTR+ 4]
mov C, [HASH_PTR+ 8]
mov D, [HASH_PTR+12]
mov E, [HASH_PTR+16]
%assign i 0
%rep W_PRECALC_AHEAD
W_PRECALC i
%assign i i+1
%endrep
%xdefine F F1
%if (%1 == 1) ;; code loops through more than one block
%%_loop:
cmp BUFFER_PTR, K_BASE ;; we use K_BASE value as a signal of a last block,
jne %%_begin ;; it is set below by: cmovae BUFFER_PTR, K_BASE
jmp %%_end
align 32
%%_begin:
%endif
RR A,B,C,D,E,0
RR D,E,A,B,C,2
RR B,C,D,E,A,4
RR E,A,B,C,D,6
RR C,D,E,A,B,8
RR A,B,C,D,E,10
RR D,E,A,B,C,12
RR B,C,D,E,A,14
RR E,A,B,C,D,16
RR C,D,E,A,B,18
%xdefine F F2
RR A,B,C,D,E,20
RR D,E,A,B,C,22
RR B,C,D,E,A,24
RR E,A,B,C,D,26
RR C,D,E,A,B,28
RR A,B,C,D,E,30
RR D,E,A,B,C,32
RR B,C,D,E,A,34
RR E,A,B,C,D,36
RR C,D,E,A,B,38
%xdefine F F3
RR A,B,C,D,E,40
RR D,E,A,B,C,42
RR B,C,D,E,A,44
RR E,A,B,C,D,46
RR C,D,E,A,B,48
RR A,B,C,D,E,50
RR D,E,A,B,C,52
RR B,C,D,E,A,54
RR E,A,B,C,D,56
RR C,D,E,A,B,58
%xdefine F F4
%if (%1 == 1) ;; if code loops through more than one block
add BUFFER_PTR, 64 ;; move to next 64-byte block
cmp BUFFER_PTR, BUFFER_END ;; check if current block is the last one
cmovae BUFFER_PTR, K_BASE ;; smart way to signal the last iteration
%else
%xdefine W_NO_TAIL_PRECALC 1 ;; no software pipelining for single block interface
%endif
RR A,B,C,D,E,60
RR D,E,A,B,C,62
RR B,C,D,E,A,64
RR E,A,B,C,D,66
RR C,D,E,A,B,68
RR A,B,C,D,E,70
RR D,E,A,B,C,72
RR B,C,D,E,A,74
RR E,A,B,C,D,76
RR C,D,E,A,B,78
UPDATE_HASH [HASH_PTR ],A
UPDATE_HASH [HASH_PTR+ 4],B
UPDATE_HASH [HASH_PTR+ 8],C
UPDATE_HASH [HASH_PTR+12],D
UPDATE_HASH [HASH_PTR+16],E
%if (%1 == 1)
jmp %%_loop
align 32
%%_end:
%endif
%xdefine W_NO_TAIL_PRECALC 0
%xdefine F %error
%endmacro
%macro F1 3
mov T1,%2
xor T1,%3
and T1,%1
xor T1,%3
%endmacro
%macro F2 3
mov T1,%3
xor T1,%2
xor T1,%1
%endmacro
%macro F3 3
mov T1,%2
mov T2,%1
or T1,%1
and T2,%2
and T1,%3
or T1,T2
%endmacro
%define F4 F2
%macro UPDATE_HASH 2
add %2, %1
mov %1, %2
%endmacro
%macro W_PRECALC 1
%xdefine i (%1)
%if (i < 20)
%xdefine K_XMM 0
%elif (i < 40)
%xdefine K_XMM 16
%elif (i < 60)
%xdefine K_XMM 32
%else
%xdefine K_XMM 48
%endif
%if (i<16 || (i>=80 && i<(80 + W_PRECALC_AHEAD)))
%if (W_NO_TAIL_PRECALC == 0)
%xdefine i ((%1) % 80) ;; pre-compute for the next iteration
%if (i == 0)
W_PRECALC_RESET
%endif
W_PRECALC_00_15
%endif
%elif (i < 32)
W_PRECALC_16_31
%elif (i < 80) ;; rounds 32-79
W_PRECALC_32_79
%endif
%endmacro
%macro W_PRECALC_RESET 0
%xdefine W W0
%xdefine W_minus_04 W4
%xdefine W_minus_08 W8
%xdefine W_minus_12 W12
%xdefine W_minus_16 W16
%xdefine W_minus_20 W20
%xdefine W_minus_24 W24
%xdefine W_minus_28 W28
%xdefine W_minus_32 W
%endmacro
%macro W_PRECALC_ROTATE 0
%xdefine W_minus_32 W_minus_28
%xdefine W_minus_28 W_minus_24
%xdefine W_minus_24 W_minus_20
%xdefine W_minus_20 W_minus_16
%xdefine W_minus_16 W_minus_12
%xdefine W_minus_12 W_minus_08
%xdefine W_minus_08 W_minus_04
%xdefine W_minus_04 W
%xdefine W W_minus_32
%endmacro
%xdefine W_PRECALC_AHEAD 16
%xdefine W_NO_TAIL_PRECALC 0
%xdefine xmm_mov movdqa
%macro W_PRECALC_00_15 0
;; message scheduling pre-compute for rounds 0-15
%if ((i & 3) == 0) ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
movdqu W_TMP, [BUFFER_PTR + (i * 4)]
%elif ((i & 3) == 1)
pshufb W_TMP, XMM_SHUFB_BSWAP
movdqa W, W_TMP
%elif ((i & 3) == 2)
paddd W_TMP, [K_BASE]
%elif ((i & 3) == 3)
movdqa [WK(i&~3)], W_TMP
W_PRECALC_ROTATE
%endif
%endmacro
%macro W_PRECALC_16_31 0
;; message scheduling pre-compute for rounds 16-31
;; calculating last 32 w[i] values in 8 XMM registers
;; pre-calculate K+w[i] values and store to mem, for later load by ALU add instruction
;;
;; "brute force" vectorization for rounds 16-31 only due to w[i]->w[i-3] dependency
;;
%if ((i & 3) == 0) ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
movdqa W, W_minus_12
palignr W, W_minus_16, 8 ;; w[i-14]
movdqa W_TMP, W_minus_04
psrldq W_TMP, 4 ;; w[i-3]
pxor W, W_minus_08
%elif ((i & 3) == 1)
pxor W_TMP, W_minus_16
pxor W, W_TMP
movdqa W_TMP2, W
movdqa W_TMP, W
pslldq W_TMP2, 12
%elif ((i & 3) == 2)
psrld W, 31
pslld W_TMP, 1
por W_TMP, W
movdqa W, W_TMP2
psrld W_TMP2, 30
pslld W, 2
%elif ((i & 3) == 3)
pxor W_TMP, W
pxor W_TMP, W_TMP2
movdqa W, W_TMP
paddd W_TMP, [K_BASE + K_XMM]
movdqa [WK(i&~3)],W_TMP
W_PRECALC_ROTATE
%endif
%endmacro
%macro W_PRECALC_32_79 0
;; in SHA-1 specification: w[i] = (w[i-3] ^ w[i-8] ^ w[i-14] ^ w[i-16]) rol 1
;; instead we do equal: w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2
;; allows more efficient vectorization since w[i]=>w[i-3] dependency is broken
;;
%if ((i & 3) == 0) ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
movdqa W_TMP, W_minus_04
pxor W, W_minus_28 ;; W is W_minus_32 before xor
palignr W_TMP, W_minus_08, 8
%elif ((i & 3) == 1)
pxor W, W_minus_16
pxor W, W_TMP
movdqa W_TMP, W
%elif ((i & 3) == 2)
psrld W, 30
pslld W_TMP, 2
por W_TMP, W
%elif ((i & 3) == 3)
movdqa W, W_TMP
paddd W_TMP, [K_BASE + K_XMM]
movdqa [WK(i&~3)],W_TMP
W_PRECALC_ROTATE
%endif
%endmacro
%macro RR 6 ;; RR does two rounds of SHA-1 back to back with W pre-calculation
;; TEMP = A
;; A = F( i, B, C, D ) + E + ROTATE_LEFT( A, 5 ) + W[i] + K(i)
;; C = ROTATE_LEFT( B, 30 )
;; D = C
;; E = D
;; B = TEMP
W_PRECALC (%6 + W_PRECALC_AHEAD)
F %2, %3, %4 ;; F returns result in T1
add %5, [WK(%6)]
rol %2, 30
mov T2, %1
add %4, [WK(%6 + 1)]
rol T2, 5
add %5, T1
W_PRECALC (%6 + W_PRECALC_AHEAD + 1)
add T2, %5
mov %5, T2
rol T2, 5
add %4, T2
F %1, %2, %3 ;; F returns result in T1
add %4, T1
rol %1, 30
;; write: %1, %2
;; rotate: %1<=%4, %2<=%5, %3<=%1, %4<=%2, %5<=%3
%endmacro
;;----------------------
section .data align=128
%xdefine K1 0x5a827999
%xdefine K2 0x6ed9eba1
%xdefine K3 0x8f1bbcdc
%xdefine K4 0xca62c1d6
align 128
K_XMM_AR:
DD K1, K1, K1, K1
DD K2, K2, K2, K2
DD K3, K3, K3, K3
DD K4, K4, K4, K4
align 16
bswap_shufb_ctl:
DD 00010203h
DD 04050607h
DD 08090a0bh
DD 0c0d0e0fh
;; dispatch pointer, points to the init routine for the first invocation
sha1_update_intel_dispatched:
DQ sha1_update_intel_init_
;;----------------------
section .text align=4096
SHA1_VECTOR_ASM sha1_update_intel_ssse3_, multiblock
align 32
sha1_update_intel_init_: ;; we get here with the first time invocation
call sha1_update_intel_dispacth_init_
INTEL_SHA1_UPDATE_FUNCNAME: ;; we get here after init
jmp qword [sha1_update_intel_dispatched]
;; CPUID feature flag based dispatch
sha1_update_intel_dispacth_init_:
push rax
push rbx
push rcx
push rdx
push rsi
lea rsi, [INTEL_SHA1_UPDATE_DEFAULT_DISPATCH]
mov eax, 1
cpuid
test ecx, 0200h ;; SSSE3 support, CPUID.1.ECX[bit 9]
jz _done
lea rsi, [sha1_update_intel_ssse3_]
_done:
mov [sha1_update_intel_dispatched], rsi
pop rsi
pop rdx
pop rcx
pop rbx
pop rax
ret
;;----------------------
;; in the case a default SHA-1 update function implementation was not provided
;; and code was invoked on a non-SSSE3 supporting CPU, dispatch handles this
;; failure in a safest way - jumps to the stub function with UD2 instruction below
sha1_intel_non_ssse3_cpu_stub_:
ud2 ;; in the case no default SHA-1 was provided non-SSSE3 CPUs safely fail here
ret
; END
;----------------------

154
pkg/crypto/sha1/sha1_test.go Normal file
View file

@ -0,0 +1,154 @@
/*
* Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file of
// Golang project:
// https://github.com/golang/go/blob/master/LICENSE
// Using this part of Minio codebase under the license
// Apache License Version 2.0 with modifications
// SHA1 hash algorithm. See RFC 3174.
package sha1
import (
"crypto/rand"
"fmt"
"io"
"testing"
)
type sha1Test struct {
out string
in string
}
var golden = []sha1Test{
{"da39a3ee5e6b4b0d3255bfef95601890afd80709", ""},
{"86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", "a"},
{"da23614e02469a0d7c7bd1bdab5c9c474b1904dc", "ab"},
{"a9993e364706816aba3e25717850c26c9cd0d89d", "abc"},
{"81fe8bfe87576c3ecb22426f8e57847382917acf", "abcd"},
{"03de6c570bfe24bfc328ccd7ca46b76eadaf4334", "abcde"},
{"1f8ac10f23c5b5bc1167bda84b833e5c057a77d2", "abcdef"},
{"2fb5e13419fc89246865e7a324f476ec624e8740", "abcdefg"},
{"425af12a0743502b322e93a015bcf868e324d56a", "abcdefgh"},
{"c63b19f1e4c8b5f76b25c49b8b87f57d8e4872a1", "abcdefghi"},
{"d68c19a0a345b7eab78d5e11e991c026ec60db63", "abcdefghij"},
{"ebf81ddcbe5bf13aaabdc4d65354fdf2044f38a7", "Discard medicine more than two years old."},
{"e5dea09392dd886ca63531aaa00571dc07554bb6", "He who has a shady past knows that nice guys finish last."},
{"45988f7234467b94e3e9494434c96ee3609d8f8f", "I wouldn't marry him with a ten foot pole."},
{"55dee037eb7460d5a692d1ce11330b260e40c988", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"},
{"b7bc5fb91080c7de6b582ea281f8a396d7c0aee8", "The days of the digital watch are numbered. -Tom Stoppard"},
{"c3aed9358f7c77f523afe86135f06b95b3999797", "Nepal premier won't resign."},
{"6e29d302bf6e3a5e4305ff318d983197d6906bb9", "For every action there is an equal and opposite government program."},
{"597f6a540010f94c15d71806a99a2c8710e747bd", "His money is twice tainted: 'taint yours and 'taint mine."},
{"6859733b2590a8a091cecf50086febc5ceef1e80", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"},
{"514b2630ec089b8aee18795fc0cf1f4860cdacad", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"},
{"c5ca0d4a7b6676fc7aa72caa41cc3d5df567ed69", "size: a.out: bad magic"},
{"74c51fa9a04eadc8c1bbeaa7fc442f834b90a00a", "The major problem is with sendmail. -Mark Horton"},
{"0b4c4ce5f52c3ad2821852a8dc00217fa18b8b66", "Give me a rock, paper and scissors and I will move the world. CCFestoon"},
{"3ae7937dd790315beb0f48330e8642237c61550a", "If the enemy is within range, then so are you."},
{"410a2b296df92b9a47412b13281df8f830a9f44b", "It's well we cannot hear the screams/That we create in others' dreams."},
{"841e7c85ca1adcddbdd0187f1289acb5c642f7f5", "You remind me of a TV show, but that's all right: I watch it anyway."},
{"163173b825d03b952601376b25212df66763e1db", "C is as portable as Stonehedge!!"},
{"32b0377f2687eb88e22106f133c586ab314d5279", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"},
{"0885aaf99b569542fd165fa44e322718f4a984e0", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"},
{"6627d6904d71420b0bf3886ab629623538689f45", "How can you write a big system without C++? -Paul Glick"},
}
func TestGolden(t *testing.T) {
for i := 0; i < len(golden); i++ {
g := golden[i]
s := fmt.Sprintf("%x", Sum([]byte(g.in)))
if s != g.out {
t.Fatalf("Sum function: sha1(%s) = %s want %s", g.in, s, g.out)
}
c := New()
for j := 0; j < 3; j++ {
if j < 2 {
io.WriteString(c, g.in)
} else {
io.WriteString(c, g.in[0:len(g.in)/2])
c.Sum(nil)
io.WriteString(c, g.in[len(g.in)/2:])
}
s := fmt.Sprintf("%x", c.Sum(nil))
if s != g.out {
t.Fatalf("sha1[%d](%s) = %s want %s", j, g.in, s, g.out)
}
c.Reset()
}
}
}
func TestSize(t *testing.T) {
c := New()
if got := c.Size(); got != Size {
t.Errorf("Size = %d; want %d", got, Size)
}
}
func TestBlockSize(t *testing.T) {
c := New()
if got := c.BlockSize(); got != BlockSize {
t.Errorf("BlockSize = %d; want %d", got, BlockSize)
}
}
// Tests that blockGeneric (pure Go) and block (in assembly for amd64, 386, arm) match.
func TestBlockGeneric(t *testing.T) {
gen, asm := New().(*digest), New().(*digest)
buf := make([]byte, BlockSize*20) // arbitrary factor
rand.Read(buf)
blockGeneric(gen, buf)
block(asm, buf)
if *gen != *asm {
t.Error("block and blockGeneric resulted in different states")
}
}
var bench = New()
var buf = make([]byte, 1024*1024)
func benchmarkSize(b *testing.B, size int) {
b.SetBytes(int64(size))
sum := make([]byte, bench.Size())
for i := 0; i < b.N; i++ {
bench.Reset()
bench.Write(buf[:size])
bench.Sum(sum[:0])
}
}
func BenchmarkHash8Bytes(b *testing.B) {
benchmarkSize(b, 8)
}
func BenchmarkHash1K(b *testing.B) {
benchmarkSize(b, 1024)
}
func BenchmarkHash8K(b *testing.B) {
benchmarkSize(b, 8192)
}
func BenchmarkHash1M(b *testing.B) {
benchmarkSize(b, 1024*1024)
}

21
pkg/crypto/sha1/sha1_yasm_darwin.go Normal file
View file

@ -0,0 +1,21 @@
// +build darwin,amd64
/*
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sha1
//go:generate yasm -f macho64 -DINTEL_SHA1_UPDATE_FUNCNAME=_sha1_update_intel sha1_sse3_amd64.asm -o sha1_sse3_amd64.syso

21
pkg/crypto/sha1/sha1_yasm_linux.go Normal file
View file

@ -0,0 +1,21 @@
// +build linux,amd64
/*
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sha1
//go:generate yasm -f elf64 sha1_sse3_amd64.asm -o sha1_sse3_amd64.syso

21
pkg/crypto/sha1/sha1_yasm_windows.go Normal file
View file

@ -0,0 +1,21 @@
// +build windows,amd64
/*
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sha1
//go:generate yasm -f win64 -DWIN_ABI=1 sha1_sse3_amd64.asm -o sha1_sse3_amd64.syso

43
pkg/crypto/sha1/sha1block.go Normal file
View file

@ -0,0 +1,43 @@
// +build amd64,cgo
// +build darwin windows
/*
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sha1
// #include <stdint.h>
// #include <stdlib.h>
// void sha1_update_intel(int32_t *hash, const char* input, size_t num_blocks);
import "C"
import (
"unsafe"
"github.com/minio/minio/pkg/cpu"
)
func block(dig *digest, p []byte) {
switch true {
case cpu.HasSSE41() == true:
blockSSE3(dig, p)
default:
blockGeneric(dig, p)
}
}
func blockSSE3(dig *digest, p []byte) {
C.sha1_update_intel((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk))
}

110
pkg/crypto/sha1/sha1block_generic.go Normal file
View file

@ -0,0 +1,110 @@
/*
* Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file of
// Golang project:
// https://github.com/golang/go/blob/master/LICENSE
// Using this part of Minio codebase under the license
// Apache License Version 2.0 with modifications
package sha1
const (
_K0 = 0x5A827999
_K1 = 0x6ED9EBA1
_K2 = 0x8F1BBCDC
_K3 = 0xCA62C1D6
)
// blockGeneric is a portable, pure Go version of the SHA1 block step.
// It's used by sha1block_generic.go and tests.
func blockGeneric(dig *digest, p []byte) {
var w [16]uint32
h0, h1, h2, h3, h4 := dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4]
for len(p) >= chunk {
// Can interlace the computation of w with the
// rounds below if needed for speed.
for i := 0; i < 16; i++ {
j := i * 4
w[i] = uint32(p[j])<<24 | uint32(p[j+1])<<16 | uint32(p[j+2])<<8 | uint32(p[j+3])
}
a, b, c, d, e := h0, h1, h2, h3, h4
// Each of the four 20-iteration rounds
// differs only in the computation of f and
// the choice of K (_K0, _K1, etc).
i := 0
for ; i < 16; i++ {
f := b&c | (^b)&d
a5 := a<<5 | a>>(32-5)
b30 := b<<30 | b>>(32-30)
t := a5 + f + e + w[i&0xf] + _K0
a, b, c, d, e = t, a, b30, c, d
}
for ; i < 20; i++ {
tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
w[i&0xf] = tmp<<1 | tmp>>(32-1)
f := b&c | (^b)&d
a5 := a<<5 | a>>(32-5)
b30 := b<<30 | b>>(32-30)
t := a5 + f + e + w[i&0xf] + _K0
a, b, c, d, e = t, a, b30, c, d
}
for ; i < 40; i++ {
tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
w[i&0xf] = tmp<<1 | tmp>>(32-1)
f := b ^ c ^ d
a5 := a<<5 | a>>(32-5)
b30 := b<<30 | b>>(32-30)
t := a5 + f + e + w[i&0xf] + _K1
a, b, c, d, e = t, a, b30, c, d
}
for ; i < 60; i++ {
tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
w[i&0xf] = tmp<<1 | tmp>>(32-1)
f := ((b | c) & d) | (b & c)
a5 := a<<5 | a>>(32-5)
b30 := b<<30 | b>>(32-30)
t := a5 + f + e + w[i&0xf] + _K2
a, b, c, d, e = t, a, b30, c, d
}
for ; i < 80; i++ {
tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
w[i&0xf] = tmp<<1 | tmp>>(32-1)
f := b ^ c ^ d
a5 := a<<5 | a>>(32-5)
b30 := b<<30 | b>>(32-30)
t := a5 + f + e + w[i&0xf] + _K3
a, b, c, d, e = t, a, b30, c, d
}
h0 += a
h1 += b
h2 += c
h3 += d
h4 += e
p = p[chunk:]
}
dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4] = h0, h1, h2, h3, h4
}

50
pkg/crypto/sha1/sha1block_linux.go Normal file
View file

@ -0,0 +1,50 @@
// +build linux,amd64,cgo
/*
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sha1
// #cgo CFLAGS: -DHAS_AVX2
// #include <stdint.h>
// #include <stdlib.h>
// void sha1_transform(int32_t *hash, const char* input, size_t num_blocks);
// void sha1_update_intel(int32_t *hash, const char* input, size_t num_blocks);
import "C"
import (
"unsafe"
"github.com/minio/minio/pkg/cpu"
)
func block(dig *digest, p []byte) {
switch true {
case cpu.HasAVX2():
blockAVX2(dig, p)
case cpu.HasSSE41():
blockSSE3(dig, p)
default:
blockGeneric(dig, p)
}
}
func blockAVX2(dig *digest, p []byte) {
C.sha1_transform((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk))
}
func blockSSE3(dig *digest, p []byte) {
C.sha1_update_intel((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk))
}

23
pkg/crypto/sha1/sha1block_nocgo.go Normal file
View file

@ -0,0 +1,23 @@
// +build !cgo arm
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sha1
func block(dig *digest, p []byte) {
blockGeneric(dig, p)
}

0
vendor/github.com/minio/minio-xl/pkg/crypto/sha512/LICENSE → pkg/crypto/sha256/LICENSE
View file

0
vendor/github.com/minio/minio-xl/pkg/crypto/sha256/sha256-avx-asm_linux_amd64.S → pkg/crypto/sha256/sha256-avx-asm_linux_amd64.S
View file

0
vendor/github.com/minio/minio-xl/pkg/crypto/sha256/sha256-avx2-asm_linux_amd64.S → pkg/crypto/sha256/sha256-avx2-asm_linux_amd64.S
View file

0
vendor/github.com/minio/minio-xl/pkg/crypto/sha256/sha256-ssse3-asm_linux_amd64.S → pkg/crypto/sha256/sha256-ssse3-asm_linux_amd64.S
View file

36
vendor/github.com/minio/minio-xl/pkg/crypto/sha256/sha256_linux.go → pkg/crypto/sha256/sha256.go
View file

@ -1,7 +1,7 @@
// +build 386 arm amd64,!cgo
// +build darwin windows 386 arm !cgo
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,36 +20,22 @@ package sha256
import (
"hash"
"io"
"crypto/sha256"
)
// Sum256 - single caller sha256 helper
func Sum256(data []byte) []byte {
d := sha256.New()
d.Write(data)
return d.Sum(nil)
}
// Size - The size of a SHA256 checksum in bytes.
const Size = 32
// Sum - io.Reader based streaming sha256 helper
func Sum(reader io.Reader) ([]byte, error) {
d := sha256.New()
var err error
for err == nil {
length := 0
byteBuffer := make([]byte, 1024*1024)
length, err = reader.Read(byteBuffer)
byteBuffer = byteBuffer[0:length]
d.Write(byteBuffer)
}
if err != io.EOF {
return nil, err
}
return d.Sum(nil), nil
}
// BlockSize - The blocksize of SHA256 in bytes.
const BlockSize = 64
// New returns a new hash.Hash computing SHA256.
func New() hash.Hash {
return sha256.New()
}
// Sum256 - single caller sha256 helper
func Sum256(data []byte) [Size]byte {
return sha256.Sum256(data)
}

56
vendor/github.com/minio/minio-xl/pkg/crypto/sha256/sha256_linux_amd64.go → pkg/crypto/sha256/sha256_linux.go
View file

@ -1,4 +1,20 @@
// +build amd64,cgo
// +build linux,amd64,cgo
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
@ -14,9 +30,8 @@ package sha256
import (
"hash"
"io"
"github.com/minio/minio-xl/pkg/cpu"
"github.com/minio/minio/pkg/cpu"
)
// Size - The size of a SHA256 checksum in bytes.
@ -79,6 +94,14 @@ func New() hash.Hash {
return d
}
// Sum256 - single caller sha256 helper
func Sum256(data []byte) [Size]byte {
var d digest
d.Reset()
d.Write(data)
return d.checkSum()
}
// Return size of checksum
func (d *digest) Size() int { return Size }
@ -152,30 +175,3 @@ func (d *digest) checkSum() [Size]byte {
return digest
}
/// Convenience functions
// Sum256 - single caller sha256 helper
func Sum256(data []byte) []byte {
var d digest
d.Reset()
d.Write(data)
return d.Sum(nil)
}
// Sum - io.Reader based streaming sha256 helper
func Sum(reader io.Reader) ([]byte, error) {
h := New()
var err error
for err == nil {
length := 0
byteBuffer := make([]byte, 1024*1024)
length, err = reader.Read(byteBuffer)
byteBuffer = byteBuffer[0:length]
h.Write(byteBuffer)
}
if err != io.EOF {
return nil, err
}
return h.Sum(nil), nil
}

16
vendor/github.com/minio/minio-xl/pkg/crypto/sha256/sha256_linux_amd64_test.go → pkg/crypto/sha256/sha256_test.go
View file

@ -1,4 +1,18 @@
// +build amd64,cgo
/*
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style

2
vendor/github.com/minio/minio-xl/pkg/crypto/sha256/sha256block_linux_amd64.go → pkg/crypto/sha256/sha256block.go
View file

@ -1,4 +1,4 @@
// +build amd64,cgo
// +build linux,amd64,cgo
//
// Minio Cloud Storage, (C) 2015 Minio, Inc.

0
vendor/github.com/minio/minio-xl/pkg/minhttp/LICENSE.Minio → pkg/crypto/sha512/LICENSE
View file

0
vendor/github.com/minio/minio-xl/pkg/crypto/sha512/sha512-avx-asm_linux_amd64.S → pkg/crypto/sha512/sha512-avx-asm_linux_amd64.S
View file

0
vendor/github.com/minio/minio-xl/pkg/crypto/sha512/sha512-avx2-asm_linux_amd64.S → pkg/crypto/sha512/sha512-avx2-asm_linux_amd64.S
View file

0
vendor/github.com/minio/minio-xl/pkg/crypto/sha512/sha512-ssse3-asm_linux_amd64.S → pkg/crypto/sha512/sha512-ssse3-asm_linux_amd64.S
View file

41
vendor/github.com/minio/minio-xl/pkg/crypto/sha512/sha512_linux.go → pkg/crypto/sha512/sha512.go
View file

@ -1,7 +1,7 @@
// +build 386 arm amd64,!cgo
// +build darwin windows 386 arm !cgo
/*
* Minio Cloud Storage, (C) 2014 Minio, Inc.
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,41 +20,22 @@ package sha512
import (
"hash"
"io"
"crypto/sha512"
)
// The size of a SHA512 checksum in bytes.
const (
Size = sha512.Size
)
// Size - The size of a SHA512 checksum in bytes.
const Size = 64
// Sum512 - single caller sha512 helper
func Sum512(data []byte) []byte {
d := sha512.New()
d.Write(data)
return d.Sum(nil)
}
// Sum - io.Reader based streaming sha512 helper
func Sum(reader io.Reader) ([]byte, error) {
d := sha512.New()
var err error
for err == nil {
length := 0
byteBuffer := make([]byte, 1024*1024)
length, err = reader.Read(byteBuffer)
byteBuffer = byteBuffer[0:length]
d.Write(byteBuffer)
}
if err != io.EOF {
return nil, err
}
return d.Sum(nil), nil
}
// BlockSize - The blocksize of SHA512 in bytes.
const BlockSize = 128
// New returns a new hash.Hash computing SHA512.
func New() hash.Hash {
return sha512.New()
}
// Sum512 - single caller sha512 helper
func Sum512(data []byte) [Size]byte {
return sha512.Sum512(data)
}

40
vendor/github.com/minio/minio-xl/pkg/crypto/sha512/sha512_linux_amd64.go → pkg/crypto/sha512/sha512_linux.go
View file

@ -1,4 +1,4 @@
// +build amd64,cgo
// +build linux,amd64,cgo
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
@ -15,9 +15,8 @@ package sha512
import (
"hash"
"io"
"github.com/minio/minio-xl/pkg/cpu"
"github.com/minio/minio/pkg/cpu"
)
// Size - The size of a SHA512 checksum in bytes.
@ -80,6 +79,14 @@ func New() hash.Hash {
return d
}
// Sum512 - single caller sha512 helper
func Sum512(data []byte) [Size]byte {
var d digest
d.Reset()
d.Write(data)
return d.checkSum()
}
// Return output array byte size
func (d *digest) Size() int { return Size }
@ -157,30 +164,3 @@ func (d *digest) checkSum() [Size]byte {
return digest
}
/// Convenience functions
// Sum512 - single caller sha512 helper
func Sum512(data []byte) []byte {
var d digest
d.Reset()
d.Write(data)
return d.Sum(nil)
}
// Sum - io.Reader based streaming sha512 helper
func Sum(reader io.Reader) ([]byte, error) {
h := New()
var err error
for err == nil {
length := 0
byteBuffer := make([]byte, 1024*1024)
length, err = reader.Read(byteBuffer)
byteBuffer = byteBuffer[0:length]
h.Write(byteBuffer)
}
if err != io.EOF {
return nil, err
}
return h.Sum(nil), nil
}

16
vendor/github.com/minio/minio-xl/pkg/crypto/sha512/sha512_linux_amd64_test.go → pkg/crypto/sha512/sha512_test.go
View file

@ -1,4 +1,18 @@
// +build amd64,cgo
/*
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style

33
vendor/github.com/minio/minio-xl/pkg/crypto/sha512/sha512block_linux_amd64.go → pkg/crypto/sha512/sha512block.go
View file

@ -1,20 +1,21 @@
// +build amd64,cgo
// +build linux,amd64,cgo
/*
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
//
// Minio Cloud Storage, (C) 2015 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Software block transform are provided by The Go Authors:
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style

1
pkg/erasure/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
*.syso

64
pkg/erasure/INSTALLGO.md Normal file
View file

@ -0,0 +1,64 @@
## Ubuntu (Kylin) 14.04
### Build Dependencies
This installation document assumes Ubuntu 14.04+ on x86-64 platform.
##### Install Git, GCC, yasm
```sh
$ sudo apt-get install git build-essential yasm
```
##### Install Go 1.5.1+
Download Go 1.5.1+ from [https://golang.org/dl/](https://golang.org/dl/).
```sh
$ wget https://storage.googleapis.com/golang/go1.5.1.linux-amd64.tar.gz
$ mkdir -p ${HOME}/bin/
$ mkdir -p ${HOME}/go/
$ tar -C ${HOME}/bin/ -xzf go1.5.1.linux-amd64.tar.gz
```
##### Setup GOROOT and GOPATH
Add the following exports to your ``~/.bashrc``. Environment variable GOROOT specifies the location of your golang binaries
and GOPATH specifies the location of your project workspace.
```sh
$ export GOROOT=${HOME}/bin/go
$ export GOPATH=${HOME}/go
$ export PATH=$PATH:${HOME}/bin/go/bin:${GOPATH}/bin
```
## OS X (Yosemite) 10.10
### Build Dependencies
This installation document assumes OS X Yosemite 10.10+ on x86-64 platform.
##### Install brew
```sh
$ ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
```
##### Install Git, Python
```sh
$ brew install git python yasm
```
##### Install Go 1.5.1+
Install golang binaries using `brew`
```sh
$ brew install go
$ mkdir -p $HOME/go
```
##### Setup GOROOT and GOPATH
Add the following exports to your ``~/.bashrc``. Environment variable GOROOT specifies the location of your golang binaries
and GOPATH specifies the location of your project workspace.
```sh
$ export GOPATH=${HOME}/go
$ export GOVERSION=$(brew list go | head -n 1 | cut -d '/' -f 6)
$ export GOROOT=$(brew --prefix)/Cellar/go/${GOVERSION}/libexec
$ export PATH=$PATH:${GOPATH}/bin
```

26
pkg/erasure/LICENSE.INTEL Normal file
View file

@ -0,0 +1,26 @@
Copyright(c) 2011-2014 Intel Corporation All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
* Neither the name of Intel Corporation nor the names of its
contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

202
pkg/erasure/LICENSE.MINIO Normal file
View file

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

25
pkg/erasure/README.md Normal file
View file

@ -0,0 +1,25 @@
## Introduction
Erasure is an open source Golang library written on top of ISAL (Intel Intelligent Storage Library) released under [Apache license v2](./LICENSE)
### Developers
* [Get Source](./CONTRIBUTING.md)
* [Build Dependencies](./BUILDDEPS.md)
* [Development Workflow](./CONTRIBUTING.md#developer-guidelines)
* [Developer discussions and bugs](https://github.com/minio/minio/issues)
### Supported platforms
| Name | Supported |
| ------------- | ------------- |
| Linux | Yes |
| Windows | Not yet |
| Mac OSX | Yes |
### Supported architectures
| Arch | Supported |
| ------------- | ------------- |
| x86-64 | Yes |
| arm64 | Not yet|
| i386 | Never |

62
pkg/erasure/ctypes.go Normal file
View file

@ -0,0 +1,62 @@
/*
* Minio Cloud Storage, (C) 2014 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package erasure
// #include <stdint.h>
import "C"
import (
"fmt"
"unsafe"
)
// intSlice2CIntArray converts Go int slice to C int array
func intSlice2CIntArray(srcErrList []int) *C.int32_t {
if len(srcErrList) == 0 {
return (*C.int32_t)(unsafe.Pointer(nil))
}
var sizeErrInt = int(unsafe.Sizeof(srcErrList[0]))
switch sizeInt {
case sizeErrInt:
return (*C.int32_t)(unsafe.Pointer(&srcErrList[0]))
case sizeInt8:
int8Array := make([]int8, len(srcErrList))
for i, v := range srcErrList {
int8Array[i] = int8(v)
}
return (*C.int32_t)(unsafe.Pointer(&int8Array[0]))
case sizeInt16:
int16Array := make([]int16, len(srcErrList))
for i, v := range srcErrList {
int16Array[i] = int16(v)
}
return (*C.int32_t)(unsafe.Pointer(&int16Array[0]))
case sizeInt32:
int32Array := make([]int32, len(srcErrList))
for i, v := range srcErrList {
int32Array[i] = int32(v)
}
return (*C.int32_t)(unsafe.Pointer(&int32Array[0]))
case sizeInt64:
int64Array := make([]int64, len(srcErrList))
for i, v := range srcErrList {
int64Array[i] = int64(v)
}
return (*C.int32_t)(unsafe.Pointer(&int64Array[0]))
default:
panic(fmt.Sprintf("Unsupported: %d", sizeInt))
}
}

66
pkg/erasure/doc.go Normal file
View file

@ -0,0 +1,66 @@
// Package erasure is a Go wrapper for the Intel Intelligent Storage
// Acceleration Library (Intel ISA-L). Intel ISA-L is a CPU optimized
// implementation of erasure coding algorithms.
//
// For more information on Intel ISA-L, please visit:
// https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version
//
// Usage:
//
// Encode encodes a block of data. The input is the original data. The output
// is a 2 tuple containing (k + m) chunks of erasure encoded data and the
// length of the original object.
//
// Decode decodes 2 tuple data containing (k + m) chunks back into its original form.
// Additionally original block length should also be provided as input.
//
// Decoded data is exactly similar in length and content as the original data.
//
// Encoding data may be performed in 3 steps.
//
// 1. Create a parse set of encoder parameters
// 2. Create a new encoder
// 3. Encode data
//
// Decoding data is also performed in 3 steps.
//
// 1. Create a parse set of encoder parameters for validation
// 2. Create a new encoder
// 3. Decode data
//
// Erasure parameters contain three configurable elements:
// ValidateParams(k, m, technique int) (ErasureParams, error)
// k - Number of rows in matrix
// m - Number of colums in matrix
// technique - Matrix type, can be either Cauchy (recommended) or Vandermonde
// constraints: k + m < Galois Field (2^8)
//
// Choosing right parity and matrix technique is left for application to decide.
//
// But here are the few points to keep in mind
//
// Matrix Type:
// - Vandermonde is most commonly used method for choosing coefficients in erasure
// encoding but does not guarantee invertable for every sub matrix.
// - Whereas Cauchy is our recommended method for choosing coefficients in erasure coding.
// Since any sub-matrix of a Cauchy matrix is invertable.
//
// Total blocks:
// - Data blocks and Parity blocks should not be greater than 'Galois Field' (2^8)
//
// Example
//
// Creating and using an encoder
// var bytes []byte
// params := erasure.ValidateParams(10, 5)
// encoder := erasure.NewErasure(params)
// encodedData, length := encoder.Encode(bytes)
//
// Creating and using a decoder
// var encodedData [][]byte
// var length int
// params := erasure.ValidateParams(10, 5)
// encoder := erasure.NewErasure(params)
// originalData, err := encoder.Decode(encodedData, length)
//
package erasure

39
pkg/erasure/ec_minio_common.h Normal file
View file

@ -0,0 +1,39 @@
/*
* Minio Cloud Storage, (C) 2014 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef __COMMON_H__
#define __COMMON_H__
#include <stdint.h>
int32_t minio_init_encoder (int k, int m,
unsigned char **encode_matrix,
unsigned char **encode_tbls);
int32_t minio_init_decoder (int32_t *error_index,
int k, int n, int errs,
unsigned char *encoding_matrix,
unsigned char **decode_matrix,
unsigned char **decode_tbls,
uint32_t **decode_index);
int32_t minio_get_source_target (int errs, int k, int m,
int32_t *error_index,
uint32_t *decode_index,
unsigned char **buffs,
unsigned char ***source,
unsigned char ***target);
#endif /* __COMMON_H__ */

142
pkg/erasure/ec_minio_decode.c Normal file
View file

@ -0,0 +1,142 @@
/*
* Minio Cloud Storage, (C) 2014 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "isa-l.h"
#include "ec_minio_common.h"
static
int32_t _minio_src_index_in_error (int r, int32_t *error_index, int errs)
{
int i;
for (i = 0; i < errs; i++) {
if (error_index[i] == r) {
// true
return 1;
}
}
// false
return 0;
}
// Separate out source data and target buffers
int32_t minio_get_source_target (int errs, int k, int m,
int32_t *error_index,
uint32_t *decode_index,
unsigned char **buffs,
unsigned char ***source,
unsigned char ***target)
{
int i;
unsigned char *tmp_source[k];
unsigned char *tmp_target[m];
if (k < 0 || m < 0) {
return -1;
}
memset (tmp_source, 0, k);
memset (tmp_target, 0, m);
for (i = 0; i < k; i++) {
tmp_source[i] = (unsigned char *) buffs[decode_index[i]];
}
for (i = 0; i < m; i++) {
if (i < errs)
tmp_target[i] = (unsigned char *) buffs[error_index[i]];
}
*source = tmp_source;
*target = tmp_target;
return 0;
}
/*
Generate decode matrix during the decoding phase
*/
int minio_init_decoder (int32_t *error_index,
int k, int n, int errs,
unsigned char *encode_matrix,
unsigned char **decode_matrix,
unsigned char **decode_tbls,
uint32_t **decode_index)
{
int i, j, r, l;
uint32_t *tmp_decode_index = (uint32_t *) malloc(sizeof(uint32_t) * k);
unsigned char *input_matrix;
unsigned char *inverse_matrix;
unsigned char *tmp_decode_matrix;
unsigned char *tmp_decode_tbls;
input_matrix = (unsigned char *) malloc(sizeof(unsigned char) * k * n);
inverse_matrix = (unsigned char *) malloc(sizeof(unsigned char) * k * n);
tmp_decode_matrix = (unsigned char *) malloc(sizeof(unsigned char) * k * n);;
tmp_decode_tbls = (unsigned char *) malloc(sizeof(unsigned char) * k * n * 32);
for (i = 0, r = 0; i < k; i++, r++) {
while (_minio_src_index_in_error(r, error_index, errs))
r++;
for (j = 0; j < k; j++) {
input_matrix[k * i + j] = encode_matrix[k * r + j];
}
tmp_decode_index[i] = r;
}
// Not all vandermonde matrix can be inverted
if (gf_invert_matrix(input_matrix, inverse_matrix, k) < 0) {
free(tmp_decode_matrix);
free(tmp_decode_tbls);
free(tmp_decode_index);
return -1;
}
for (l = 0; l < errs; l++) {
if (error_index[l] < k) {
// decoding matrix elements for data chunks
for (j = 0; j < k; j++) {
tmp_decode_matrix[k * l + j] =
inverse_matrix[k *
error_index[l] + j];
}
} else {
// decoding matrix element for coding chunks
for (i = 0; i < k; i++) {
unsigned char s = 0;
for (j = 0; j < k; j++) {
s ^= gf_mul(inverse_matrix[j * k + i],
encode_matrix[k *
error_index[l] + j]);
}
tmp_decode_matrix[k * l + i] = s;
}
}
}
ec_init_tables (k, errs, tmp_decode_matrix, tmp_decode_tbls);
*decode_matrix = tmp_decode_matrix;
*decode_tbls = tmp_decode_tbls;
*decode_index = tmp_decode_index;
return 0;
}

55
pkg/erasure/ec_minio_encode.c Normal file
View file

@ -0,0 +1,55 @@
/*
* Minio Cloud Storage, (C) 2014 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdlib.h>
#include <stdio.h>
#include "isa-l.h"
#include "ec_minio_common.h"
/*
Generate encode matrix during the encoding phase
*/
int32_t minio_init_encoder (int k, int m, unsigned char **encode_matrix, unsigned char **encode_tbls)
{
unsigned char *tmp_matrix;
unsigned char *tmp_tbls;
tmp_matrix = (unsigned char *) malloc (k * (k + m));
tmp_tbls = (unsigned char *) malloc (k * (k + m) * 32);
if (k < 5) {
/*
Commonly used method for choosing coefficients in erasure
encoding but does not guarantee invertable for every sub
matrix. For large k it is possible to find cases where the
decode matrix chosen from sources and parity not in erasure
are not invertable. Users may want to adjust for k > 5.
-- Intel
*/
gf_gen_rs_matrix (tmp_matrix, k + m, k);
} else {
gf_gen_cauchy1_matrix (tmp_matrix, k + m, k);
}
ec_init_tables(k, m, &tmp_matrix[k * k], tmp_tbls);
*encode_matrix = tmp_matrix;
*encode_tbls = tmp_tbls;
return 0;
}

123
pkg/erasure/erasure_decode.go Normal file
View file

@ -0,0 +1,123 @@
/*
* Minio Cloud Storage, (C) 2014 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package erasure
// #cgo CFLAGS: -O0
// #include <stdlib.h>
// #include "isa-l.h"
// #include "ec_minio_common.h"
import "C"
import (
"errors"
"fmt"
"unsafe"
)
// Decode decodes erasure coded blocks of data into its original
// form. Erasure coded data contains K data blocks and M parity
// blocks. Decode can withstand data loss up to any M number of blocks.
//
// "encodedDataBlocks" is an array of K data blocks and M parity
// blocks. Data blocks are position and order dependent. Missing blocks
// are set to "nil". There must be at least "K" number of data|parity
// blocks.
//
// "dataLen" is the length of original source data
func (e *Erasure) Decode(encodedDataBlocks [][]byte, dataLen int) (decodedData []byte, err error) {
e.mutex.Lock()
defer e.mutex.Unlock()
var source, target **C.uchar
k := int(e.params.K)
m := int(e.params.M)
n := k + m
// We need the data and parity blocks preserved in the same order. Missing blocks are set to nil.
if len(encodedDataBlocks) != n {
msg := fmt.Sprintf("Encoded data blocks slice must of length [%d]", n)
return nil, errors.New(msg)
}
// Length of a single encoded block
encodedBlockLen := GetEncodedBlockLen(dataLen, uint8(k))
// Keep track of errors per block.
missingEncodedBlocks := make([]int, n+1)
var missingEncodedBlocksCount int
// Check for the missing encoded blocks
for i := range encodedDataBlocks {
if encodedDataBlocks[i] == nil || len(encodedDataBlocks[i]) == 0 {
missingEncodedBlocks[missingEncodedBlocksCount] = i
missingEncodedBlocksCount++
}
}
// Cannot reconstruct original data. Need at least M number of data or parity blocks.
if missingEncodedBlocksCount > m {
return nil, fmt.Errorf("Cannot reconstruct original data. Need at least [%d] data or parity blocks", m)
}
// Convert from Go int slice to C int array
missingEncodedBlocksC := intSlice2CIntArray(missingEncodedBlocks[:missingEncodedBlocksCount])
// Allocate buffer for the missing blocks
for i := range encodedDataBlocks {
if encodedDataBlocks[i] == nil || len(encodedDataBlocks[i]) == 0 {
encodedDataBlocks[i] = make([]byte, encodedBlockLen)
}
}
// If not already initialized, recompute and cache
if e.decodeMatrix == nil || e.decodeTbls == nil || e.decodeIndex == nil {
var decodeMatrix, decodeTbls *C.uchar
var decodeIndex *C.uint32_t
C.minio_init_decoder(missingEncodedBlocksC, C.int(k), C.int(n), C.int(missingEncodedBlocksCount),
e.encodeMatrix, &decodeMatrix, &decodeTbls, &decodeIndex)
// cache this for future needs
e.decodeMatrix = decodeMatrix
e.decodeTbls = decodeTbls
e.decodeIndex = decodeIndex
}
// Make a slice of pointers to encoded blocks. Necessary to bridge to the C world.
pointers := make([]*byte, n)
for i := range encodedDataBlocks {
pointers[i] = &encodedDataBlocks[i][0]
}
// Get pointers to source "data" and target "parity" blocks from the output byte array.
ret := C.minio_get_source_target(C.int(missingEncodedBlocksCount), C.int(k), C.int(m), missingEncodedBlocksC,
e.decodeIndex, (**C.uchar)(unsafe.Pointer(&pointers[0])), &source, &target)
if int(ret) == -1 {
return nil, errors.New("Unable to decode data")
}
// Decode data
C.ec_encode_data(C.int(encodedBlockLen), C.int(k), C.int(missingEncodedBlocksCount), e.decodeTbls,
source, target)
// Allocate buffer to output buffer
decodedData = make([]byte, 0, encodedBlockLen*int(k))
for i := 0; i < int(k); i++ {
decodedData = append(decodedData, encodedDataBlocks[i]...)
}
return decodedData[:dataLen], nil
}

174
pkg/erasure/erasure_encode.go Normal file
View file

@ -0,0 +1,174 @@
/*
* Minio Cloud Storage, (C) 2014 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package erasure
// #include <stdlib.h>
// #include "isa-l.h"
// #include "ec_minio_common.h"
import "C"
import (
"errors"
"sync"
"unsafe"
)
// Block alignment
const (
SIMDAlign = 32
)
// Params is a configuration set for building an encoder. It is created using ValidateParams().
type Params struct {
K uint8
M uint8
}
// Erasure is an object used to encode and decode data.
type Erasure struct {
params *Params
encodeMatrix, encodeTbls *C.uchar
decodeMatrix, decodeTbls *C.uchar
decodeIndex *C.uint32_t
mutex *sync.Mutex
}
// ValidateParams creates an Params object.
//
// k and m represent the matrix size, which corresponds to the protection level
// technique is the matrix type. Valid inputs are Cauchy (recommended) or Vandermonde.
//
func ValidateParams(k, m uint8) (*Params, error) {
if k < 1 {
return nil, errors.New("k cannot be zero")
}
if m < 1 {
return nil, errors.New("m cannot be zero")
}
if k+m > 255 {
return nil, errors.New("(k + m) cannot be bigger than Galois field GF(2^8) - 1")
}
return &Params{
K: k,
M: m,
}, nil
}
// NewErasure creates an encoder object with a given set of parameters.
func NewErasure(ep *Params) *Erasure {
var k = C.int(ep.K)
var m = C.int(ep.M)
var encodeMatrix *C.uchar
var encodeTbls *C.uchar
C.minio_init_encoder(k, m, &encodeMatrix, &encodeTbls)
return &Erasure{
params: ep,
encodeMatrix: encodeMatrix,
encodeTbls: encodeTbls,
decodeMatrix: nil,
decodeTbls: nil,
decodeIndex: nil,
mutex: new(sync.Mutex),
}
}
// GetEncodedBlocksLen - total length of all encoded blocks
func GetEncodedBlocksLen(inputLen int, k, m uint8) (outputLen int) {
outputLen = GetEncodedBlockLen(inputLen, k) * int(k+m)
return outputLen
}
// GetEncodedBlockLen - length per block of encoded blocks
func GetEncodedBlockLen(inputLen int, k uint8) (encodedOutputLen int) {
alignment := int(k) * SIMDAlign
remainder := inputLen % alignment
paddedInputLen := inputLen
if remainder != 0 {
paddedInputLen = inputLen + (alignment - remainder)
}
encodedOutputLen = paddedInputLen / int(k)
return encodedOutputLen
}
// Encode erasure codes a block of data in "k" data blocks and "m" parity blocks.
// Output is [k+m][]blocks of data and parity slices.
func (e *Erasure) Encode(inputData []byte) (encodedBlocks [][]byte, err error) {
e.mutex.Lock()
defer e.mutex.Unlock()
k := int(e.params.K) // "k" data blocks
m := int(e.params.M) // "m" parity blocks
n := k + m // "n" total encoded blocks
// Length of a single encoded chunk.
// Total number of encoded chunks = "k" data + "m" parity blocks
encodedBlockLen := GetEncodedBlockLen(len(inputData), uint8(k))
// Length of total number of "k" data chunks
encodedDataBlocksLen := encodedBlockLen * k
// Length of extra padding required for the data blocks.
encodedDataBlocksPadLen := encodedDataBlocksLen - len(inputData)
// Extend inputData buffer to accommodate coded data blocks if necesssary
if encodedDataBlocksPadLen > 0 {
padding := make([]byte, encodedDataBlocksPadLen)
// Expand with new padded blocks to the byte array
inputData = append(inputData, padding...)
}
// Extend inputData buffer to accommodate coded parity blocks
{ // Local Scope
encodedParityBlocksLen := encodedBlockLen * m
parityBlocks := make([]byte, encodedParityBlocksLen)
inputData = append(inputData, parityBlocks...)
}
// Allocate memory to the "encoded blocks" return buffer
encodedBlocks = make([][]byte, n) // Return buffer
// Necessary to bridge Go to the C world. C requires 2D arry of pointers to
// byte array. "encodedBlocks" is a 2D slice.
pointersToEncodedBlock := make([]*byte, n) // Pointers to encoded blocks.
// Copy data block slices to encoded block buffer
for i := 0; i < k; i++ {
encodedBlocks[i] = inputData[i*encodedBlockLen : (i+1)*encodedBlockLen]
pointersToEncodedBlock[i] = &encodedBlocks[i][0]
}
// Copy erasure block slices to encoded block buffer
for i := k; i < n; i++ {
encodedBlocks[i] = make([]byte, encodedBlockLen)
pointersToEncodedBlock[i] = &encodedBlocks[i][0]
}
// Erasure code the data into K data blocks and M parity
// blocks. Only the parity blocks are filled. Data blocks remain
// intact.
C.ec_encode_data(C.int(encodedBlockLen), C.int(k), C.int(m), e.encodeTbls,
(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[:k][0])), // Pointers to data blocks
(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[k:][0]))) // Pointers to parity blocks
return encodedBlocks, nil
}

82
pkg/erasure/erasure_test.go Normal file
View file

@ -0,0 +1,82 @@
// +build !windows
/*
* Minio Cloud Storage, (C) 2014 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package erasure
import (
"bytes"
"testing"
. "gopkg.in/check.v1"
)
type MySuite struct{}
var _ = Suite(&MySuite{})
func Test(t *testing.T) { TestingT(t) }
const (
k = 10
m = 5
)
func corruptChunks(chunks [][]byte, errorIndex []int) [][]byte {
for _, err := range errorIndex {
chunks[err] = nil
}
return chunks
}
func (s *MySuite) TestEncodeDecodeFailure(c *C) {
ep, err := ValidateParams(k, m)
c.Assert(err, IsNil)
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
e := NewErasure(ep)
chunks, err := e.Encode(data)
c.Assert(err, IsNil)
errorIndex := []int{0, 3, 5, 9, 11, 13}
chunks = corruptChunks(chunks, errorIndex)
_, err = e.Decode(chunks, len(data))
c.Assert(err, Not(IsNil))
}
func (s *MySuite) TestEncodeDecodeSuccess(c *C) {
ep, err := ValidateParams(k, m)
c.Assert(err, IsNil)
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
e := NewErasure(ep)
chunks, err := e.Encode(data)
c.Assert(err, IsNil)
errorIndex := []int{0, 3, 5, 9, 13}
chunks = corruptChunks(chunks, errorIndex)
recoveredData, err := e.Decode(chunks, len(data))
c.Assert(err, IsNil)
if !bytes.Equal(data, recoveredData) {
c.Fatalf("Recovered data mismatches with original data")
}
}

38
pkg/erasure/stdint.go Normal file
View file

@ -0,0 +1,38 @@
/*
* Minio Cloud Storage, (C) 2014 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package erasure
//
// int sizeInt()
// {
// return sizeof(int);
// }
import "C"
import "unsafe"
var (
// See http://golang.org/ref/spec#Numeric_types
sizeInt = int(C.sizeInt())
// SizeInt8 is the byte size of a int8.
sizeInt8 = int(unsafe.Sizeof(int8(0)))
// SizeInt16 is the byte size of a int16.
sizeInt16 = int(unsafe.Sizeof(int16(0)))
// SizeInt32 is the byte size of a int32.
sizeInt32 = int(unsafe.Sizeof(int32(0)))
// SizeInt64 is the byte size of a int64.
sizeInt64 = int(unsafe.Sizeof(int64(0)))
)

4
pkg/fs/config.go
View file

@ -17,8 +17,8 @@
package fs
import (
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio-xl/pkg/quick"
"github.com/minio/minio/pkg/probe"
"github.com/minio/minio/pkg/quick"
)
var multipartsMetadataPath, bucketsMetadataPath string

10
pkg/fs/errors.go
View file

@ -39,16 +39,6 @@ func (e ExpiredPresignedRequest) Error() string {
return "Presigned request already expired"
}
// SignatureDoesNotMatch invalid signature
type SignatureDoesNotMatch struct {
SignatureSent string
SignatureCalculated string
}
func (e SignatureDoesNotMatch) Error() string {
return "The request signature we calculated does not match the signature you provided"
}
// InvalidArgument invalid argument
type InvalidArgument struct{}

2
pkg/fs/fs-bucket-listobjects.go
View file

@ -25,8 +25,8 @@ import (
"strings"
"time"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/ioutils"
"github.com/minio/minio/pkg/probe"
)
// listObjectsParams - list objects input parameters.

2
pkg/fs/fs-bucket.go
View file

@ -22,8 +22,8 @@ import (
"path/filepath"
"strings"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/disk"
"github.com/minio/minio/pkg/probe"
)
/// Bucket Operations

17
pkg/fs/fs-multipart.go
View file

@ -33,12 +33,13 @@ import (
"strings"
"time"
"github.com/minio/minio-xl/pkg/atomic"
"github.com/minio/minio-xl/pkg/crypto/sha256"
"github.com/minio/minio-xl/pkg/crypto/sha512"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/atomic"
"github.com/minio/minio/pkg/crypto/sha256"
"github.com/minio/minio/pkg/crypto/sha512"
"github.com/minio/minio/pkg/disk"
"github.com/minio/minio/pkg/mimedb"
"github.com/minio/minio/pkg/probe"
signV4 "github.com/minio/minio/pkg/signature"
)
// isValidUploadID - is upload id.
@ -264,7 +265,7 @@ func (a partNumber) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a partNumber) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
// CreateObjectPart - create a part in a multipart session
func (fs Filesystem) CreateObjectPart(bucket, object, uploadID, expectedMD5Sum string, partID int, size int64, data io.Reader, signature *Signature) (string, *probe.Error) {
func (fs Filesystem) CreateObjectPart(bucket, object, uploadID, expectedMD5Sum string, partID int, size int64, data io.Reader, signature *signV4.Signature) (string, *probe.Error) {
di, err := disk.GetInfo(fs.path)
if err != nil {
return "", probe.NewError(err)
@ -350,7 +351,7 @@ func (fs Filesystem) CreateObjectPart(bucket, object, uploadID, expectedMD5Sum s
}
if !ok {
partFile.CloseAndPurge()
return "", probe.NewError(SignatureDoesNotMatch{})
return "", probe.NewError(signV4.SigDoesNotMatch{})
}
}
partFile.Close()
@ -398,7 +399,7 @@ func (fs Filesystem) CreateObjectPart(bucket, object, uploadID, expectedMD5Sum s
}
// CompleteMultipartUpload - complete a multipart upload and persist the data
func (fs Filesystem) CompleteMultipartUpload(bucket, object, uploadID string, data io.Reader, signature *Signature) (ObjectMetadata, *probe.Error) {
func (fs Filesystem) CompleteMultipartUpload(bucket, object, uploadID string, data io.Reader, signature *signV4.Signature) (ObjectMetadata, *probe.Error) {
// Check bucket name is valid.
if !IsValidBucketName(bucket) {
return ObjectMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
@ -447,7 +448,7 @@ func (fs Filesystem) CompleteMultipartUpload(bucket, object, uploadID string, da
}
if !ok {
file.CloseAndPurge()
return ObjectMetadata{}, probe.NewError(SignatureDoesNotMatch{})
return ObjectMetadata{}, probe.NewError(signV4.SigDoesNotMatch{})
}
}
completeMultipartUpload := &CompleteMultipartUpload{}

11
pkg/fs/fs-object.go
View file

@ -28,12 +28,13 @@ import (
"encoding/hex"
"runtime"
"github.com/minio/minio-xl/pkg/atomic"
"github.com/minio/minio-xl/pkg/crypto/sha256"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/atomic"
"github.com/minio/minio/pkg/crypto/sha256"
"github.com/minio/minio/pkg/disk"
"github.com/minio/minio/pkg/ioutils"
"github.com/minio/minio/pkg/mimedb"
"github.com/minio/minio/pkg/probe"
signV4 "github.com/minio/minio/pkg/signature"
)
/// Object Operations
@ -198,7 +199,7 @@ func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) bool {
}
// CreateObject - create an object.
func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size int64, data io.Reader, signature *Signature) (ObjectMetadata, *probe.Error) {
func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size int64, data io.Reader, signature *signV4.Signature) (ObjectMetadata, *probe.Error) {
di, e := disk.GetInfo(fs.path)
if e != nil {
return ObjectMetadata{}, probe.NewError(e)
@ -293,7 +294,7 @@ func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size in
}
if !ok {
file.CloseAndPurge()
return ObjectMetadata{}, probe.NewError(SignatureDoesNotMatch{})
return ObjectMetadata{}, probe.NewError(signV4.SigDoesNotMatch{})
}
}
file.Close()

2
pkg/fs/fs.go
View file

@ -22,7 +22,7 @@ import (
"sync"
"time"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/probe"
)
// Filesystem - local variables

0
vendor/github.com/minio/minio-xl/pkg/minhttp/LICENSE.Facebook → pkg/minhttp/LICENSE.Facebook
View file

202
pkg/minhttp/LICENSE.Minio Normal file
View file

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

2
vendor/github.com/minio/minio-xl/pkg/minhttp/http_nix.go → pkg/minhttp/http_nix.go
View file

@ -34,7 +34,7 @@ import (
"time"
"github.com/facebookgo/httpdown"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/probe"
)
// An app contains one or more servers and their associated configuration.

2
vendor/github.com/minio/minio-xl/pkg/minhttp/http_windows.go → pkg/minhttp/http_windows.go
View file

@ -34,7 +34,7 @@ import (
"time"
"github.com/facebookgo/httpdown"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/probe"
)
// An app contains one or more servers and their associated configuration.

2
vendor/github.com/minio/minio-xl/pkg/minhttp/kill_windows.go → pkg/minhttp/kill_windows.go
View file

@ -1,4 +1,4 @@
// !build windows
// +build windows
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.

0
vendor/github.com/minio/minio-xl/pkg/minhttp/listen.go → pkg/minhttp/listen.go
View file

4
vendor/github.com/minio/minio-xl/pkg/minhttp/net.go → pkg/minhttp/net.go
View file

@ -25,7 +25,7 @@ import (
"strings"
"sync"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/probe"
)
// This package is a fork https://github.com/facebookgo/grace
@ -34,7 +34,7 @@ import (
// This package provides a family of Listen functions that either open a
// fresh connection or provide an inherited connection from when the process
// was started. This behaves like their counterparts in the net pacakge, but
// was started. This behaves like their counterparts in the net package, but
// transparently provide support for graceful restarts without dropping
// connections. This is provided in a systemd socket activation compatible form
// to allow using socket activation.

0
vendor/github.com/minio/minio-xl/pkg/probe/probe.go → pkg/probe/probe.go
View file

4
vendor/github.com/minio/minio-xl/pkg/probe/probe_test.go → pkg/probe/probe_test.go
View file

@ -19,7 +19,7 @@ import (
"os"
"testing"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/probe"
. "gopkg.in/check.v1"
)
@ -44,7 +44,7 @@ func testDummy2() *probe.Error {
func (s *MySuite) TestProbe(c *C) {
probe.Init() // Set project's root source path.
probe.SetAppInfo("Release-Tag", "RELEASE.Sat-19-Sep-2015-06-15-16-GMT")
probe.SetAppInfo("Commit-ID", "7390cc957239")
es := testDummy2().Trace("TopOfStack")
// Uncomment the following Println to visually test probe call trace.
// fmt.Println("Expecting a simulated error here.", es)

0
vendor/github.com/minio/minio-xl/pkg/probe/wrapper.go → pkg/probe/wrapper.go
View file

0
vendor/github.com/minio/minio-xl/pkg/quick/errorutil.go → pkg/quick/errorutil.go
View file

7
vendor/github.com/minio/minio-xl/pkg/quick/quick.go → pkg/quick/quick.go
View file

@ -30,8 +30,8 @@ import (
"sync"
"github.com/fatih/structs"
"github.com/minio/minio-xl/pkg/atomic"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/atomic"
"github.com/minio/minio/pkg/probe"
)
// Config - generic config interface functions
@ -51,7 +51,8 @@ type config struct {
lock *sync.RWMutex
}
// CheckData - checks the validity of config data. Data sould be of type struct and contain a string type field called "Version"
// CheckData - checks the validity of config data. Data should be of
// type struct and contain a string type field called "Version".
func CheckData(data interface{}) *probe.Error {
if !structs.IsStruct(data) {
return probe.NewError(fmt.Errorf("Invalid argument type. Expecing \"struct\" type."))

2
vendor/github.com/minio/minio-xl/pkg/quick/quick_test.go → pkg/quick/quick_test.go
View file

@ -22,7 +22,7 @@ import (
"os"
"testing"
"github.com/minio/minio-xl/pkg/quick"
"github.com/minio/minio/pkg/quick"
. "gopkg.in/check.v1"
)

48
pkg/signature/errors.go Normal file
View file

@ -0,0 +1,48 @@
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package signature
// MissingDateHeader date header missing
type MissingDateHeader struct{}
func (e MissingDateHeader) Error() string {
return "Missing date header"
}
// MissingExpiresQuery expires query string missing
type MissingExpiresQuery struct{}
func (e MissingExpiresQuery) Error() string {
return "Missing expires query string"
}
// ExpiredPresignedRequest request already expired
type ExpiredPresignedRequest struct{}
func (e ExpiredPresignedRequest) Error() string {
return "Presigned request already expired"
}
// SigDoesNotMatch invalid signature
type SigDoesNotMatch struct {
SignatureSent string
SignatureCalculated string
}
func (e SigDoesNotMatch) Error() string {
return "The request signature we calculated does not match the signature you provided"
}

4
pkg/fs/postpolicyform.go → pkg/signature/postpolicyform.go
View file

@ -14,7 +14,7 @@
* limitations under the License.
*/
package fs
package signature
import (
"encoding/json"
@ -22,7 +22,7 @@ import (
"reflect"
"time"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/probe"
)
// toString - Safely convert interface to string without causing panic.

9
pkg/fs/signature.go → pkg/signature/signature-v4.go
View file

@ -14,7 +14,7 @@
* limitations under the License.
*/
package fs
package signature
import (
"bytes"
@ -29,8 +29,8 @@ import (
"time"
"unicode/utf8"
"github.com/minio/minio-xl/pkg/crypto/sha256"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio/pkg/crypto/sha256"
"github.com/minio/minio/pkg/probe"
)
// Signature - local variables
@ -243,7 +243,8 @@ func (r Signature) getScope(t time.Time) string {
func (r Signature) getStringToSign(canonicalRequest string, t time.Time) string {
stringToSign := authHeaderPrefix + "\n" + t.Format(iso8601Format) + "\n"
stringToSign = stringToSign + r.getScope(t) + "\n"
stringToSign = stringToSign + hex.EncodeToString(sha256.Sum256([]byte(canonicalRequest)))
canonicalRequestBytes := sha256.Sum256([]byte(canonicalRequest))
stringToSign = stringToSign + hex.EncodeToString(canonicalRequestBytes[:])
return stringToSign
}

44
pkg/tasker/commands.go Normal file
View file

@ -0,0 +1,44 @@
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package tasker
// Command is number that uniquely identifies a command function.
type Command uint8
// Enumerate the task commands.
const (
// CmdNOOP does nothing. It is a default placeholder. Uninitialized variable of this type will point to NOOP command by default.
CmdNOOP Command = iota
// CmdSignalEnd gracefully ends current task. Never ending tasks (loop over) or Batched jobs will not take the next iteration,
// but may finish the current state to completion.
CmdSignalEnd
// CmdSignalAbort ends the current task at hand immediately. It may still cleanup dangling issues quickly.
CmdSignalAbort
// CmdSignalSuspend suspends the current task.
CmdSignalSuspend
// CmdSignalResume resumes a suspended task.
CmdSignalResume
// CmdPriorityLow is optimized to conserve resources and complete the task at a slow pace. This option is ideal for batch processed tasks.
CmdPriorityLow
// CmdPriorityMedium is the default priority. It is a balanced option between resources and speed.
CmdPriorityMedium
// CmdPriorityHigh is optimized for speed. This option is ideal for short lived tasks (like meta-data related) that are latency sensitive. Use this option wisely.
CmdPriorityHigh
// CmdPrioritySuper is an exclusive priority. All tasks with priority lower than Super (including High) are paused
// temporarily until this task completes. Anytime you consider using this priority level, please seek for approval.
CmdPrioritySuper
)

56
pkg/tasker/handle.go Normal file
View file

@ -0,0 +1,56 @@
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package tasker
import "github.com/minio/minio/pkg/probe"
// Handle as the name suggests is a handle (self reference) to its
// own task structure. Task has limited privileges over itself. Only the
// task controller (TaskCtl) can manage the task by sending commands to
// the task over channels.
type Handle struct {
this taskRef
cmdCh <-chan Command // Channel to receive commands from TaskCtl.
statusCh chan<- status // Channel to send completion status and error (if any) to TaskCtl.
closeCh chan<- taskRef // Channel to notify the TaskCtl about ending this task.
}
// Listen returns a channel to receive commands.
func (t Handle) Listen() <-chan Command {
return t.cmdCh
}
// StatusDone acknowledges successful completion of a command.
func (t Handle) StatusDone() {
t.statusCh <- status{code: statusDone, err: nil}
}
// StatusBusy rejects a command with busy status.
func (t Handle) StatusBusy() {
t.statusCh <- status{code: statusBusy, err: nil}
}
// StatusFail returns failure status.
func (t Handle) StatusFail(err *probe.Error) {
t.statusCh <- status{code: statusFail, err: err}
}
// Close notifies the TaskCtl about the end of this Task. Owner of the
// task must invoke Close() when it is done performing its job.
func (t Handle) Close() {
t.closeCh <- t.this
}

35
pkg/tasker/status.go Normal file
View file

@ -0,0 +1,35 @@
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package tasker
import "github.com/minio/minio/pkg/probe"
// StatusCode denotes the completion status of a command.
type statusCode int8
// Enumerate task return status codes.
const (
statusDone statusCode = iota
statusBusy
statusFail
)
// Status returns the completion status and error (if any) of a command.
type status struct {
code statusCode // Completion code.
err *probe.Error // Error if any.
}

103
pkg/tasker/task.go Normal file
View file

@ -0,0 +1,103 @@
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package tasker
import (
"container/list"
"sync"
)
// NOTE: Task is a private entity. It is created and managed by TaskCtl
// entirely. Only TaskCtl and Handle objects are exposed outside.
// taskRef is a unique reference ID to a task. It is assigned by the
// TaskCtl during the creation of a task. All tasfRef variables are
// named "this".
type taskRef *list.Element
// Task is an abstract concept built on top of Go routines and
// channels. Tasks themselves are expected to co-operate and comply with
// the TaskCtl commands.
type task struct {
mutex *sync.Mutex
this taskRef // Refence to task entry in the TaskCtl's task list.
name string // Free form name.
priority Command // Current priority.
cmdCh chan Command // Channel to receive commands from TaskCtl.
statusCh chan status // Channel to send completion status and error (if any) to TaskCtl.
closeCh chan taskRef // Channel to notify the TaskCtl about ending this task.
}
// NewTask creates a new task structure and returns a handle to
// it. Only the task controller has access to the task structure. The
// caller routine only receives a handle to its task structure. Task
// handle is like a reference to task self. Caller is expected to listen
// for commands from the task controller and comply with it co-operatively.
// this: Task reference is unique identifier assigned by the TaskCtl.
// name: Free form name of the task. Eg. "Late Night Disk Scrubber".
func newTask(name string) task {
return task{
// this: Is set by the TaskCtl's NewTask function.
mutex: &sync.Mutex{},
name: name,
priority: CmdPriorityMedium,
cmdCh: make(chan Command),
statusCh: make(chan status),
closeCh: make(chan taskRef),
}
}
// getHandle returns a handle to the task. Handle has limited access to the task structure and it is safe to be exposed.
func (t task) getHandle() Handle {
t.mutex.Lock()
defer t.mutex.Unlock()
// Make a handle with limited access to channels (only send or receive).
return Handle{
cmdCh: t.cmdCh,
statusCh: t.statusCh,
closeCh: t.closeCh,
}
}
// command method sends a command code to the task and returns its completion status.
func (t task) command(cmd Command) status {
t.mutex.Lock()
defer t.mutex.Unlock()
t.cmdCh <- cmd
return <-t.statusCh
}
// close releases all the resources held by this task.
func (t task) close() {
t.mutex.Lock()
defer t.mutex.Unlock()
// Task can be ended in 2 ways.
// 1) Calling application invokes Handle.Close().
// 2) TaskCtl.Shutdown() ending the task's life.
// In either case, task.close() is invoked only via the
// TaskCtl. Handle.Close() only sends a message to the TaskCtl to
// initiate a close call.
close(t.cmdCh)
close(t.statusCh)
close(t.closeCh)
}

164
pkg/tasker/taskctl.go Normal file
View file

@ -0,0 +1,164 @@
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package tasker
import (
"container/list"
"sync"
)
// TaskCtl (Task Controller) is a framework to create and manage
// tasks.
type TaskCtl struct {
mutex *sync.Mutex // Lock
// List of tasks managed by this task controller.
tasks *list.List
}
// New creates a new TaskCtl to create and control a collection of tasks.
// Single application can create multiple task controllers to manage different set of tasks separately.
func New(name string) *TaskCtl {
return &TaskCtl{
mutex: &sync.Mutex{},
tasks: list.New(),
}
}
// NewTask creates a new task structure and returns a handle to it. Only the task controller
// has access to the task structure. The caller routine only receives a handle to its task structure.
// Task handle is like a reference to task self. Caller is expected to listen for commands from
// the task controller and comply with it co-operatively.
func (tc *TaskCtl) NewTask(name string) Handle {
tc.mutex.Lock()
defer tc.mutex.Unlock()
// Create a new task.
tsk := newTask(name)
// Register this task in the TaskCtl's tasklist and save the reference.
tsk.this = tc.tasks.PushBack(tsk)
// Free task from the tasklist upon close call.
go func() {
// Release the tasks resources upon return of this function.
defer tsk.close()
// Will be notified here upon task's end of life.
this := <-tsk.closeCh
tc.mutex.Lock()
defer tc.mutex.Unlock()
// Release the task structure from the task list.
tc.tasks.Remove(this)
}()
// Return a handle to this task.
return tsk.getHandle()
}
// Shutdown ends all tasks, including the suspended ones.
func (tc *TaskCtl) Shutdown() {
tc.mutex.Lock()
defer tc.mutex.Unlock()
var wg sync.WaitGroup
// End all tasks.
for e := tc.tasks.Front(); e != nil; e = e.Next() {
wg.Add(1)
thisTask := e.Value.(task) // Make a local copy for go routine.
// End tasks in background. Flow of events from here is as follows: thisTask.handle.Close() -> tc.NewTask() -> this.task.close().
go func() {
thisTask.getHandle().Close()
wg.Done()
}()
}
wg.Wait() // Wait for all tasks to end gracefully.
// Reset the task pool.
tc.tasks = nil
}
// Suspend puts all tasks to sleep.
func (tc *TaskCtl) Suspend() bool {
tc.mutex.Lock()
defer tc.mutex.Unlock()
var wg sync.WaitGroup
// If any one of the task fails to suspend, this flag will set to false.
statusAll := make([]status, tc.tasks.Len())
// Suspend all tasks.
i := 0
for e := tc.tasks.Front(); e != nil; e = e.Next() {
wg.Add(1)
locTask := e.Value.(task) // Make a local copy for go routine.
locI := i // local i
// Suspend a task in background.
go func(locI int) {
defer wg.Done()
statusAll[locI] = locTask.command(CmdSignalSuspend)
}(locI)
i++
}
wg.Wait() // Wait for all tasks to suspend gracefully.
for _, st := range statusAll {
if st.code != statusDone {
return false
}
}
return true
}
// Resume wakes up all suspended task from sleep.
func (tc *TaskCtl) Resume() bool {
tc.mutex.Lock()
defer tc.mutex.Unlock()
var wg sync.WaitGroup
// If any one of the task fails to suspend, this flag will set to false.
statusAll := make([]status, tc.tasks.Len())
i := 0
// Resume all suspended tasks.
for e := tc.tasks.Front(); e != nil; e = e.Next() {
wg.Add(1)
locTask := e.Value.(task) // Make a local copy for go routine.
locI := i // local i
// Resume a task in background.
go func(locI int) {
defer wg.Done()
statusAll[locI] = locTask.command(CmdSignalResume)
}(locI)
i++
}
wg.Wait() // Wait for all tasks to resume.
for _, st := range statusAll {
if st.code != statusDone {
return false
}
}
return true
}

38
pkg/tasker/taskctl_test.go Normal file
View file

@ -0,0 +1,38 @@
/*
* Quick - Quick key value store for config files and persistent state files
*
* Minio Client (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package tasker_test
import (
"testing"
"github.com/minio/minio/pkg/tasker"
. "gopkg.in/check.v1"
)
func Test(t *testing.T) { TestingT(t) }
type MySuite struct{}
var _ = Suite(&MySuite{})
func (s *MySuite) TestCheckData(c *C) {
testTasks := tasker.New("Test Task")
testTasks.Shutdown()
// c.Assert(err, Not(IsNil))
}

Some files were not shown because too many files have changed in this diff Show more