maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Improve disk formatting stage for large disk sets (#8690)

Browse source
This commit is contained in:
Harshavardhana 2019-12-23 16:31:03 -08:00 committed by kannappanr
parent 725172e13b
commit f68a7005c0
12 changed files with 139 additions and 206 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
cmd/config/errors-utils.go
View file

@ -39,7 +39,10 @@ type Err struct {
// Return the error message
func (u Err) Error() string {
if u.detail == "" {
return u.msg
if u.msg != "" {
return u.msg
}
return "<nil>"
}
return u.detail
}
@ -77,6 +80,10 @@ func newErrFn(msg, action, hint string) ErrFn {
// ErrorToErr inspects the passed error and transforms it
// to the appropriate UI error.
func ErrorToErr(err error) Err {
if err == nil {
return Err{}
}
// If this is already a Err, do nothing
if e, ok := err.(Err); ok {
return e
@ -95,7 +102,6 @@ func ErrorToErr(err error) Err {
// Failed to identify what type of error this, return a simple UI error
return Err{msg: err.Error()}
}
}
// FmtError converts a fatal error message to a more clear error

172
cmd/format-xl.go
View file

@ -155,24 +155,9 @@ func newFormatXLV3(numSets int, setLen int) *formatXLV3 {
return format
}
// Returns formatXL.XL.Version information, this code is specifically
// used to read XL `format.json` and capture any version information
// that it may have.
func formatXLGetVersion(formatPath string) (string, error) {
format := &formatXLVersionDetect{}
b, err := ioutil.ReadFile(formatPath)
if err != nil {
return "", err
}
if err = json.Unmarshal(b, format); err != nil {
return "", err
}
return format.XL.Version, nil
}
// Returns format meta format version from `format.json`. This code
// is specifically used to detect meta format.
func formatMetaGetFormatBackendXL(formatPath string) (string, error) {
// Returns format XL version after reading `format.json`, returns
// successfully the version only if the backend is XL.
func formatGetBackendXLVersion(formatPath string) (string, error) {
meta := &formatMetaV1{}
b, err := ioutil.ReadFile(formatPath)
if err != nil {
@ -184,7 +169,15 @@ func formatMetaGetFormatBackendXL(formatPath string) (string, error) {
if meta.Version != formatMetaVersionV1 {
return "", fmt.Errorf(`format.Version expected: %s, got: %s`, formatMetaVersionV1, meta.Version)
}
return meta.Format, nil
if meta.Format != formatBackendXL {
return "", fmt.Errorf(`found backend %s, expected %s`, meta.Format, formatBackendXL)
}
// XL backend found, proceed to detect version.
format := &formatXLVersionDetect{}
if err = json.Unmarshal(b, format); err != nil {
return "", err
}
return format.XL.Version, nil
}
// Migrates all previous versions to latest version of `format.json`,
@ -192,30 +185,27 @@ func formatMetaGetFormatBackendXL(formatPath string) (string, error) {
// first before it V2 migrates to V3.
func formatXLMigrate(export string) error {
formatPath := pathJoin(export, minioMetaBucket, formatConfigFile)
backend, err := formatMetaGetFormatBackendXL(formatPath)
if err != nil {
return err
}
if backend != formatBackendXL {
return fmt.Errorf(`Disk %s: found backend %s, expected %s`, export, backend, formatBackendXL)
}
version, err := formatXLGetVersion(formatPath)
version, err := formatGetBackendXLVersion(formatPath)
if err != nil {
return err
}
switch version {
case formatXLVersionV1:
if err = formatXLMigrateV1ToV2(export); err != nil {
if err = formatXLMigrateV1ToV2(export, version); err != nil {
return err
}
// Migrate successful v1 => v2, proceed to v2 => v3
version = formatXLVersionV2
fallthrough
case formatXLVersionV2:
if err = formatXLMigrateV2ToV3(export); err != nil {
if err = formatXLMigrateV2ToV3(export, version); err != nil {
return err
}
// Migrate successful v2 => v3, v3 is latest
version = formatXLVersionV3
fallthrough
case formatXLVersionV3:
// format-V3 is the latest verion.
// v3 is the latest version, return.
return nil
}
return fmt.Errorf(`%s: unknown format version %s`, export, version)
@ -223,16 +213,13 @@ func formatXLMigrate(export string) error {
// Migrates version V1 of format.json to version V2 of format.json,
// migration fails upon any error.
func formatXLMigrateV1ToV2(export string) error {
formatPath := pathJoin(export, minioMetaBucket, formatConfigFile)
version, err := formatXLGetVersion(formatPath)
if err != nil {
return err
}
func formatXLMigrateV1ToV2(export, version string) error {
if version != formatXLVersionV1 {
return fmt.Errorf(`Disk %s: format version expected %s, found %s`, export, formatXLVersionV1, version)
}
formatPath := pathJoin(export, minioMetaBucket, formatConfigFile)
formatV1 := &formatXLV1{}
b, err := ioutil.ReadFile(formatPath)
if err != nil {
@ -260,15 +247,12 @@ func formatXLMigrateV1ToV2(export string) error {
}
// Migrates V2 for format.json to V3 (Flat hierarchy for multipart)
func formatXLMigrateV2ToV3(export string) error {
formatPath := pathJoin(export, minioMetaBucket, formatConfigFile)
version, err := formatXLGetVersion(formatPath)
if err != nil {
return err
}
func formatXLMigrateV2ToV3(export, version string) error {
if version != formatXLVersionV2 {
return fmt.Errorf(`Disk %s: format version expected %s, found %s`, export, formatXLVersionV2, version)
}
formatPath := pathJoin(export, minioMetaBucket, formatConfigFile)
formatV2 := &formatXLV2{}
b, err := ioutil.ReadFile(formatPath)
if err != nil {
@ -360,18 +344,18 @@ func saveFormatXL(disk StorageAPI, format interface{}) error {
return err
}
tmpFormatJSON := mustGetUUID() + ".json"
tmpFormat := mustGetUUID()
// Purge any existing temporary file, okay to ignore errors here.
defer disk.DeleteFile(minioMetaBucket, tmpFormatJSON)
defer disk.DeleteFile(minioMetaBucket, tmpFormat)
// Append file `format.json.tmp`.
if err = disk.WriteAll(minioMetaBucket, tmpFormatJSON, bytes.NewReader(formatBytes)); err != nil {
// write to unique file.
if err = disk.WriteAll(minioMetaBucket, tmpFormat, bytes.NewReader(formatBytes)); err != nil {
return err
}
// Rename file `uuid.json` --> `format.json`.
return disk.RenameFile(minioMetaBucket, tmpFormatJSON, minioMetaBucket, formatConfigFile)
return disk.RenameFile(minioMetaBucket, tmpFormat, minioMetaBucket, formatConfigFile)
}
var ignoredHiddenDirectories = []string{
@ -635,6 +619,39 @@ func formatXLV3Check(reference *formatXLV3, format *formatXLV3) error {
return fmt.Errorf("Disk ID %s not found in any disk sets %s", this, format.XL.Sets)
}
// Initializes meta volume on all input storage disks.
func initFormatXLMetaVolume(storageDisks []StorageAPI, formats []*formatXLV3) error {
// This happens for the first time, but keep this here since this
// is the only place where it can be made expensive optimizing all
// other calls. Create minio meta volume, if it doesn't exist yet.
// Initialize errs to collect errors inside go-routine.
g := errgroup.WithNErrs(len(storageDisks))
// Initialize all disks in parallel.
for index := range storageDisks {
index := index
g.Go(func() error {
if formats[index] == nil || storageDisks[index] == nil {
// Ignore create meta volume on disks which are not found.
return nil
}
return makeFormatXLMetaVolumes(storageDisks[index])
}, index)
}
// Return upon first error.
for _, err := range g.Wait() {
if err == nil {
continue
}
return toObjectErr(err, minioMetaBucket)
}
// Return success here.
return nil
}
// saveFormatXLAll - populates `format.json` on disks in its order.
func saveFormatXLAll(ctx context.Context, storageDisks []StorageAPI, formats []*formatXLV3) error {
g := errgroup.WithNErrs(len(storageDisks))
@ -646,6 +663,9 @@ func saveFormatXLAll(ctx context.Context, storageDisks []StorageAPI, formats []*
if formats[index] == nil || storageDisks[index] == nil {
return errDiskNotFound
}
if err := makeFormatXLMetaVolumes(storageDisks[index]); err != nil {
return err
}
return saveFormatXL(storageDisks[index], formats[index])
}, index)
}
@ -745,11 +765,6 @@ func initFormatXL(ctx context.Context, storageDisks []StorageAPI, setCount, driv
}
}
// Initialize meta volume, if volume already exists ignores it.
if err := initFormatXLMetaVolume(storageDisks, formats); err != nil {
return format, fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %w", err)
}
// Save formats `format.json` across all disks.
if err := saveFormatXLAll(ctx, storageDisks, formats); err != nil {
return nil, err
@ -760,23 +775,9 @@ func initFormatXL(ctx context.Context, storageDisks []StorageAPI, setCount, driv
// Make XL backend meta volumes.
func makeFormatXLMetaVolumes(disk StorageAPI) error {
// Attempt to create `.minio.sys`.
if err := disk.MakeVol(minioMetaBucket); err != nil {
if !IsErrIgnored(err, initMetaVolIgnoredErrs...) {
return err
}
}
if err := disk.MakeVol(minioMetaTmpBucket); err != nil {
if !IsErrIgnored(err, initMetaVolIgnoredErrs...) {
return err
}
}
if err := disk.MakeVol(minioMetaBackgroundOpsBucket); err != nil {
if !IsErrIgnored(err, initMetaVolIgnoredErrs...) {
return err
}
}
if err := disk.MakeVol(minioMetaMultipartBucket); err != nil {
// Attempt to create MinIO internal buckets.
err := disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, minioMetaBackgroundOpsBucket)
if err != nil {
if !IsErrIgnored(err, initMetaVolIgnoredErrs...) {
return err
}
@ -786,39 +787,6 @@ func makeFormatXLMetaVolumes(disk StorageAPI) error {
var initMetaVolIgnoredErrs = append(baseIgnoredErrs, errVolumeExists)
// Initializes meta volume on all input storage disks.
func initFormatXLMetaVolume(storageDisks []StorageAPI, formats []*formatXLV3) error {
// This happens for the first time, but keep this here since this
// is the only place where it can be made expensive optimizing all
// other calls. Create minio meta volume, if it doesn't exist yet.
// Initialize errs to collect errors inside go-routine.
g := errgroup.WithNErrs(len(storageDisks))
// Initialize all disks in parallel.
for index := range storageDisks {
index := index
g.Go(func() error {
if formats[index] == nil || storageDisks[index] == nil {
// Ignore create meta volume on disks which are not found.
return nil
}
return makeFormatXLMetaVolumes(storageDisks[index])
}, index)
}
// Return upon first error.
for _, err := range g.Wait() {
if err == nil {
continue
}
return toObjectErr(err, minioMetaBucket)
}
// Return success here.
return nil
}
// Get all UUIDs which are present in reference format should
// be present in the list of formats provided, those are considered
// as online UUIDs.

92
cmd/format-xl_test.go
View file

@ -149,95 +149,6 @@ func TestFormatXLEmpty(t *testing.T) {
}
}
// Tests format xl get version.
func TestFormatXLGetVersion(t *testing.T) {
// Get test root.
rootPath, err := getTestRoot()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
v := &formatXLVersionDetect{}
v.XL.Version = "1"
b, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
if err = ioutil.WriteFile(pathJoin(rootPath, formatConfigFile), b, os.FileMode(0644)); err != nil {
t.Fatal(err)
}
_, err = formatXLGetVersion("not-found")
if err == nil {
t.Fatal("Expected to fail but found success")
}
vstr, err := formatXLGetVersion(pathJoin(rootPath, formatConfigFile))
if err != nil {
t.Fatal(err)
}
if vstr != "1" {
t.Fatalf("Expected version '1', got '%s'", vstr)
}
}
// Tests format get backend format.
func TestFormatMetaGetFormatBackendXL(t *testing.T) {
// Get test root.
rootPath, err := getTestRoot()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
m := &formatMetaV1{
Format: "fs",
Version: formatMetaVersionV1,
}
b, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
if err = ioutil.WriteFile(pathJoin(rootPath, formatConfigFile), b, os.FileMode(0644)); err != nil {
t.Fatal(err)
}
_, err = formatMetaGetFormatBackendXL("not-found")
if err == nil {
t.Fatal("Expected to fail but found success")
}
format, err := formatMetaGetFormatBackendXL(pathJoin(rootPath, formatConfigFile))
if err != nil {
t.Fatal(err)
}
if format != m.Format {
t.Fatalf("Expected format value %s, got %s", m.Format, format)
}
m = &formatMetaV1{
Format: "xl",
Version: "2",
}
b, err = json.Marshal(m)
if err != nil {
t.Fatal(err)
}
if err = ioutil.WriteFile(pathJoin(rootPath, formatConfigFile), b, os.FileMode(0644)); err != nil {
t.Fatal(err)
}
_, err = formatMetaGetFormatBackendXL(pathJoin(rootPath, formatConfigFile))
if err == nil {
t.Fatal("Expected to fail with incompatible meta version")
}
}
// Tests xl format migration.
func TestFormatXLMigrate(t *testing.T) {
// Get test root.
@ -271,10 +182,11 @@ func TestFormatXLMigrate(t *testing.T) {
t.Fatal(err)
}
migratedVersion, err := formatXLGetVersion(pathJoin(rootPath, minioMetaBucket, formatConfigFile))
migratedVersion, err := formatGetBackendXLVersion(pathJoin(rootPath, minioMetaBucket, formatConfigFile))
if err != nil {
t.Fatal(err)
}
if migratedVersion != formatXLVersionV3 {
t.Fatalf("expected version: %s, got: %s", formatXLVersionV3, migratedVersion)
}

7
cmd/naughty-disk_test.go
View file

@ -91,6 +91,13 @@ func (d *naughtyDisk) DiskInfo() (info DiskInfo, err error) {
return d.disk.DiskInfo()
}
func (d *naughtyDisk) MakeVolBulk(volumes ...string) (err error) {
if err := d.calcError(); err != nil {
return err
}
return d.disk.MakeVolBulk(volumes...)
}
func (d *naughtyDisk) MakeVol(volume string) (err error) {
if err := d.calcError(); err != nil {
return err

7
cmd/posix-diskid-check.go
View file

@ -74,6 +74,13 @@ func (p *posixDiskIDCheck) DiskInfo() (info DiskInfo, err error) {
return p.storage.DiskInfo()
}
func (p *posixDiskIDCheck) MakeVolBulk(volumes ...string) (err error) {
if p.isDiskStale() {
return errDiskNotFound
}
return p.storage.MakeVolBulk(volumes...)
}
func (p *posixDiskIDCheck) MakeVol(volume string) (err error) {
if p.isDiskStale() {
return errDiskNotFound

9
cmd/posix.go
View file

@ -551,6 +551,15 @@ func (s *posix) SetDiskID(id string) {
// storage rest server for remote disks.
}
func (s *posix) MakeVolBulk(volumes ...string) (err error) {
for _, volume := range volumes {
if err = s.MakeVol(volume); err != nil {
return err
}
}
return nil
}
// Make a volume entry.
func (s *posix) MakeVol(volume string) (err error) {
defer func() {

14
cmd/prepare-storage.go
View file

@ -223,12 +223,13 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
// Assign globalDeploymentID on first run for the
// minio server managing the first disk
globalDeploymentID = format.ID
} else {
// The first will always recreate some directories inside .minio.sys
// such as, tmp, multipart and background-ops
if firstDisk {
initFormatXLMetaVolume(storageDisks, formatConfigs)
}
return format, nil
}
// The first will always recreate some directories inside .minio.sys
// such as, tmp, multipart and background-ops
if firstDisk {
initFormatXLMetaVolume(storageDisks, formatConfigs)
}
// Return error when quorum unformatted disks - indicating we are
@ -288,6 +289,7 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
if err = formatXLFixLocalDeploymentID(endpoints, storageDisks, format); err != nil {
return nil, err
}
return format, nil
}

1
cmd/storage-interface.go
View file

@ -36,6 +36,7 @@ type StorageAPI interface {
// Volume operations.
MakeVol(volume string) (err error)
MakeVolBulk(volumes ...string) (err error)
ListVols() (vols []VolInfo, err error)
StatVol(volume string) (vol VolInfo, err error)
DeleteVol(volume string) (err error)

10
cmd/storage-rest-client.go
View file

@ -28,6 +28,7 @@ import (
"net/url"
"path"
"strconv"
"strings"
"sync/atomic"
"github.com/minio/minio/cmd/http"
@ -189,6 +190,15 @@ func (client *storageRESTClient) DiskInfo() (info DiskInfo, err error) {
return info, err
}
// MakeVolBulk - create multiple volumes in a bulk operation.
func (client *storageRESTClient) MakeVolBulk(volumes ...string) (err error) {
values := make(url.Values)
values.Set(storageRESTVolumes, strings.Join(volumes, ","))
respBody, err := client.call(storageRESTMethodMakeVolBulk, values, nil, -1)
defer http.DrainBody(respBody)
return err
}
// MakeVol - create a volume on a remote disk.
func (client *storageRESTClient) MakeVol(volume string) (err error) {
values := make(url.Values)

2
cmd/storage-rest-common.go
View file

@ -26,6 +26,7 @@ const (
storageRESTMethodDiskInfo = "/diskinfo"
storageRESTMethodCrawlAndGetDataUsage = "/crawlandgetdatausage"
storageRESTMethodMakeVol = "/makevol"
storageRESTMethodMakeVolBulk = "/makevolbulk"
storageRESTMethodStatVol = "/statvol"
storageRESTMethodDeleteVol = "/deletevol"
storageRESTMethodListVols = "/listvols"
@ -47,6 +48,7 @@ const (
const (
storageRESTVolume = "volume"
storageRESTVolumes = "volumes"
storageRESTDirPath = "dir-path"
storageRESTFilePath = "file-path"
storageRESTSrcVolume = "source-volume"

15
cmd/storage-rest-server.go
View file

@ -26,6 +26,7 @@ import (
"net/http"
"path"
"strconv"
"strings"
"time"
"github.com/gorilla/mux"
@ -150,6 +151,19 @@ func (s *storageRESTServer) MakeVolHandler(w http.ResponseWriter, r *http.Reques
}
}
// MakeVolBulkHandler - create multiple volumes as a bulk operation.
func (s *storageRESTServer) MakeVolBulkHandler(w http.ResponseWriter, r *http.Request) {
if !s.IsValid(w, r) {
return
}
vars := mux.Vars(r)
volumes := strings.Split(vars[storageRESTVolumes], ",")
err := s.storage.MakeVolBulk(volumes...)
if err != nil {
s.writeErrorResponse(w, err)
}
}
// ListVolsHandler - list volumes.
func (s *storageRESTServer) ListVolsHandler(w http.ResponseWriter, r *http.Request) {
if !s.IsValid(w, r) {
@ -605,6 +619,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpointZones EndpointZones
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDiskInfo).HandlerFunc(httpTraceHdrs(server.DiskInfoHandler))
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCrawlAndGetDataUsage).HandlerFunc(httpTraceHdrs(server.CrawlAndGetDataUsageHandler))
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodMakeVol).HandlerFunc(httpTraceHdrs(server.MakeVolHandler)).Queries(restQueries(storageRESTVolume)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodMakeVolBulk).HandlerFunc(httpTraceHdrs(server.MakeVolBulkHandler)).Queries(restQueries(storageRESTVolumes)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatVol).HandlerFunc(httpTraceHdrs(server.StatVolHandler)).Queries(restQueries(storageRESTVolume)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteVol).HandlerFunc(httpTraceHdrs(server.DeleteVolHandler)).Queries(restQueries(storageRESTVolume)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodListVols).HandlerFunc(httpTraceHdrs(server.ListVolsHandler))

6
cmd/xl-sets.go
View file

@ -1491,12 +1491,6 @@ func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealRe
}
}
// Initialize meta volume, if volume already exists ignores it, all disks which
// are not found are ignored as well.
if err = initFormatXLMetaVolume(storageDisks, tmpNewFormats); err != nil {
return madmin.HealResultItem{}, fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %w", err)
}
// Save formats `format.json` across all disks.
if err = saveFormatXLAll(ctx, storageDisks, tmpNewFormats); err != nil {
return madmin.HealResultItem{}, err