Migration regression got introduced in 9083bc152e
adding more unit tests to catch this scenario, we need to fix this by
re-writing the formats after the migration to 'V3'.
This bug only happens when a user is migrating directly from V1 to V3,
not from V1 to V2 and V2 to V3.
Added additional unit tests to cover these situations as well.
Fixes #5667
187 lines
5.8 KiB
Go
187 lines
5.8 KiB
Go
/*
|
|
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package cmd
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"time"
|
|
|
|
"github.com/minio/mc/pkg/console"
|
|
"github.com/minio/minio/pkg/errors"
|
|
)
|
|
|
|
var printEndpointError = func() func(Endpoint, error) {
|
|
printOnce := make(map[Endpoint]map[string]bool)
|
|
|
|
return func(endpoint Endpoint, err error) {
|
|
m, ok := printOnce[endpoint]
|
|
if !ok {
|
|
m = make(map[string]bool)
|
|
m[err.Error()] = true
|
|
printOnce[endpoint] = m
|
|
errorIf(err, "%s: %s", endpoint, err)
|
|
return
|
|
}
|
|
if m[err.Error()] {
|
|
return
|
|
}
|
|
m[err.Error()] = true
|
|
errorIf(err, "%s: %s", endpoint, err)
|
|
}
|
|
}()
|
|
|
|
// Migrates backend format of local disks.
|
|
func formatXLMigrateLocalEndpoints(endpoints EndpointList) error {
|
|
for _, endpoint := range endpoints {
|
|
if !endpoint.IsLocal {
|
|
continue
|
|
}
|
|
formatPath := pathJoin(endpoint.Path, minioMetaBucket, formatConfigFile)
|
|
if _, err := os.Stat(formatPath); err != nil {
|
|
if os.IsNotExist(err) {
|
|
continue
|
|
}
|
|
return err
|
|
}
|
|
if err := formatXLMigrate(endpoint.Path); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Cleans up tmp directory of local disks.
|
|
func formatXLCleanupTmpLocalEndpoints(endpoints EndpointList) error {
|
|
for _, endpoint := range endpoints {
|
|
if !endpoint.IsLocal {
|
|
continue
|
|
}
|
|
formatPath := pathJoin(endpoint.Path, minioMetaBucket, formatConfigFile)
|
|
if _, err := os.Stat(formatPath); err != nil {
|
|
if os.IsNotExist(err) {
|
|
continue
|
|
}
|
|
return err
|
|
}
|
|
if err := os.RemoveAll(pathJoin(endpoint.Path, minioMetaTmpBucket)); err != nil {
|
|
return err
|
|
}
|
|
if err := os.MkdirAll(pathJoin(endpoint.Path, minioMetaTmpBucket), 0777); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Format disks before initialization of object layer.
|
|
func waitForFormatXL(firstDisk bool, endpoints EndpointList, setCount, disksPerSet int) (format *formatXLV3, err error) {
|
|
if len(endpoints) == 0 || setCount == 0 || disksPerSet == 0 {
|
|
return nil, errInvalidArgument
|
|
}
|
|
|
|
if err = formatXLMigrateLocalEndpoints(endpoints); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err = formatXLCleanupTmpLocalEndpoints(endpoints); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Done channel is used to close any lingering retry routine, as soon
|
|
// as this function returns.
|
|
doneCh := make(chan struct{})
|
|
|
|
// Indicate to our retry routine to exit cleanly, upon this function return.
|
|
defer close(doneCh)
|
|
|
|
// prepare getElapsedTime() to calculate elapsed time since we started trying formatting disks.
|
|
// All times are rounded to avoid showing milli, micro and nano seconds
|
|
formatStartTime := time.Now().Round(time.Second)
|
|
getElapsedTime := func() string {
|
|
return time.Now().Round(time.Second).Sub(formatStartTime).String()
|
|
}
|
|
|
|
// Wait on the jitter retry loop.
|
|
retryTimerCh := newRetryTimerSimple(doneCh)
|
|
for {
|
|
select {
|
|
case _ = <-retryTimerCh:
|
|
// Attempt to load all `format.json` from all disks.
|
|
formatConfigs, sErrs := loadFormatXLAll(endpoints)
|
|
|
|
// Pre-emptively check if one of the formatted disks
|
|
// is invalid. This function returns success for the
|
|
// most part unless one of the formats is not consistent
|
|
// with expected XL format. For example if a user is
|
|
// trying to pool FS backend into an XL set.
|
|
if err = checkFormatXLValues(formatConfigs); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
for i, sErr := range sErrs {
|
|
if _, ok := formatCriticalErrors[errors.Cause(sErr)]; ok {
|
|
return nil, fmt.Errorf("Disk %s: %s", endpoints[i], sErr)
|
|
}
|
|
}
|
|
|
|
if shouldInitXLDisks(sErrs) {
|
|
if !firstDisk {
|
|
console.Println("Waiting for the first server to format the disks.")
|
|
continue
|
|
}
|
|
return initFormatXL(endpoints, setCount, disksPerSet)
|
|
}
|
|
|
|
// Following function is added to fix a regressions which was introduced
|
|
// in release RELEASE.2018-03-16T22-52-12Z after migrating v1 to v2 to v3.
|
|
// This migration failed to capture '.This' field properly which indicates
|
|
// the disk UUID association. Below function is called to handle and fix
|
|
// this regression, for more info refer https://github.com/minio/minio/issues/5667
|
|
if err = fixFormatXLV3(endpoints, formatConfigs); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// If any of the .This field is still empty we wait them to be fixed.
|
|
if formatXLV3ThisEmpty(formatConfigs) {
|
|
continue
|
|
}
|
|
|
|
format, err = getFormatXLInQuorum(formatConfigs)
|
|
if err == nil {
|
|
for i := range formatConfigs {
|
|
if formatConfigs[i] == nil {
|
|
continue
|
|
}
|
|
if err = formatXLV3Check(format, formatConfigs[i]); err != nil {
|
|
return nil, fmt.Errorf("%s format error: %s", endpoints[i], err)
|
|
}
|
|
}
|
|
if len(format.XL.Sets) != globalXLSetCount {
|
|
return nil, fmt.Errorf("Current backend format is inconsistent with input args (%s), Expected set count %d, got %d", endpoints, len(format.XL.Sets), globalXLSetCount)
|
|
}
|
|
if len(format.XL.Sets[0]) != globalXLSetDriveCount {
|
|
return nil, fmt.Errorf("Current backend format is inconsistent with input args (%s), Expected drive count per set %d, got %d", endpoints, len(format.XL.Sets[0]), globalXLSetDriveCount)
|
|
}
|
|
return format, nil
|
|
}
|
|
console.Printf("Waiting for a minimum of %d disks to come online (elapsed %s)\n", len(endpoints)/2, getElapsedTime())
|
|
case <-globalOSSignalCh:
|
|
return nil, fmt.Errorf("Initializing data volumes gracefully stopped")
|
|
}
|
|
}
|
|
}
|