2016-05-04 21:18:20 +02:00
/ *
2019-04-09 20:39:42 +02:00
* MinIO Cloud Storage , ( C ) 2016 , 2017 , 2018 MinIO , Inc .
2016-05-04 21:18:20 +02:00
*
* Licensed under the Apache License , Version 2.0 ( the "License" ) ;
* you may not use this file except in compliance with the License .
* You may obtain a copy of the License at
*
* http : //www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing , software
* distributed under the License is distributed on an "AS IS" BASIS ,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
* See the License for the specific language governing permissions and
* limitations under the License .
* /
2016-08-19 01:23:42 +02:00
package cmd
2016-05-04 21:18:20 +02:00
2016-05-07 09:59:43 +02:00
import (
2019-09-25 03:47:26 +02:00
"bytes"
2018-04-06 00:04:40 +02:00
"context"
2020-01-13 22:09:10 +01:00
"encoding/hex"
2016-05-07 09:59:43 +02:00
"encoding/json"
"fmt"
2018-02-16 02:45:57 +01:00
"io/ioutil"
2018-07-19 05:17:35 +02:00
"reflect"
2020-01-13 22:09:10 +01:00
"sync"
2018-02-16 02:45:57 +01:00
2018-02-21 03:42:09 +01:00
humanize "github.com/dustin/go-humanize"
2020-01-13 22:09:10 +01:00
"github.com/minio/minio/cmd/config/storageclass"
2018-07-19 05:17:35 +02:00
"github.com/minio/minio/cmd/logger"
2020-01-13 22:09:10 +01:00
"github.com/minio/minio/pkg/color"
2019-10-14 18:44:51 +02:00
"github.com/minio/minio/pkg/sync/errgroup"
2018-02-21 03:42:09 +01:00
sha256 "github.com/minio/sha256-simd"
2017-06-13 02:40:28 +02:00
)
const (
// Represents XL backend.
formatBackendXL = "xl"
2018-02-16 02:45:57 +01:00
// formatXLV1.XL.Version - version '1'.
2018-01-08 23:30:55 +01:00
formatXLVersionV1 = "1"
2018-02-16 02:45:57 +01:00
// formatXLV2.XL.Version - version '2'.
formatXLVersionV2 = "2"
2018-03-15 21:55:23 +01:00
// formatXLV3.XL.Version - version '3'.
formatXLVersionV3 = "3"
2018-02-16 02:45:57 +01:00
// Distribution algorithm used.
formatXLVersionV2DistributionAlgo = "CRCMOD"
2018-01-08 23:30:55 +01:00
)
2017-06-13 02:40:28 +02:00
2018-02-16 02:45:57 +01:00
// Offline disk UUID represents an offline disk.
const offlineDiskUUID = "ffffffff-ffff-ffff-ffff-ffffffffffff"
// Healing is only supported for the list of errors mentioned here.
var formatHealErrors = map [ error ] struct { } {
errUnformattedDisk : { } ,
errDiskNotFound : { } ,
}
// List of errors considered critical for disk formatting.
var formatCriticalErrors = map [ error ] struct { } {
errCorruptedFormat : { } ,
errFaultyDisk : { } ,
}
// Used to detect the version of "xl" format.
type formatXLVersionDetect struct {
XL struct {
Version string ` json:"version" `
} ` json:"xl" `
}
// Represents the V1 backend disk structure version
// under `.minio.sys` and actual data namespace.
2018-01-08 23:30:55 +01:00
// formatXLV1 - structure holds format config version '1'.
type formatXLV1 struct {
formatMetaV1
XL struct {
Version string ` json:"version" ` // Version of 'xl' format.
Disk string ` json:"disk" ` // Disk field carries assigned disk uuid.
// JBOD field carries the input disk order generated the first
// time when fresh disks were supplied.
JBOD [ ] string ` json:"jbod" `
} ` json:"xl" ` // XL field holds xl format.
2017-06-13 02:40:28 +02:00
}
2018-02-16 02:45:57 +01:00
// Represents the V2 backend disk structure version
// under `.minio.sys` and actual data namespace.
// formatXLV2 - structure holds format config version '2'.
2018-07-17 05:26:42 +02:00
// The V2 format to support "large bucket" support where a bucket
// can span multiple erasure sets.
2018-02-16 02:45:57 +01:00
type formatXLV2 struct {
2018-07-17 05:26:42 +02:00
formatMetaV1
XL struct {
2018-02-16 02:45:57 +01:00
Version string ` json:"version" ` // Version of 'xl' format.
This string ` json:"this" ` // This field carries assigned disk uuid.
// Sets field carries the input disk order generated the first
// time when fresh disks were supplied, it is a two dimensional
// array second dimension represents list of disks used per set.
Sets [ ] [ ] string ` json:"sets" `
// Distribution algorithm represents the hashing algorithm
// to pick the right set index for an object.
DistributionAlgo string ` json:"distributionAlgo" `
} ` json:"xl" `
}
2018-03-15 21:55:23 +01:00
// formatXLV3 struct is same as formatXLV2 struct except that formatXLV3.XL.Version is "3" indicating
// the simplified multipart backend which is a flat hierarchy now.
// In .minio.sys/multipart we have:
// sha256(bucket/object)/uploadID/[xl.json, part.1, part.2 ....]
type formatXLV3 struct {
2018-07-17 05:26:42 +02:00
formatMetaV1
XL struct {
2018-03-15 21:55:23 +01:00
Version string ` json:"version" ` // Version of 'xl' format.
This string ` json:"this" ` // This field carries assigned disk uuid.
// Sets field carries the input disk order generated the first
// time when fresh disks were supplied, it is a two dimensional
// array second dimension represents list of disks used per set.
Sets [ ] [ ] string ` json:"sets" `
// Distribution algorithm represents the hashing algorithm
// to pick the right set index for an object.
DistributionAlgo string ` json:"distributionAlgo" `
} ` json:"xl" `
}
2019-11-21 13:24:51 +01:00
func ( f * formatXLV3 ) Clone ( ) * formatXLV3 {
b , err := json . Marshal ( f )
if err != nil {
panic ( err )
}
var dst formatXLV3
if err = json . Unmarshal ( b , & dst ) ; err != nil {
panic ( err )
}
return & dst
}
2018-02-16 02:45:57 +01:00
// Returns formatXL.XL.Version
2018-03-15 21:55:23 +01:00
func newFormatXLV3 ( numSets int , setLen int ) * formatXLV3 {
format := & formatXLV3 { }
2018-02-16 02:45:57 +01:00
format . Version = formatMetaVersionV1
format . Format = formatBackendXL
2018-07-19 05:17:35 +02:00
format . ID = mustGetUUID ( )
2018-03-15 21:55:23 +01:00
format . XL . Version = formatXLVersionV3
2018-02-16 02:45:57 +01:00
format . XL . DistributionAlgo = formatXLVersionV2DistributionAlgo
format . XL . Sets = make ( [ ] [ ] string , numSets )
for i := 0 ; i < numSets ; i ++ {
format . XL . Sets [ i ] = make ( [ ] string , setLen )
for j := 0 ; j < setLen ; j ++ {
format . XL . Sets [ i ] [ j ] = mustGetUUID ( )
}
}
return format
}
2019-12-24 01:31:03 +01:00
// Returns format XL version after reading `format.json`, returns
// successfully the version only if the backend is XL.
func formatGetBackendXLVersion ( formatPath string ) ( string , error ) {
2018-02-16 02:45:57 +01:00
meta := & formatMetaV1 { }
b , err := ioutil . ReadFile ( formatPath )
if err != nil {
return "" , err
}
if err = json . Unmarshal ( b , meta ) ; err != nil {
return "" , err
}
if meta . Version != formatMetaVersionV1 {
return "" , fmt . Errorf ( ` format.Version expected: %s, got: %s ` , formatMetaVersionV1 , meta . Version )
}
2019-12-24 01:31:03 +01:00
if meta . Format != formatBackendXL {
return "" , fmt . Errorf ( ` found backend %s, expected %s ` , meta . Format , formatBackendXL )
}
// XL backend found, proceed to detect version.
format := & formatXLVersionDetect { }
if err = json . Unmarshal ( b , format ) ; err != nil {
return "" , err
}
return format . XL . Version , nil
2018-02-16 02:45:57 +01:00
}
// Migrates all previous versions to latest version of `format.json`,
// this code calls migration in sequence, such as V1 is migrated to V2
// first before it V2 migrates to V3.
func formatXLMigrate ( export string ) error {
formatPath := pathJoin ( export , minioMetaBucket , formatConfigFile )
2019-12-24 01:31:03 +01:00
version , err := formatGetBackendXLVersion ( formatPath )
2018-02-16 02:45:57 +01:00
if err != nil {
return err
}
switch version {
case formatXLVersionV1 :
2019-12-24 01:31:03 +01:00
if err = formatXLMigrateV1ToV2 ( export , version ) ; err != nil {
2018-02-16 02:45:57 +01:00
return err
}
2019-12-24 01:31:03 +01:00
// Migrate successful v1 => v2, proceed to v2 => v3
version = formatXLVersionV2
2018-02-16 02:45:57 +01:00
fallthrough
case formatXLVersionV2 :
2019-12-24 01:31:03 +01:00
if err = formatXLMigrateV2ToV3 ( export , version ) ; err != nil {
2018-03-15 21:55:23 +01:00
return err
}
2019-12-24 01:31:03 +01:00
// Migrate successful v2 => v3, v3 is latest
version = formatXLVersionV3
2018-03-15 21:55:23 +01:00
fallthrough
case formatXLVersionV3 :
2019-12-24 01:31:03 +01:00
// v3 is the latest version, return.
2018-02-16 02:45:57 +01:00
return nil
}
return fmt . Errorf ( ` %s: unknown format version %s ` , export , version )
}
// Migrates version V1 of format.json to version V2 of format.json,
// migration fails upon any error.
2019-12-24 01:31:03 +01:00
func formatXLMigrateV1ToV2 ( export , version string ) error {
2018-02-16 02:45:57 +01:00
if version != formatXLVersionV1 {
return fmt . Errorf ( ` Disk %s: format version expected %s, found %s ` , export , formatXLVersionV1 , version )
}
2019-12-24 01:31:03 +01:00
formatPath := pathJoin ( export , minioMetaBucket , formatConfigFile )
2018-02-16 02:45:57 +01:00
formatV1 := & formatXLV1 { }
b , err := ioutil . ReadFile ( formatPath )
if err != nil {
return err
}
if err = json . Unmarshal ( b , formatV1 ) ; err != nil {
return err
}
2016-06-03 01:34:15 +02:00
2018-03-15 21:55:23 +01:00
formatV2 := & formatXLV2 { }
formatV2 . Version = formatMetaVersionV1
formatV2 . Format = formatBackendXL
formatV2 . XL . Version = formatXLVersionV2
formatV2 . XL . DistributionAlgo = formatXLVersionV2DistributionAlgo
2018-03-19 17:13:00 +01:00
formatV2 . XL . This = formatV1 . XL . Disk
2018-03-15 21:55:23 +01:00
formatV2 . XL . Sets = make ( [ ] [ ] string , 1 )
formatV2 . XL . Sets [ 0 ] = make ( [ ] string , len ( formatV1 . XL . JBOD ) )
2018-02-16 02:45:57 +01:00
copy ( formatV2 . XL . Sets [ 0 ] , formatV1 . XL . JBOD )
b , err = json . Marshal ( formatV2 )
if err != nil {
return err
}
return ioutil . WriteFile ( formatPath , b , 0644 )
}
2017-08-03 13:37:02 +02:00
2018-03-15 21:55:23 +01:00
// Migrates V2 for format.json to V3 (Flat hierarchy for multipart)
2019-12-24 01:31:03 +01:00
func formatXLMigrateV2ToV3 ( export , version string ) error {
2018-03-15 21:55:23 +01:00
if version != formatXLVersionV2 {
return fmt . Errorf ( ` Disk %s: format version expected %s, found %s ` , export , formatXLVersionV2 , version )
}
2019-12-24 01:31:03 +01:00
formatPath := pathJoin ( export , minioMetaBucket , formatConfigFile )
2018-03-15 21:55:23 +01:00
formatV2 := & formatXLV2 { }
b , err := ioutil . ReadFile ( formatPath )
if err != nil {
return err
}
err = json . Unmarshal ( b , formatV2 )
if err != nil {
return err
}
2018-08-06 06:15:28 +02:00
if err = removeAll ( pathJoin ( export , minioMetaMultipartBucket ) ) ; err != nil {
2018-03-15 21:55:23 +01:00
return err
}
2018-08-06 06:15:28 +02:00
if err = mkdirAll ( pathJoin ( export , minioMetaMultipartBucket ) , 0755 ) ; err != nil {
2018-03-15 21:55:23 +01:00
return err
}
// format-V2 struct is exactly same as format-V1 except that version is "3"
// which indicates the simplified multipart backend.
formatV3 := formatXLV3 { }
formatV3 . Version = formatV2 . Version
formatV3 . Format = formatV2 . Format
formatV3 . XL = formatV2 . XL
formatV3 . XL . Version = formatXLVersionV3
b , err = json . Marshal ( formatV3 )
if err != nil {
return err
}
return ioutil . WriteFile ( formatPath , b , 0644 )
}
2018-02-16 02:45:57 +01:00
// countErrs - count a specific error.
func countErrs ( errs [ ] error , err error ) int {
var i = 0
for _ , err1 := range errs {
2018-04-10 18:36:37 +02:00
if err1 == err {
2018-02-16 02:45:57 +01:00
i ++
}
}
return i
2016-06-03 01:34:15 +02:00
}
2018-02-16 02:45:57 +01:00
// Does all errors indicate we need to initialize all disks?.
func shouldInitXLDisks ( errs [ ] error ) bool {
return countErrs ( errs , errUnformattedDisk ) == len ( errs )
}
2018-04-13 00:43:38 +02:00
// Check if unformatted disks are equal to write quorum.
func quorumUnformattedDisks ( errs [ ] error ) bool {
return countErrs ( errs , errUnformattedDisk ) >= ( len ( errs ) / 2 ) + 1
}
2018-02-16 02:45:57 +01:00
// loadFormatXLAll - load all format config from all input disks in parallel.
2020-03-27 22:48:30 +01:00
func loadFormatXLAll ( storageDisks [ ] StorageAPI , heal bool ) ( [ ] * formatXLV3 , [ ] error ) {
2016-06-03 01:34:15 +02:00
// Initialize list of errors.
2019-10-14 18:44:51 +02:00
g := errgroup . WithNErrs ( len ( storageDisks ) )
2016-06-03 01:34:15 +02:00
// Initialize format configs.
2018-04-04 06:58:48 +02:00
var formats = make ( [ ] * formatXLV3 , len ( storageDisks ) )
2016-06-03 01:34:15 +02:00
2018-01-22 23:54:55 +01:00
// Load format from each disk in parallel
2019-10-14 18:44:51 +02:00
for index := range storageDisks {
index := index
g . Go ( func ( ) error {
if storageDisks [ index ] == nil {
return errDiskNotFound
}
format , err := loadFormatXL ( storageDisks [ index ] )
if err != nil {
return err
2016-06-03 01:34:15 +02:00
}
2018-01-08 23:30:55 +01:00
formats [ index ] = format
2020-03-27 22:48:30 +01:00
if ! heal {
// If no healing required, make the disks valid and
// online.
storageDisks [ index ] . SetDiskID ( format . XL . This )
}
2019-10-14 18:44:51 +02:00
return nil
} , index )
2016-06-03 01:34:15 +02:00
}
2019-10-14 18:44:51 +02:00
// Return all formats and errors if any.
return formats , g . Wait ( )
2016-06-03 01:34:15 +02:00
}
2020-03-27 22:48:30 +01:00
func saveFormatXL ( disk StorageAPI , format interface { } , diskID string ) error {
if format == nil || disk == nil {
return errDiskNotFound
}
if err := makeFormatXLMetaVolumes ( disk ) ; err != nil {
return err
}
2018-02-16 02:45:57 +01:00
// Marshal and write to disk.
formatBytes , err := json . Marshal ( format )
if err != nil {
return err
2017-08-03 13:37:02 +02:00
}
2019-12-24 01:31:03 +01:00
tmpFormat := mustGetUUID ( )
2019-09-25 03:47:26 +02:00
2018-02-16 02:45:57 +01:00
// Purge any existing temporary file, okay to ignore errors here.
2019-12-24 01:31:03 +01:00
defer disk . DeleteFile ( minioMetaBucket , tmpFormat )
2017-08-03 13:37:02 +02:00
2019-12-24 01:31:03 +01:00
// write to unique file.
if err = disk . WriteAll ( minioMetaBucket , tmpFormat , bytes . NewReader ( formatBytes ) ) ; err != nil {
2018-02-16 02:45:57 +01:00
return err
2017-08-03 13:37:02 +02:00
}
2019-09-25 03:47:26 +02:00
// Rename file `uuid.json` --> `format.json`.
2020-03-27 22:48:30 +01:00
if err = disk . RenameFile ( minioMetaBucket , tmpFormat , minioMetaBucket , formatConfigFile ) ; err != nil {
return err
}
disk . SetDiskID ( diskID )
return nil
2016-05-20 11:22:22 +02:00
}
2020-04-07 01:51:18 +02:00
var ignoredHiddenDirectories = map [ string ] struct { } {
minioMetaBucket : { } ,
".snapshot" : { } ,
"lost+found" : { } ,
"$RECYCLE.BIN" : { } ,
"System Volume Information" : { } ,
2018-11-02 19:31:55 +01:00
}
2020-04-07 01:51:18 +02:00
func isHiddenDirectories ( vols ... VolInfo ) bool {
for _ , vol := range vols {
if _ , ok := ignoredHiddenDirectories [ vol . Name ] ; ok {
continue
2018-11-02 19:31:55 +01:00
}
2020-04-07 01:51:18 +02:00
return false
2018-11-02 19:31:55 +01:00
}
2020-04-07 01:51:18 +02:00
return true
2018-11-02 19:31:55 +01:00
}
2018-02-16 02:45:57 +01:00
// loadFormatXL - loads format.json from disk.
2018-03-15 21:55:23 +01:00
func loadFormatXL ( disk StorageAPI ) ( format * formatXLV3 , err error ) {
2016-06-25 23:51:06 +02:00
buf , err := disk . ReadAll ( minioMetaBucket , formatConfigFile )
if err != nil {
2016-05-20 11:22:22 +02:00
// 'file not found' and 'volume not found' as
// same. 'volume not found' usually means its a fresh disk.
if err == errFileNotFound || err == errVolumeNotFound {
var vols [ ] VolInfo
vols , err = disk . ListVols ( )
if err != nil {
return nil , err
}
2020-04-07 01:51:18 +02:00
if ! isHiddenDirectories ( vols ... ) {
// 'format.json' not found, but we found user data, reject such disks.
2016-05-20 11:22:22 +02:00
return nil , errCorruptedFormat
}
// No other data found, its a fresh disk.
return nil , errUnformattedDisk
}
2016-05-07 09:59:43 +02:00
return nil , err
2016-05-04 21:18:20 +02:00
}
2016-06-24 11:06:23 +02:00
// Try to decode format json into formatConfigV1 struct.
2018-03-15 21:55:23 +01:00
format = & formatXLV3 { }
2016-06-25 23:51:06 +02:00
if err = json . Unmarshal ( buf , format ) ; err != nil {
2016-05-07 09:59:43 +02:00
return nil , err
2016-05-04 21:18:20 +02:00
}
2016-06-24 11:06:23 +02:00
// Success.
2016-05-20 11:22:22 +02:00
return format , nil
}
2018-02-16 02:45:57 +01:00
// Valid formatXL basic versions.
2018-03-15 21:55:23 +01:00
func checkFormatXLValue ( formatXL * formatXLV3 ) error {
2018-02-16 02:45:57 +01:00
// Validate format version and format type.
if formatXL . Version != formatMetaVersionV1 {
return fmt . Errorf ( "Unsupported version of backend format [%s] found" , formatXL . Version )
2016-07-26 12:18:47 +02:00
}
2018-02-16 02:45:57 +01:00
if formatXL . Format != formatBackendXL {
return fmt . Errorf ( "Unsupported backend format [%s] found" , formatXL . Format )
2017-08-03 13:37:02 +02:00
}
2018-03-15 21:55:23 +01:00
if formatXL . XL . Version != formatXLVersionV3 {
2018-02-16 02:45:57 +01:00
return fmt . Errorf ( "Unsupported XL backend format found [%s]" , formatXL . XL . Version )
2018-01-22 23:54:55 +01:00
}
return nil
2017-08-03 13:37:02 +02:00
}
2018-02-16 02:45:57 +01:00
// Check all format values.
2020-03-08 21:30:25 +01:00
func checkFormatXLValues ( formats [ ] * formatXLV3 , drivesPerSet int ) error {
2018-02-21 03:42:09 +01:00
for i , formatXL := range formats {
2018-02-16 02:45:57 +01:00
if formatXL == nil {
continue
}
if err := checkFormatXLValue ( formatXL ) ; err != nil {
return err
}
if len ( formats ) != len ( formatXL . XL . Sets ) * len ( formatXL . XL . Sets [ 0 ] ) {
2018-02-21 03:42:09 +01:00
return fmt . Errorf ( "%s disk is already being used in another erasure deployment. (Number of disks specified: %d but the number of disks found in the %s disk's format.json: %d)" ,
humanize . Ordinal ( i + 1 ) , len ( formats ) , humanize . Ordinal ( i + 1 ) , len ( formatXL . XL . Sets ) * len ( formatXL . XL . Sets [ 0 ] ) )
2018-02-16 02:45:57 +01:00
}
2020-04-27 23:39:57 +02:00
// Only if custom erasure drive count is set,
// we should fail here other proceed to honor what
// is present on the disk.
if globalCustomErasureDriveCount && len ( formatXL . XL . Sets [ 0 ] ) != drivesPerSet {
2020-03-08 21:30:25 +01:00
return fmt . Errorf ( "%s disk is already formatted with %d drives per erasure set. This cannot be changed to %d, please revert your MINIO_ERASURE_SET_DRIVE_COUNT setting" , humanize . Ordinal ( i + 1 ) , len ( formatXL . XL . Sets [ 0 ] ) , drivesPerSet )
}
2017-08-03 13:37:02 +02:00
}
2018-02-16 02:45:57 +01:00
return nil
}
2017-08-03 13:37:02 +02:00
2018-07-19 05:17:35 +02:00
// Get Deployment ID for the XL sets from format.json.
// This need not be in quorum. Even if one of the format.json
// file has this value, we assume it is valid.
// If more than one format.json's have different id, it is considered a corrupt
// backend format.
func formatXLGetDeploymentID ( refFormat * formatXLV3 , formats [ ] * formatXLV3 ) ( string , error ) {
var deploymentID string
for _ , format := range formats {
if format == nil || format . ID == "" {
continue
}
if reflect . DeepEqual ( format . XL . Sets , refFormat . XL . Sets ) {
// Found an ID in one of the format.json file
// Set deploymentID for the first time.
if deploymentID == "" {
deploymentID = format . ID
} else if deploymentID != format . ID {
// DeploymentID found earlier doesn't match with the
// current format.json's ID.
return "" , errCorruptedFormat
}
}
}
return deploymentID , nil
}
// formatXLFixDeploymentID - Add deployment id if it is not present.
2019-11-20 02:42:27 +01:00
func formatXLFixDeploymentID ( endpoints Endpoints , storageDisks [ ] StorageAPI , refFormat * formatXLV3 ) ( err error ) {
2018-07-19 05:17:35 +02:00
// Attempt to load all `format.json` from all disks.
var sErrs [ ] error
2020-03-27 22:48:30 +01:00
formats , sErrs := loadFormatXLAll ( storageDisks , false )
2018-07-19 05:17:35 +02:00
for i , sErr := range sErrs {
if _ , ok := formatCriticalErrors [ sErr ] ; ok {
2019-12-02 18:28:01 +01:00
return fmt . Errorf ( "Disk %s: %w" , endpoints [ i ] , sErr )
2018-07-19 05:17:35 +02:00
}
}
for index := range formats {
// If the XL sets do not match, set those formats to nil,
// We do not have to update the ID on those format.json file.
if formats [ index ] != nil && ! reflect . DeepEqual ( formats [ index ] . XL . Sets , refFormat . XL . Sets ) {
formats [ index ] = nil
}
}
refFormat . ID , err = formatXLGetDeploymentID ( refFormat , formats )
if err != nil {
return err
}
// If ID is set, then some other node got the lock
// before this node could and generated an ID
// for the deployment. No need to generate one.
if refFormat . ID != "" {
return nil
}
// ID is generated for the first time,
// We set the ID in all the formats and update.
refFormat . ID = mustGetUUID ( )
for _ , format := range formats {
if format != nil {
format . ID = refFormat . ID
}
}
// Deployment ID needs to be set on all the disks.
// Save `format.json` across all disks.
2020-04-09 18:30:02 +02:00
return saveFormatXLAll ( GlobalContext , storageDisks , formats )
2018-07-19 05:17:35 +02:00
}
// Update only the valid local disks which have not been updated before.
2019-11-20 02:42:27 +01:00
func formatXLFixLocalDeploymentID ( endpoints Endpoints , storageDisks [ ] StorageAPI , refFormat * formatXLV3 ) error {
2018-07-19 05:17:35 +02:00
// If this server was down when the deploymentID was updated
// then we make sure that we update the local disks with the deploymentID.
2020-03-27 22:48:30 +01:00
// Initialize errs to collect errors inside go-routine.
g := errgroup . WithNErrs ( len ( storageDisks ) )
for index := range storageDisks {
index := index
g . Go ( func ( ) error {
if endpoints [ index ] . IsLocal && storageDisks [ index ] != nil && storageDisks [ index ] . IsOnline ( ) {
format , err := loadFormatXL ( storageDisks [ index ] )
if err != nil {
// Disk can be offline etc.
// ignore the errors seen here.
return nil
}
if format . ID != "" {
return nil
}
if ! reflect . DeepEqual ( format . XL . Sets , refFormat . XL . Sets ) {
return nil
}
format . ID = refFormat . ID
if err := saveFormatXL ( storageDisks [ index ] , format , format . XL . This ) ; err != nil {
2020-04-09 18:30:02 +02:00
logger . LogIf ( GlobalContext , err )
2020-03-27 22:48:30 +01:00
return fmt . Errorf ( "Unable to save format.json, %w" , err )
}
2018-07-19 05:17:35 +02:00
}
2020-03-27 22:48:30 +01:00
return nil
} , index )
}
for _ , err := range g . Wait ( ) {
if err != nil {
return err
2018-07-19 05:17:35 +02:00
}
}
return nil
}
2018-02-16 02:45:57 +01:00
// Get backend XL format in quorum `format.json`.
2018-03-15 21:55:23 +01:00
func getFormatXLInQuorum ( formats [ ] * formatXLV3 ) ( * formatXLV3 , error ) {
2018-02-16 02:45:57 +01:00
formatHashes := make ( [ ] string , len ( formats ) )
for i , format := range formats {
if format == nil {
2018-01-22 23:54:55 +01:00
continue
}
2018-02-16 02:45:57 +01:00
h := sha256 . New ( )
for _ , set := range format . XL . Sets {
for _ , diskID := range set {
h . Write ( [ ] byte ( diskID ) )
2016-06-23 02:18:31 +02:00
}
2016-06-02 01:15:56 +02:00
}
2018-02-16 02:45:57 +01:00
formatHashes [ i ] = hex . EncodeToString ( h . Sum ( nil ) )
2016-06-02 01:15:56 +02:00
}
2016-07-26 12:18:47 +02:00
2018-02-16 02:45:57 +01:00
formatCountMap := make ( map [ string ] int )
for _ , hash := range formatHashes {
if hash == "" {
continue
}
formatCountMap [ hash ] ++
}
2016-06-02 01:15:56 +02:00
2018-02-16 02:45:57 +01:00
maxHash := ""
maxCount := 0
for hash , count := range formatCountMap {
if count > maxCount {
maxCount = count
maxHash = hash
2016-07-29 01:49:59 +02:00
}
2018-02-16 02:45:57 +01:00
}
if maxCount < len ( formats ) / 2 {
return nil , errXLReadQuorum
}
for i , hash := range formatHashes {
if hash == maxHash {
2019-11-21 13:24:51 +01:00
format := formats [ i ] . Clone ( )
2018-02-16 02:45:57 +01:00
format . XL . This = ""
2019-11-21 13:24:51 +01:00
return format , nil
2016-07-29 01:49:59 +02:00
}
}
2018-02-16 02:45:57 +01:00
return nil , errXLReadQuorum
2016-07-29 01:49:59 +02:00
}
2018-03-15 21:55:23 +01:00
func formatXLV3Check ( reference * formatXLV3 , format * formatXLV3 ) error {
2019-11-21 13:24:51 +01:00
tmpFormat := format . Clone ( )
2018-02-16 02:45:57 +01:00
this := tmpFormat . XL . This
tmpFormat . XL . This = ""
if len ( reference . XL . Sets ) != len ( format . XL . Sets ) {
return fmt . Errorf ( "Expected number of sets %d, got %d" , len ( reference . XL . Sets ) , len ( format . XL . Sets ) )
}
2017-08-03 13:37:02 +02:00
2018-02-16 02:45:57 +01:00
// Make sure that the sets match.
for i := range reference . XL . Sets {
if len ( reference . XL . Sets [ i ] ) != len ( format . XL . Sets [ i ] ) {
return fmt . Errorf ( "Each set should be of same size, expected %d got %d" ,
len ( reference . XL . Sets [ i ] ) , len ( format . XL . Sets [ i ] ) )
2016-07-29 01:49:59 +02:00
}
2018-02-16 02:45:57 +01:00
for j := range reference . XL . Sets [ i ] {
if reference . XL . Sets [ i ] [ j ] != format . XL . Sets [ i ] [ j ] {
return fmt . Errorf ( "UUID on positions %d:%d do not match with, expected %s got %s" ,
i , j , reference . XL . Sets [ i ] [ j ] , format . XL . Sets [ i ] [ j ] )
2016-07-29 01:49:59 +02:00
}
}
2018-02-16 02:45:57 +01:00
}
// Make sure that the diskID is found in the set.
for i := 0 ; i < len ( tmpFormat . XL . Sets ) ; i ++ {
for j := 0 ; j < len ( tmpFormat . XL . Sets [ i ] ) ; j ++ {
if this == tmpFormat . XL . Sets [ i ] [ j ] {
return nil
2016-07-29 01:49:59 +02:00
}
}
}
2018-02-16 02:45:57 +01:00
return fmt . Errorf ( "Disk ID %s not found in any disk sets %s" , this , format . XL . Sets )
2016-07-29 01:49:59 +02:00
}
2020-01-15 21:36:52 +01:00
// Initializes meta volume only on local storage disks.
func initXLMetaVolumesInLocalDisks ( storageDisks [ ] StorageAPI , formats [ ] * formatXLV3 ) error {
// Compute the local disks eligible for meta volumes (re)initialization
var disksToInit [ ] StorageAPI
for index := range storageDisks {
if formats [ index ] == nil || storageDisks [ index ] == nil || storageDisks [ index ] . Hostname ( ) != "" {
// Ignore create meta volume on disks which are not found or not local.
continue
}
disksToInit = append ( disksToInit , storageDisks [ index ] )
}
2019-12-24 01:31:03 +01:00
// Initialize errs to collect errors inside go-routine.
2020-01-15 21:36:52 +01:00
g := errgroup . WithNErrs ( len ( disksToInit ) )
2019-12-24 01:31:03 +01:00
// Initialize all disks in parallel.
2020-01-15 21:36:52 +01:00
for index := range disksToInit {
// Initialize a new index variable in each loop so each
// goroutine will return its own instance of index variable.
2019-12-24 01:31:03 +01:00
index := index
g . Go ( func ( ) error {
2020-02-06 22:12:36 +01:00
return makeFormatXLMetaVolumes ( disksToInit [ index ] )
2019-12-24 01:31:03 +01:00
} , index )
}
// Return upon first error.
for _ , err := range g . Wait ( ) {
if err == nil {
continue
}
return toObjectErr ( err , minioMetaBucket )
}
// Return success here.
return nil
}
2018-02-16 02:45:57 +01:00
// saveFormatXLAll - populates `format.json` on disks in its order.
2018-04-06 00:04:40 +02:00
func saveFormatXLAll ( ctx context . Context , storageDisks [ ] StorageAPI , formats [ ] * formatXLV3 ) error {
2019-10-14 18:44:51 +02:00
g := errgroup . WithNErrs ( len ( storageDisks ) )
2016-07-29 01:49:59 +02:00
2018-02-16 02:45:57 +01:00
// Write `format.json` to all disks.
2019-10-14 18:44:51 +02:00
for index := range storageDisks {
index := index
g . Go ( func ( ) error {
2020-03-27 22:48:30 +01:00
return saveFormatXL ( storageDisks [ index ] , formats [ index ] , formats [ index ] . XL . This )
2019-10-14 18:44:51 +02:00
} , index )
2016-07-29 01:49:59 +02:00
}
2020-03-31 18:32:16 +02:00
writeQuorum := getWriteQuorum ( len ( storageDisks ) )
2019-10-14 18:44:51 +02:00
// Wait for the routines to finish.
return reduceWriteQuorumErrs ( ctx , g . Wait ( ) , nil , writeQuorum )
2018-02-16 02:45:57 +01:00
}
2018-04-04 06:58:48 +02:00
// relinquishes the underlying connection for all storage disks.
func closeStorageDisks ( storageDisks [ ] StorageAPI ) {
for _ , disk := range storageDisks {
if disk == nil {
continue
}
disk . Close ( )
}
}
2019-09-28 01:47:12 +02:00
// Initialize storage disks for each endpoint.
// Errors are returned for each endpoint with matching index.
2019-11-20 02:42:27 +01:00
func initStorageDisksWithErrors ( endpoints Endpoints ) ( [ ] StorageAPI , [ ] error ) {
2018-02-16 02:45:57 +01:00
// Bootstrap disks.
storageDisks := make ( [ ] StorageAPI , len ( endpoints ) )
2019-10-14 18:44:51 +02:00
g := errgroup . WithNErrs ( len ( endpoints ) )
for index := range endpoints {
index := index
g . Go ( func ( ) error {
storageDisk , err := newStorageAPI ( endpoints [ index ] )
if err != nil {
return err
}
storageDisks [ index ] = storageDisk
return nil
} , index )
2019-08-02 21:17:26 +02:00
}
2019-10-14 18:44:51 +02:00
return storageDisks , g . Wait ( )
2019-08-02 21:17:26 +02:00
}
2018-03-19 17:13:00 +01:00
// formatXLV3ThisEmpty - find out if '.This' field is empty
// in any of the input `formats`, if yes return true.
func formatXLV3ThisEmpty ( formats [ ] * formatXLV3 ) bool {
for _ , format := range formats {
if format == nil {
continue
}
// NOTE: This code is specifically needed when migrating version
// V1 to V2 to V3, in a scenario such as this we only need to handle
// single sets since we never used to support multiple sets in releases
// with V1 format version.
if len ( format . XL . Sets ) > 1 {
continue
}
if format . XL . This == "" {
return true
}
}
return false
}
// fixFormatXLV3 - fix format XL configuration on all disks.
2019-11-20 02:42:27 +01:00
func fixFormatXLV3 ( storageDisks [ ] StorageAPI , endpoints Endpoints , formats [ ] * formatXLV3 ) error {
2020-03-27 22:48:30 +01:00
g := errgroup . WithNErrs ( len ( formats ) )
for i := range formats {
i := i
g . Go ( func ( ) error {
if formats [ i ] == nil || ! endpoints [ i ] . IsLocal {
return nil
}
// NOTE: This code is specifically needed when migrating version
// V1 to V2 to V3, in a scenario such as this we only need to handle
// single sets since we never used to support multiple sets in releases
// with V1 format version.
if len ( formats [ i ] . XL . Sets ) > 1 {
return nil
}
if formats [ i ] . XL . This == "" {
formats [ i ] . XL . This = formats [ i ] . XL . Sets [ 0 ] [ i ]
if err := saveFormatXL ( storageDisks [ i ] , formats [ i ] , formats [ i ] . XL . This ) ; err != nil {
return err
}
2018-03-19 17:13:00 +01:00
}
2020-03-27 22:48:30 +01:00
return nil
} , i )
}
for _ , err := range g . Wait ( ) {
if err != nil {
return err
2018-03-19 17:13:00 +01:00
}
}
return nil
2020-03-27 22:48:30 +01:00
2018-03-19 17:13:00 +01:00
}
2018-02-16 02:45:57 +01:00
// initFormatXL - save XL format configuration on all disks.
2019-11-21 13:24:51 +01:00
func initFormatXL ( ctx context . Context , storageDisks [ ] StorageAPI , setCount , drivesPerSet int , deploymentID string ) ( * formatXLV3 , error ) {
format := newFormatXLV3 ( setCount , drivesPerSet )
2018-04-04 06:58:48 +02:00
formats := make ( [ ] * formatXLV3 , len ( storageDisks ) )
2020-01-13 22:09:10 +01:00
wantAtMost := ecDrivesNoConfig ( drivesPerSet )
2016-05-04 21:18:20 +02:00
2018-02-16 02:45:57 +01:00
for i := 0 ; i < setCount ; i ++ {
2020-01-13 22:09:10 +01:00
hostCount := make ( map [ string ] int , drivesPerSet )
2019-11-21 13:24:51 +01:00
for j := 0 ; j < drivesPerSet ; j ++ {
2020-01-13 22:09:10 +01:00
disk := storageDisks [ i * drivesPerSet + j ]
2019-11-21 13:24:51 +01:00
newFormat := format . Clone ( )
2018-02-16 02:45:57 +01:00
newFormat . XL . This = format . XL . Sets [ i ] [ j ]
2019-11-20 11:09:30 +01:00
if deploymentID != "" {
newFormat . ID = deploymentID
}
2020-01-13 22:09:10 +01:00
hostCount [ disk . Hostname ( ) ] ++
2019-11-21 13:24:51 +01:00
formats [ i * drivesPerSet + j ] = newFormat
2018-02-16 02:45:57 +01:00
}
2020-01-13 22:09:10 +01:00
if len ( hostCount ) > 0 {
var once sync . Once
for host , count := range hostCount {
if count > wantAtMost {
if host == "" {
host = "local"
}
once . Do ( func ( ) {
if len ( hostCount ) == 1 {
return
}
logger . Info ( " * Set %v:" , i + 1 )
for j := 0 ; j < drivesPerSet ; j ++ {
disk := storageDisks [ i * drivesPerSet + j ]
logger . Info ( " - Drive: %s" , disk . String ( ) )
}
} )
logger . Info ( color . Yellow ( "WARNING:" ) + " Host %v has more than %v drives of set. " +
"A host failure will result in data becoming unavailable." , host , wantAtMost )
}
}
}
2016-05-04 21:18:20 +02:00
}
2018-02-16 02:45:57 +01:00
// Save formats `format.json` across all disks.
2019-11-21 13:24:51 +01:00
if err := saveFormatXLAll ( ctx , storageDisks , formats ) ; err != nil {
2018-02-16 02:45:57 +01:00
return nil , err
2017-04-18 19:35:17 +02:00
}
2018-02-16 02:45:57 +01:00
2019-11-21 13:24:51 +01:00
return getFormatXLInQuorum ( formats )
2017-04-18 19:35:17 +02:00
}
2020-01-13 22:09:10 +01:00
// ecDrivesNoConfig returns the erasure coded drives in a set if no config has been set.
// It will attempt to read it from env variable and fall back to drives/2.
func ecDrivesNoConfig ( drivesPerSet int ) int {
ecDrives := globalStorageClass . GetParityForSC ( storageclass . STANDARD )
if ecDrives == 0 {
cfg , err := storageclass . LookupConfig ( nil , drivesPerSet )
if err == nil {
ecDrives = cfg . Standard . Parity
}
if ecDrives == 0 {
ecDrives = drivesPerSet / 2
}
}
return ecDrives
}
2018-02-16 02:45:57 +01:00
// Make XL backend meta volumes.
func makeFormatXLMetaVolumes ( disk StorageAPI ) error {
2020-03-27 22:48:30 +01:00
if disk == nil {
return errDiskNotFound
}
2019-12-24 01:31:03 +01:00
// Attempt to create MinIO internal buckets.
2020-03-19 00:19:29 +01:00
return disk . MakeVolBulk ( minioMetaBucket , minioMetaTmpBucket , minioMetaMultipartBucket , dataUsageBucket )
2016-12-12 00:18:55 +01:00
}
2018-02-16 02:45:57 +01:00
var initMetaVolIgnoredErrs = append ( baseIgnoredErrs , errVolumeExists )
// Get all UUIDs which are present in reference format should
// be present in the list of formats provided, those are considered
// as online UUIDs.
2018-03-15 21:55:23 +01:00
func getOnlineUUIDs ( refFormat * formatXLV3 , formats [ ] * formatXLV3 ) ( onlineUUIDs [ ] string ) {
2018-02-16 02:45:57 +01:00
for _ , format := range formats {
if format == nil {
continue
}
for _ , set := range refFormat . XL . Sets {
for _ , uuid := range set {
if format . XL . This == uuid {
onlineUUIDs = append ( onlineUUIDs , uuid )
}
}
}
2016-05-07 09:59:43 +02:00
}
2018-02-16 02:45:57 +01:00
return onlineUUIDs
}
2016-06-23 02:18:31 +02:00
2018-02-16 02:45:57 +01:00
// Look for all UUIDs which are not present in reference format
// but are present in the onlineUUIDs list, construct of list such
// offline UUIDs.
2018-03-15 21:55:23 +01:00
func getOfflineUUIDs ( refFormat * formatXLV3 , formats [ ] * formatXLV3 ) ( offlineUUIDs [ ] string ) {
2018-02-16 02:45:57 +01:00
onlineUUIDs := getOnlineUUIDs ( refFormat , formats )
for i , set := range refFormat . XL . Sets {
for j , uuid := range set {
var found bool
for _ , onlineUUID := range onlineUUIDs {
if refFormat . XL . Sets [ i ] [ j ] == onlineUUID {
found = true
}
}
if ! found {
offlineUUIDs = append ( offlineUUIDs , uuid )
}
}
2016-05-04 21:18:20 +02:00
}
2018-02-16 02:45:57 +01:00
return offlineUUIDs
2018-01-08 23:30:55 +01:00
}
2018-02-16 02:45:57 +01:00
// Mark all UUIDs that are offline.
2018-03-15 21:55:23 +01:00
func markUUIDsOffline ( refFormat * formatXLV3 , formats [ ] * formatXLV3 ) {
2018-02-16 02:45:57 +01:00
offlineUUIDs := getOfflineUUIDs ( refFormat , formats )
for i , set := range refFormat . XL . Sets {
for j := range set {
for _ , offlineUUID := range offlineUUIDs {
if refFormat . XL . Sets [ i ] [ j ] == offlineUUID {
refFormat . XL . Sets [ i ] [ j ] = offlineDiskUUID
}
}
}
2016-08-31 04:22:27 +02:00
}
2018-02-16 02:45:57 +01:00
}
2016-08-31 04:22:27 +02:00
2018-02-16 02:45:57 +01:00
// Initialize a new set of set formats which will be written to all disks.
2019-11-21 13:24:51 +01:00
func newHealFormatSets ( refFormat * formatXLV3 , setCount , drivesPerSet int , formats [ ] * formatXLV3 , errs [ ] error ) [ ] [ ] * formatXLV3 {
2018-03-15 21:55:23 +01:00
newFormats := make ( [ ] [ ] * formatXLV3 , setCount )
2018-02-16 02:45:57 +01:00
for i := range refFormat . XL . Sets {
2019-11-21 13:24:51 +01:00
newFormats [ i ] = make ( [ ] * formatXLV3 , drivesPerSet )
2018-02-16 02:45:57 +01:00
}
for i := range refFormat . XL . Sets {
for j := range refFormat . XL . Sets [ i ] {
2019-11-21 13:24:51 +01:00
if errs [ i * drivesPerSet + j ] == errUnformattedDisk || errs [ i * drivesPerSet + j ] == nil {
2018-03-15 21:55:23 +01:00
newFormats [ i ] [ j ] = & formatXLV3 { }
2018-02-16 02:45:57 +01:00
newFormats [ i ] [ j ] . Version = refFormat . Version
2019-01-23 03:32:06 +01:00
newFormats [ i ] [ j ] . ID = refFormat . ID
2018-02-16 02:45:57 +01:00
newFormats [ i ] [ j ] . Format = refFormat . Format
newFormats [ i ] [ j ] . XL . Version = refFormat . XL . Version
newFormats [ i ] [ j ] . XL . DistributionAlgo = refFormat . XL . DistributionAlgo
}
2019-11-21 13:24:51 +01:00
if errs [ i * drivesPerSet + j ] == errUnformattedDisk {
2018-02-16 02:45:57 +01:00
newFormats [ i ] [ j ] . XL . This = ""
newFormats [ i ] [ j ] . XL . Sets = nil
continue
}
2019-11-21 13:24:51 +01:00
if errs [ i * drivesPerSet + j ] == nil {
newFormats [ i ] [ j ] . XL . This = formats [ i * drivesPerSet + j ] . XL . This
2018-02-16 02:45:57 +01:00
newFormats [ i ] [ j ] . XL . Sets = nil
}
}
}
return newFormats
2016-05-04 21:18:20 +02:00
}