/* * Minio Cloud Storage, (C) 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "encoding/json" "errors" "fmt" "strings" "github.com/skyrings/skyring-common/tools/uuid" ) type fsFormat struct { Version string `json:"version"` } type xlFormat struct { Version string `json:"version"` Disk string `json:"disk"` JBOD []string `json:"jbod"` } type formatConfigV1 struct { Version string `json:"version"` Format string `json:"format"` FS *fsFormat `json:"fs,omitempty"` XL *xlFormat `json:"xl,omitempty"` } // checkJBODConsistency - validate xl jbod order if they are consistent. func checkJBODConsistency(formatConfigs []*formatConfigV1) error { var firstJBOD []string // Extract first valid JBOD. for _, format := range formatConfigs { if format == nil { continue } firstJBOD = format.XL.JBOD break } jbodStr := strings.Join(firstJBOD, ".") for _, format := range formatConfigs { if format == nil { continue } savedJBODStr := strings.Join(format.XL.JBOD, ".") if jbodStr != savedJBODStr { return errors.New("Inconsistent disks.") } } return nil } func findIndex(disk string, jbod []string) int { for index, uuid := range jbod { if uuid == disk { return index } } return -1 } // reorderDisks - reorder disks in JBOD order. func reorderDisks(bootstrapDisks []StorageAPI, formatConfigs []*formatConfigV1) ([]StorageAPI, error) { var savedJBOD []string for _, format := range formatConfigs { if format == nil { continue } savedJBOD = format.XL.JBOD break } // Pick the first JBOD list to verify the order and construct new set of disk slice. var newDisks = make([]StorageAPI, len(bootstrapDisks)) var unclaimedJBODIndex = make(map[int]struct{}) for fIndex, format := range formatConfigs { if format == nil { unclaimedJBODIndex[fIndex] = struct{}{} continue } jIndex := findIndex(format.XL.Disk, savedJBOD) if jIndex == -1 { return nil, errors.New("Unrecognized uuid " + format.XL.Disk + " found") } newDisks[jIndex] = bootstrapDisks[fIndex] } // Save the unclaimed jbods as well. for index, disk := range newDisks { if disk == nil { for fIndex := range unclaimedJBODIndex { newDisks[index] = bootstrapDisks[fIndex] delete(unclaimedJBODIndex, fIndex) break } continue } } return newDisks, nil } // loadFormat - load format from disk. func loadFormat(disk StorageAPI) (format *formatConfigV1, err error) { var buffer []byte buffer, err = readAll(disk, minioMetaBucket, formatConfigFile) if err != nil { // 'file not found' and 'volume not found' as // same. 'volume not found' usually means its a fresh disk. if err == errFileNotFound || err == errVolumeNotFound { var vols []VolInfo vols, err = disk.ListVols() if err != nil { return nil, err } if len(vols) > 1 { // 'format.json' not found, but we found user data. return nil, errCorruptedFormat } // No other data found, its a fresh disk. return nil, errUnformattedDisk } return nil, err } format = &formatConfigV1{} err = json.Unmarshal(buffer, format) if err != nil { return nil, err } return format, nil } // Heals any missing format.json on the drives. Returns error only for unexpected errors // as regular errors can be ignored since there might be enough quorum to be operational. func healFormatXL(bootstrapDisks []StorageAPI) error { uuidUsage := make([]struct { uuid string // Disk uuid inuse bool // indicates if the uuid is used by any disk }, len(bootstrapDisks)) needHeal := make([]bool, len(bootstrapDisks)) // Slice indicating which drives needs healing. // Returns any unused drive UUID. getUnusedUUID := func() string { for index := range uuidUsage { if !uuidUsage[index].inuse { uuidUsage[index].inuse = true return uuidUsage[index].uuid } } return "" } formatConfigs := make([]*formatConfigV1, len(bootstrapDisks)) var referenceConfig *formatConfigV1 for index, disk := range bootstrapDisks { formatXL, err := loadFormat(disk) if err == errUnformattedDisk { // format.json is missing, should be healed. needHeal[index] = true continue } if err == nil { if referenceConfig == nil { // this config will be used to update the drives missing format.json referenceConfig = formatXL } formatConfigs[index] = formatXL } else { // Abort format.json healing if any one of the drives is not available because we don't // know if that drive is down permanently or temporarily. So we don't want to reuse // its uuid for any other disks. // Return nil so that operations can continue if quorum is available. return nil } } if referenceConfig == nil { // All disks are fresh, format.json will be written by initFormatXL() return nil } for index, diskUUID := range referenceConfig.XL.JBOD { uuidUsage[index].uuid = diskUUID uuidUsage[index].inuse = false } for _, config := range formatConfigs { if config == nil { continue } for index := range uuidUsage { if config.XL.Disk == uuidUsage[index].uuid { uuidUsage[index].inuse = true break } } } for index, heal := range needHeal { if !heal { // Previously we detected that heal is not needed on the disk. continue } config := &formatConfigV1{} *config = *referenceConfig config.XL.Disk = getUnusedUUID() if config.XL.Disk == "" { // getUnusedUUID() should have returned an unused uuid, if not return error. return errUnexpected } formatBytes, err := json.Marshal(config) if err != nil { return err } // Fresh disk without format.json _, _ = bootstrapDisks[index].AppendFile(minioMetaBucket, formatConfigFile, formatBytes) // Ignore any error from AppendFile() as quorum might still be there to be operational. } return nil } // loadFormatXL - load XL format.json. func loadFormatXL(bootstrapDisks []StorageAPI) (disks []StorageAPI, err error) { var unformattedDisksFoundCnt = 0 var diskNotFoundCount = 0 formatConfigs := make([]*formatConfigV1, len(bootstrapDisks)) // Heal missing format.json on the drives. if err = healFormatXL(bootstrapDisks); err != nil { // There was an unexpected unrecoverable error during healing. return } for index, disk := range bootstrapDisks { var formatXL *formatConfigV1 formatXL, err = loadFormat(disk) if err != nil { if err == errUnformattedDisk { unformattedDisksFoundCnt++ continue } else if err == errDiskNotFound { diskNotFoundCount++ continue } return nil, err } // Save valid formats. formatConfigs[index] = formatXL } // If all disks indicate that 'format.json' is not available // return 'errUnformattedDisk'. if unformattedDisksFoundCnt == len(bootstrapDisks) { return nil, errUnformattedDisk } else if diskNotFoundCount == len(bootstrapDisks) { return nil, errDiskNotFound } else if diskNotFoundCount > len(bootstrapDisks)-(len(bootstrapDisks)/2+1) { return nil, errReadQuorum } else if unformattedDisksFoundCnt > len(bootstrapDisks)-(len(bootstrapDisks)/2+1) { return nil, errReadQuorum } if err = checkFormatXL(formatConfigs); err != nil { return nil, err } // Erasure code requires disks to be presented in the same order each time. return reorderDisks(bootstrapDisks, formatConfigs) } // checkFormatXL - verifies if format.json format is intact. func checkFormatXL(formatConfigs []*formatConfigV1) error { for _, formatXL := range formatConfigs { if formatXL == nil { continue } // Validate format version and format type. if formatXL.Version != "1" { return fmt.Errorf("Unsupported version of backend format [%s] found.", formatXL.Version) } if formatXL.Format != "xl" { return fmt.Errorf("Unsupported backend format [%s] found.", formatXL.Format) } if formatXL.XL.Version != "1" { return fmt.Errorf("Unsupported XL backend format found [%s]", formatXL.XL.Version) } if len(formatConfigs) != len(formatXL.XL.JBOD) { return fmt.Errorf("Number of disks %d did not match the backend format %d", len(formatConfigs), len(formatXL.XL.JBOD)) } } return checkJBODConsistency(formatConfigs) } // initFormatXL - save XL format configuration on all disks. func initFormatXL(storageDisks []StorageAPI) (err error) { var ( jbod = make([]string, len(storageDisks)) formats = make([]*formatConfigV1, len(storageDisks)) saveFormatErrCnt = 0 ) for index, disk := range storageDisks { if err = disk.MakeVol(minioMetaBucket); err != nil { if err != errVolumeExists { saveFormatErrCnt++ // Check for write quorum. if saveFormatErrCnt <= len(storageDisks)-(len(storageDisks)/2+3) { continue } return errWriteQuorum } } var u *uuid.UUID u, err = uuid.New() if err != nil { saveFormatErrCnt++ // Check for write quorum. if saveFormatErrCnt <= len(storageDisks)-(len(storageDisks)/2+3) { continue } return err } formats[index] = &formatConfigV1{ Version: "1", Format: "xl", XL: &xlFormat{ Version: "1", Disk: u.String(), }, } jbod[index] = formats[index].XL.Disk } for index, disk := range storageDisks { formats[index].XL.JBOD = jbod formatBytes, err := json.Marshal(formats[index]) if err != nil { return err } n, err := disk.AppendFile(minioMetaBucket, formatConfigFile, formatBytes) if err != nil { return err } if n != int64(len(formatBytes)) { return errUnexpected } } return nil }