maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Heal only when atleast one of the disk is unformatted (#5866)

Browse source 
Current healing has an issue when disks are healed
even when they are offline without knowing if disk
is unformatted. This can lead to issues of pre-maturely
removing the disk from the set just because it was
temporarily offline.

There is an increasing number of `mc admin heal` usage
on a cron or regular basis. It is possible that if healing
code saw disk is offline it might prematurely take it down,
this causes availability issues.

Fixes #5826
This commit is contained in:
Harshavardhana 2018-04-30 20:37:39 -07:00 committed by Nitish Tiwari
parent e43cd0d5c8
commit 5f9041571f
2 changed files with 12 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
cmd/format-xl.go
View file

@ -288,10 +288,10 @@ func formatXLMigrateV2ToV3(export string) error {
return ioutil.WriteFile(formatPath, b, 0644)
}
// Returns true, if one of the errors is non-nil.
func hasAnyErrors(errs []error) bool {
// Returns true, if one of the errors is non-nil and is Unformatted disk.
func hasAnyErrorsUnformatted(errs []error) bool {
for _, err := range errs {
if err != nil {
if err != nil && err == errUnformattedDisk {
return true
}
}

16
cmd/xl-sets.go
View file

@ -998,8 +998,9 @@ func (s *xlSets) ReloadFormat(ctx context.Context, dryRun bool) (err error) {
return nil
}
// HealFormat - heals missing `format.json` on freshly or corrupted
// disks (missing format.json but does have erasure coded data in it).
// HealFormat - heals missing `format.json` on fresh unformatted disks.
// TODO: In future support corrupted disks missing format.json but has erasure
// coded data in it.
func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealResultItem, err error) {
// Acquire lock on format.json
formatLock := s.getHashedSet(formatConfigFile).nsMutex.NewNSLock(minioMetaBucket, formatConfigFile)
@ -1051,11 +1052,6 @@ func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealRe
}
}
// no errors found, no healing is required.
if !hasAnyErrors(sErrs) {
return res, errNoHealRequired
}
for index, sErr := range sErrs {
if sErr != nil {
// Look for acceptable heal errors, for any other
@ -1066,6 +1062,12 @@ func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealRe
}
}
if !hasAnyErrorsUnformatted(sErrs) {
// No unformatted disks found disks are either offline
// or online, no healing is required.
return res, errNoHealRequired
}
// All disks are unformatted, return quorum error.
if shouldInitXLDisks(sErrs) {
return res, errXLReadQuorum