2019-10-28 18:27:49 +01:00
|
|
|
/*
|
|
|
|
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
package cmd
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/minio/minio/cmd/logger"
|
|
|
|
"github.com/minio/minio/pkg/madmin"
|
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
|
|
|
bgHealingUUID = "0000-0000-0000-0000"
|
2020-07-10 18:26:21 +02:00
|
|
|
// sleep for an hour after a lock timeout
|
|
|
|
// before retrying to acquire lock again.
|
|
|
|
leaderLockTimeoutSleepInterval = time.Hour
|
|
|
|
// heal entire namespace once in 30 days
|
|
|
|
healInterval = 30 * 24 * time.Hour
|
2019-10-28 18:27:49 +01:00
|
|
|
)
|
|
|
|
|
2020-07-10 18:26:21 +02:00
|
|
|
var leaderLockTimeout = newDynamicTimeout(30*time.Second, time.Minute)
|
2019-10-28 18:27:49 +01:00
|
|
|
|
|
|
|
// NewBgHealSequence creates a background healing sequence
|
|
|
|
// operation which crawls all objects and heal them.
|
2020-06-29 22:07:26 +02:00
|
|
|
func newBgHealSequence() *healSequence {
|
2019-10-28 18:27:49 +01:00
|
|
|
|
|
|
|
reqInfo := &logger.ReqInfo{API: "BackgroundHeal"}
|
2020-05-04 07:35:40 +02:00
|
|
|
ctx, cancelCtx := context.WithCancel(logger.SetReqInfo(GlobalContext, reqInfo))
|
2019-10-28 18:27:49 +01:00
|
|
|
|
|
|
|
hs := madmin.HealOpts{
|
|
|
|
// Remove objects that do not have read-quorum
|
|
|
|
Remove: true,
|
|
|
|
ScanMode: madmin.HealNormalScan,
|
|
|
|
}
|
|
|
|
|
|
|
|
return &healSequence{
|
2020-04-01 21:14:00 +02:00
|
|
|
sourceCh: make(chan healSource),
|
2020-05-01 05:23:00 +02:00
|
|
|
respCh: make(chan healResult),
|
2019-10-28 18:27:49 +01:00
|
|
|
startTime: UTCNow(),
|
|
|
|
clientToken: bgHealingUUID,
|
|
|
|
settings: hs,
|
|
|
|
currentStatus: healSequenceStatus{
|
|
|
|
Summary: healNotStartedStatus,
|
|
|
|
HealSettings: hs,
|
|
|
|
},
|
2020-05-04 07:35:40 +02:00
|
|
|
cancelCtx: cancelCtx,
|
|
|
|
ctx: ctx,
|
|
|
|
reportProgress: false,
|
|
|
|
scannedItemsMap: make(map[madmin.HealItemType]int64),
|
|
|
|
healedItemsMap: make(map[madmin.HealItemType]int64),
|
|
|
|
healFailedItemsMap: make(map[string]int64),
|
2019-10-28 18:27:49 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func getLocalBackgroundHealStatus() madmin.BgHealState {
|
|
|
|
bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
|
|
|
|
if !ok {
|
|
|
|
return madmin.BgHealState{}
|
|
|
|
}
|
|
|
|
|
|
|
|
return madmin.BgHealState{
|
2020-03-25 06:40:45 +01:00
|
|
|
ScannedItemsCount: bgSeq.getScannedItemsCount(),
|
2019-10-28 18:27:49 +01:00
|
|
|
LastHealActivity: bgSeq.lastHealActivity,
|
2020-03-12 07:00:31 +01:00
|
|
|
NextHealRound: UTCNow().Add(durationToNextHealRound(bgSeq.lastHealActivity)),
|
2019-10-28 18:27:49 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// healErasureSet lists and heals all objects in a specific erasure set
|
2020-06-13 05:04:01 +02:00
|
|
|
func healErasureSet(ctx context.Context, setIndex int, xlObj *erasureObjects, drivesPerSet int) error {
|
2019-10-28 18:27:49 +01:00
|
|
|
buckets, err := xlObj.ListBuckets(ctx)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get background heal sequence to send elements to heal
|
|
|
|
var bgSeq *healSequence
|
|
|
|
var ok bool
|
|
|
|
for {
|
|
|
|
bgSeq, ok = globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
|
|
|
|
if ok {
|
|
|
|
break
|
|
|
|
}
|
2020-05-04 07:35:40 +02:00
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return nil
|
|
|
|
case <-time.After(time.Second):
|
|
|
|
continue
|
|
|
|
}
|
2019-10-28 18:27:49 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// Heal all buckets with all objects
|
|
|
|
for _, bucket := range buckets {
|
|
|
|
// Heal current bucket
|
2020-04-01 21:14:00 +02:00
|
|
|
bgSeq.sourceCh <- healSource{
|
2020-06-13 05:04:01 +02:00
|
|
|
bucket: bucket.Name,
|
2020-04-01 21:14:00 +02:00
|
|
|
}
|
2019-10-28 18:27:49 +01:00
|
|
|
|
2020-06-13 05:04:01 +02:00
|
|
|
var entryChs []FileInfoVersionsCh
|
2020-06-10 02:09:19 +02:00
|
|
|
for _, disk := range xlObj.getLoadBalancedDisks() {
|
|
|
|
if disk == nil {
|
|
|
|
// Disk can be offline
|
|
|
|
continue
|
|
|
|
}
|
2020-06-13 05:04:01 +02:00
|
|
|
|
|
|
|
entryCh, err := disk.WalkVersions(bucket.Name, "", "", true, ctx.Done())
|
2020-06-10 02:09:19 +02:00
|
|
|
if err != nil {
|
|
|
|
// Disk walk returned error, ignore it.
|
|
|
|
continue
|
|
|
|
}
|
2020-06-13 05:04:01 +02:00
|
|
|
|
|
|
|
entryChs = append(entryChs, FileInfoVersionsCh{
|
2020-06-10 02:09:19 +02:00
|
|
|
Ch: entryCh,
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
entriesValid := make([]bool, len(entryChs))
|
2020-06-13 05:04:01 +02:00
|
|
|
entries := make([]FileInfoVersions, len(entryChs))
|
2020-06-10 02:09:19 +02:00
|
|
|
|
|
|
|
for {
|
2020-06-13 05:04:01 +02:00
|
|
|
entry, quorumCount, ok := lexicallySortedEntryVersions(entryChs, entries, entriesValid)
|
2020-06-10 02:09:19 +02:00
|
|
|
if !ok {
|
2020-06-13 05:04:01 +02:00
|
|
|
break
|
2020-06-10 02:09:19 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
if quorumCount == drivesPerSet {
|
|
|
|
// Skip good entries.
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2020-06-13 05:04:01 +02:00
|
|
|
for _, version := range entry.Versions {
|
|
|
|
bgSeq.sourceCh <- healSource{
|
|
|
|
bucket: bucket.Name,
|
|
|
|
object: version.Name,
|
|
|
|
versionID: version.VersionID,
|
|
|
|
}
|
2020-04-01 21:14:00 +02:00
|
|
|
}
|
2019-10-28 18:27:49 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-04-01 21:14:00 +02:00
|
|
|
// deepHealObject heals given object path in deep to fix bitrot.
|
2020-06-13 05:04:01 +02:00
|
|
|
func deepHealObject(bucket, object, versionID string) {
|
2020-04-01 21:14:00 +02:00
|
|
|
// Get background heal sequence to send elements to heal
|
|
|
|
bgSeq, _ := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
|
|
|
|
|
|
|
|
bgSeq.sourceCh <- healSource{
|
2020-06-13 05:04:01 +02:00
|
|
|
bucket: bucket,
|
|
|
|
object: object,
|
|
|
|
versionID: versionID,
|
|
|
|
opts: &madmin.HealOpts{ScanMode: madmin.HealDeepScan},
|
2020-04-01 21:14:00 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-03-12 07:00:31 +01:00
|
|
|
// Returns the duration to the next background healing round
|
|
|
|
func durationToNextHealRound(lastHeal time.Time) time.Duration {
|
|
|
|
if lastHeal.IsZero() {
|
|
|
|
lastHeal = globalBootTime
|
|
|
|
}
|
|
|
|
|
|
|
|
d := lastHeal.Add(healInterval).Sub(UTCNow())
|
|
|
|
if d < 0 {
|
|
|
|
return time.Second
|
|
|
|
}
|
|
|
|
return d
|
|
|
|
}
|
|
|
|
|
2019-10-28 18:27:49 +01:00
|
|
|
// Healing leader will take the charge of healing all erasure sets
|
2020-06-13 05:04:01 +02:00
|
|
|
func execLeaderTasks(ctx context.Context, z *erasureZones) {
|
2020-03-25 06:40:45 +01:00
|
|
|
// So that we don't heal immediately, but after one month.
|
|
|
|
lastScanTime := UTCNow()
|
|
|
|
// Get background heal sequence to send elements to heal
|
|
|
|
var bgSeq *healSequence
|
|
|
|
var ok bool
|
|
|
|
for {
|
|
|
|
bgSeq, ok = globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
|
|
|
|
if ok {
|
|
|
|
break
|
|
|
|
}
|
2020-05-04 07:35:40 +02:00
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
|
|
|
case <-time.After(time.Second):
|
|
|
|
continue
|
|
|
|
}
|
2020-03-25 06:40:45 +01:00
|
|
|
}
|
2019-10-28 18:27:49 +01:00
|
|
|
for {
|
2020-03-22 20:16:36 +01:00
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
|
|
|
case <-time.NewTimer(durationToNextHealRound(lastScanTime)).C:
|
2020-03-25 06:40:45 +01:00
|
|
|
bgSeq.resetHealStatusCounters()
|
2020-03-22 20:16:36 +01:00
|
|
|
for _, zone := range z.zones {
|
|
|
|
// Heal set by set
|
|
|
|
for i, set := range zone.sets {
|
2020-06-10 02:09:19 +02:00
|
|
|
if err := healErasureSet(ctx, i, set, zone.drivesPerSet); err != nil {
|
2020-03-22 20:16:36 +01:00
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
continue
|
|
|
|
}
|
2019-11-20 02:42:27 +01:00
|
|
|
}
|
2019-10-28 18:27:49 +01:00
|
|
|
}
|
2020-03-22 20:16:36 +01:00
|
|
|
lastScanTime = UTCNow()
|
2019-10-28 18:27:49 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-03-22 20:16:36 +01:00
|
|
|
func startGlobalHeal(ctx context.Context, objAPI ObjectLayer) {
|
2020-06-13 05:04:01 +02:00
|
|
|
zones, ok := objAPI.(*erasureZones)
|
2019-10-28 18:27:49 +01:00
|
|
|
if !ok {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2020-03-22 20:16:36 +01:00
|
|
|
execLeaderTasks(ctx, zones)
|
2019-10-28 18:27:49 +01:00
|
|
|
}
|
|
|
|
|
2020-03-22 20:16:36 +01:00
|
|
|
func initGlobalHeal(ctx context.Context, objAPI ObjectLayer) {
|
|
|
|
go startGlobalHeal(ctx, objAPI)
|
2019-10-28 18:27:49 +01:00
|
|
|
}
|