2018-01-22 23:54:55 +01:00
|
|
|
/*
|
2019-04-09 20:39:42 +02:00
|
|
|
* MinIO Cloud Storage, (C) 2017 MinIO, Inc.
|
2018-01-22 23:54:55 +01:00
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
package cmd
|
|
|
|
|
|
|
|
import (
|
2018-03-14 20:01:47 +01:00
|
|
|
"context"
|
2018-01-22 23:54:55 +01:00
|
|
|
"encoding/json"
|
|
|
|
"fmt"
|
2018-11-05 04:24:16 +01:00
|
|
|
"net/http"
|
2018-01-22 23:54:55 +01:00
|
|
|
"strings"
|
|
|
|
"sync"
|
|
|
|
"time"
|
|
|
|
|
2018-03-14 20:01:47 +01:00
|
|
|
"github.com/minio/minio/cmd/logger"
|
2018-01-22 23:54:55 +01:00
|
|
|
"github.com/minio/minio/pkg/madmin"
|
|
|
|
)
|
|
|
|
|
|
|
|
// healStatusSummary - overall short summary of a healing sequence
|
|
|
|
type healStatusSummary string
|
|
|
|
|
|
|
|
// healStatusSummary constants
|
|
|
|
const (
|
|
|
|
healNotStartedStatus healStatusSummary = "not started"
|
|
|
|
healRunningStatus = "running"
|
|
|
|
healStoppedStatus = "stopped"
|
|
|
|
healFinishedStatus = "finished"
|
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
|
|
|
// a heal sequence with this many un-consumed heal result
|
|
|
|
// items blocks until heal-status consumption resumes or is
|
|
|
|
// aborted due to timeout.
|
|
|
|
maxUnconsumedHealResultItems = 1000
|
|
|
|
|
|
|
|
// if no heal-results are consumed (via the heal-status API)
|
|
|
|
// for this timeout duration, the heal sequence is aborted.
|
|
|
|
healUnconsumedTimeout = 24 * time.Hour
|
|
|
|
|
|
|
|
// time-duration to keep heal sequence state after it
|
|
|
|
// completes.
|
|
|
|
keepHealSeqStateDuration = time.Minute * 10
|
2019-10-28 18:27:49 +01:00
|
|
|
|
|
|
|
// nopHeal is a no operating healing action to
|
|
|
|
// wait for the current healing operation to finish
|
|
|
|
nopHeal = ""
|
2018-01-22 23:54:55 +01:00
|
|
|
)
|
|
|
|
|
|
|
|
var (
|
|
|
|
errHealIdleTimeout = fmt.Errorf("healing results were not consumed for too long")
|
|
|
|
errHealPushStopNDiscard = fmt.Errorf("heal push stopped due to heal stop signal")
|
2018-06-29 01:02:02 +02:00
|
|
|
errHealStopSignalled = fmt.Errorf("heal stop signaled")
|
2018-01-22 23:54:55 +01:00
|
|
|
|
2018-11-12 20:07:43 +01:00
|
|
|
errFnHealFromAPIErr = func(ctx context.Context, err error) error {
|
2019-02-12 10:25:52 +01:00
|
|
|
apiErr := toAPIError(ctx, err)
|
2018-01-22 23:54:55 +01:00
|
|
|
return fmt.Errorf("Heal internal error: %s: %s",
|
|
|
|
apiErr.Code, apiErr.Description)
|
|
|
|
}
|
|
|
|
)
|
|
|
|
|
|
|
|
// healSequenceStatus - accumulated status of the heal sequence
|
|
|
|
type healSequenceStatus struct {
|
|
|
|
// lock to update this structure as it is concurrently
|
|
|
|
// accessed
|
|
|
|
updateLock *sync.RWMutex
|
|
|
|
|
|
|
|
// summary and detail for failures
|
|
|
|
Summary healStatusSummary `json:"Summary"`
|
|
|
|
FailureDetail string `json:"Detail,omitempty"`
|
|
|
|
StartTime time.Time `json:"StartTime"`
|
|
|
|
|
|
|
|
// disk information
|
|
|
|
NumDisks int `json:"NumDisks"`
|
|
|
|
|
|
|
|
// settings for the heal sequence
|
|
|
|
HealSettings madmin.HealOpts `json:"Settings"`
|
|
|
|
|
|
|
|
// slice of available heal result records
|
|
|
|
Items []madmin.HealResultItem `json:"Items"`
|
|
|
|
}
|
|
|
|
|
|
|
|
// structure to hold state of all heal sequences in server memory
|
|
|
|
type allHealState struct {
|
|
|
|
sync.Mutex
|
|
|
|
|
|
|
|
// map of heal path to heal sequence
|
|
|
|
healSeqMap map[string]*healSequence
|
|
|
|
}
|
|
|
|
|
2019-06-09 07:14:07 +02:00
|
|
|
// initHealState - initialize healing apparatus
|
|
|
|
func initHealState() *allHealState {
|
|
|
|
healState := &allHealState{
|
2018-01-22 23:54:55 +01:00
|
|
|
healSeqMap: make(map[string]*healSequence),
|
|
|
|
}
|
2018-11-16 23:59:51 +01:00
|
|
|
|
2019-06-09 07:14:07 +02:00
|
|
|
go healState.periodicHealSeqsClean()
|
|
|
|
|
|
|
|
return healState
|
2018-11-16 23:59:51 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
func (ahs *allHealState) periodicHealSeqsClean() {
|
|
|
|
// Launch clean-up routine to remove this heal sequence (after
|
|
|
|
// it ends) from the global state after timeout has elapsed.
|
|
|
|
ticker := time.NewTicker(time.Minute * 5)
|
|
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-ticker.C:
|
|
|
|
now := UTCNow()
|
|
|
|
ahs.Lock()
|
|
|
|
for path, h := range ahs.healSeqMap {
|
|
|
|
if h.hasEnded() && h.endTime.Add(keepHealSeqStateDuration).Before(now) {
|
|
|
|
delete(ahs.healSeqMap, path)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
ahs.Unlock()
|
2019-01-05 23:16:43 +01:00
|
|
|
case <-GlobalServiceDoneCh:
|
2018-11-16 23:59:51 +01:00
|
|
|
// server could be restarting - need
|
|
|
|
// to exit immediately
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
2018-01-22 23:54:55 +01:00
|
|
|
}
|
|
|
|
|
2019-06-26 01:42:24 +02:00
|
|
|
// getHealSequenceByToken - Retrieve a heal sequence by token. The second
|
|
|
|
// argument returns if a heal sequence actually exists.
|
|
|
|
func (ahs *allHealState) getHealSequenceByToken(token string) (h *healSequence, exists bool) {
|
|
|
|
ahs.Lock()
|
|
|
|
defer ahs.Unlock()
|
|
|
|
for _, healSeq := range ahs.healSeqMap {
|
|
|
|
if healSeq.clientToken == token {
|
|
|
|
return healSeq, true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil, false
|
|
|
|
}
|
|
|
|
|
2018-01-22 23:54:55 +01:00
|
|
|
// getHealSequence - Retrieve a heal sequence by path. The second
|
|
|
|
// argument returns if a heal sequence actually exists.
|
|
|
|
func (ahs *allHealState) getHealSequence(path string) (h *healSequence, exists bool) {
|
|
|
|
ahs.Lock()
|
|
|
|
defer ahs.Unlock()
|
|
|
|
h, exists = ahs.healSeqMap[path]
|
|
|
|
return h, exists
|
|
|
|
}
|
|
|
|
|
2019-02-12 10:25:52 +01:00
|
|
|
func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
|
2018-11-05 04:24:16 +01:00
|
|
|
var hsp madmin.HealStopSuccess
|
|
|
|
he, exists := ahs.getHealSequence(path)
|
|
|
|
if !exists {
|
|
|
|
hsp = madmin.HealStopSuccess{
|
|
|
|
ClientToken: "invalid",
|
|
|
|
StartTime: UTCNow(),
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
hsp = madmin.HealStopSuccess{
|
|
|
|
ClientToken: he.clientToken,
|
|
|
|
ClientAddress: he.clientAddress,
|
|
|
|
StartTime: he.startTime,
|
|
|
|
}
|
|
|
|
|
|
|
|
he.stop()
|
|
|
|
for !he.hasEnded() {
|
|
|
|
time.Sleep(1 * time.Second)
|
|
|
|
}
|
|
|
|
ahs.Lock()
|
|
|
|
defer ahs.Unlock()
|
|
|
|
// Heal sequence explicitly stopped, remove it.
|
|
|
|
delete(ahs.healSeqMap, path)
|
|
|
|
}
|
|
|
|
|
|
|
|
b, err := json.Marshal(&hsp)
|
2019-02-12 10:25:52 +01:00
|
|
|
return b, toAdminAPIErr(context.Background(), err)
|
2018-11-05 04:24:16 +01:00
|
|
|
}
|
|
|
|
|
2018-01-22 23:54:55 +01:00
|
|
|
// LaunchNewHealSequence - launches a background routine that performs
|
|
|
|
// healing according to the healSequence argument. For each heal
|
|
|
|
// sequence, state is stored in the `globalAllHealState`, which is a
|
|
|
|
// map of the heal path to `healSequence` which holds state about the
|
|
|
|
// heal sequence.
|
|
|
|
//
|
|
|
|
// Heal results are persisted in server memory for
|
|
|
|
// `keepHealSeqStateDuration`. This function also launches a
|
|
|
|
// background routine to clean up heal results after the
|
|
|
|
// aforementioned duration.
|
|
|
|
func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
|
2019-02-12 10:25:52 +01:00
|
|
|
respBytes []byte, apiErr APIError, errMsg string) {
|
2018-01-22 23:54:55 +01:00
|
|
|
|
|
|
|
existsAndLive := false
|
|
|
|
he, exists := ahs.getHealSequence(h.path)
|
|
|
|
if exists {
|
|
|
|
if !he.hasEnded() || len(he.currentStatus.Items) > 0 {
|
|
|
|
existsAndLive = true
|
|
|
|
}
|
|
|
|
}
|
2018-11-05 04:24:16 +01:00
|
|
|
|
2018-01-22 23:54:55 +01:00
|
|
|
if existsAndLive {
|
|
|
|
// A heal sequence exists on the given path.
|
|
|
|
if h.forceStarted {
|
2018-11-05 04:24:16 +01:00
|
|
|
// stop the running heal sequence - wait for it to finish.
|
2018-01-22 23:54:55 +01:00
|
|
|
he.stop()
|
|
|
|
for !he.hasEnded() {
|
2018-11-05 04:24:16 +01:00
|
|
|
time.Sleep(1 * time.Second)
|
2018-01-22 23:54:55 +01:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
errMsg = "Heal is already running on the given path " +
|
|
|
|
"(use force-start option to stop and start afresh). " +
|
2018-11-05 04:24:16 +01:00
|
|
|
fmt.Sprintf("The heal was started by IP %s at %s, token is %s",
|
|
|
|
h.clientAddress, h.startTime.Format(http.TimeFormat), h.clientToken)
|
2019-02-12 10:25:52 +01:00
|
|
|
return nil, errorCodes.ToAPIErr(ErrHealAlreadyRunning), errMsg
|
2018-01-22 23:54:55 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
ahs.Lock()
|
|
|
|
defer ahs.Unlock()
|
|
|
|
|
|
|
|
// Check if new heal sequence to be started overlaps with any
|
|
|
|
// existing, running sequence
|
|
|
|
for k, hSeq := range ahs.healSeqMap {
|
|
|
|
if !hSeq.hasEnded() && (strings.HasPrefix(k, h.path) ||
|
|
|
|
strings.HasPrefix(h.path, k)) {
|
|
|
|
|
|
|
|
errMsg = "The provided heal sequence path overlaps with an existing " +
|
|
|
|
fmt.Sprintf("heal path: %s", k)
|
2019-02-12 10:25:52 +01:00
|
|
|
return nil, errorCodes.ToAPIErr(ErrHealOverlappingPaths), errMsg
|
2018-01-22 23:54:55 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Add heal state and start sequence
|
|
|
|
ahs.healSeqMap[h.path] = h
|
|
|
|
|
|
|
|
// Launch top-level background heal go-routine
|
|
|
|
go h.healSequenceStart()
|
|
|
|
|
|
|
|
b, err := json.Marshal(madmin.HealStartSuccess{
|
2018-03-27 01:36:57 +02:00
|
|
|
ClientToken: h.clientToken,
|
|
|
|
ClientAddress: h.clientAddress,
|
|
|
|
StartTime: h.startTime,
|
2018-01-22 23:54:55 +01:00
|
|
|
})
|
|
|
|
if err != nil {
|
2018-11-12 20:07:43 +01:00
|
|
|
logger.LogIf(h.ctx, err)
|
2019-02-12 10:25:52 +01:00
|
|
|
return nil, toAPIError(h.ctx, err), ""
|
2018-01-22 23:54:55 +01:00
|
|
|
}
|
2019-02-12 10:25:52 +01:00
|
|
|
return b, noError, ""
|
2018-01-22 23:54:55 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// PopHealStatusJSON - Called by heal-status API. It fetches the heal
|
|
|
|
// status results from global state and returns its JSON
|
|
|
|
// representation. The clientToken helps ensure there aren't
|
|
|
|
// conflicting clients fetching status.
|
|
|
|
func (ahs *allHealState) PopHealStatusJSON(path string,
|
|
|
|
clientToken string) ([]byte, APIErrorCode) {
|
|
|
|
|
|
|
|
// fetch heal state for given path
|
|
|
|
h, exists := ahs.getHealSequence(path)
|
|
|
|
if !exists {
|
|
|
|
// If there is no such heal sequence, return error.
|
|
|
|
return nil, ErrHealNoSuchProcess
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check if client-token is valid
|
|
|
|
if clientToken != h.clientToken {
|
|
|
|
return nil, ErrHealInvalidClientToken
|
|
|
|
}
|
|
|
|
|
|
|
|
// Take lock to access and update the heal-sequence
|
|
|
|
h.currentStatus.updateLock.Lock()
|
|
|
|
defer h.currentStatus.updateLock.Unlock()
|
|
|
|
|
|
|
|
numItems := len(h.currentStatus.Items)
|
|
|
|
|
|
|
|
// calculate index of most recently available heal result
|
|
|
|
// record.
|
|
|
|
lastResultIndex := h.lastSentResultIndex
|
|
|
|
if numItems > 0 {
|
|
|
|
lastResultIndex = h.currentStatus.Items[numItems-1].ResultIndex
|
|
|
|
}
|
|
|
|
|
|
|
|
// After sending status to client, and before relinquishing
|
|
|
|
// the updateLock, reset Item to nil and record the result
|
|
|
|
// index sent to the client.
|
|
|
|
defer func(i int64) {
|
|
|
|
h.lastSentResultIndex = i
|
|
|
|
h.currentStatus.Items = nil
|
|
|
|
}(lastResultIndex)
|
|
|
|
|
|
|
|
jbytes, err := json.Marshal(h.currentStatus)
|
|
|
|
if err != nil {
|
2018-11-12 20:07:43 +01:00
|
|
|
logger.LogIf(h.ctx, err)
|
2018-01-22 23:54:55 +01:00
|
|
|
return nil, ErrInternalError
|
|
|
|
}
|
|
|
|
|
|
|
|
return jbytes, ErrNone
|
|
|
|
}
|
|
|
|
|
|
|
|
// healSequence - state for each heal sequence initiated on the
|
|
|
|
// server.
|
|
|
|
type healSequence struct {
|
|
|
|
// bucket, and prefix on which heal seq. was initiated
|
|
|
|
bucket, objPrefix string
|
|
|
|
|
2018-11-05 04:24:16 +01:00
|
|
|
// path is just pathJoin(bucket, objPrefix)
|
2018-01-22 23:54:55 +01:00
|
|
|
path string
|
|
|
|
|
2019-06-09 07:14:07 +02:00
|
|
|
// List of entities (format, buckets, objects) to heal
|
|
|
|
sourceCh chan string
|
|
|
|
|
|
|
|
// Report healing progress, false if this is a background
|
|
|
|
// healing since currently there is no entity which will
|
|
|
|
// receive realtime healing status
|
|
|
|
reportProgress bool
|
|
|
|
|
2018-01-22 23:54:55 +01:00
|
|
|
// time at which heal sequence was started
|
|
|
|
startTime time.Time
|
|
|
|
|
2018-11-16 23:59:51 +01:00
|
|
|
// time at which heal sequence has ended
|
|
|
|
endTime time.Time
|
|
|
|
|
2018-01-22 23:54:55 +01:00
|
|
|
// Heal client info
|
|
|
|
clientToken, clientAddress string
|
|
|
|
|
|
|
|
// was this heal sequence force started?
|
|
|
|
forceStarted bool
|
|
|
|
|
|
|
|
// heal settings applied to this heal sequence
|
|
|
|
settings madmin.HealOpts
|
|
|
|
|
|
|
|
// current accumulated status of the heal sequence
|
|
|
|
currentStatus healSequenceStatus
|
|
|
|
|
2018-06-29 01:02:02 +02:00
|
|
|
// channel signaled by background routine when traversal has
|
2018-01-22 23:54:55 +01:00
|
|
|
// completed
|
|
|
|
traverseAndHealDoneCh chan error
|
|
|
|
|
|
|
|
// channel to signal heal sequence to stop (e.g. from the
|
|
|
|
// heal-stop API)
|
|
|
|
stopSignalCh chan struct{}
|
|
|
|
|
|
|
|
// the last result index sent to client
|
|
|
|
lastSentResultIndex int64
|
2018-03-14 20:01:47 +01:00
|
|
|
|
2019-06-26 01:42:24 +02:00
|
|
|
// Number of total items scanned
|
|
|
|
scannedItemsCount int64
|
|
|
|
|
|
|
|
// The time of the last scan/heal activity
|
|
|
|
lastHealActivity time.Time
|
|
|
|
|
2018-03-14 20:01:47 +01:00
|
|
|
// Holds the request-info for logging
|
|
|
|
ctx context.Context
|
2018-01-22 23:54:55 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// NewHealSequence - creates healSettings, assumes bucket and
|
|
|
|
// objPrefix are already validated.
|
|
|
|
func newHealSequence(bucket, objPrefix, clientAddr string,
|
|
|
|
numDisks int, hs madmin.HealOpts, forceStart bool) *healSequence {
|
|
|
|
|
2018-04-06 00:04:40 +02:00
|
|
|
reqInfo := &logger.ReqInfo{RemoteHost: clientAddr, API: "Heal", BucketName: bucket}
|
|
|
|
reqInfo.AppendTags("prefix", objPrefix)
|
|
|
|
ctx := logger.SetReqInfo(context.Background(), reqInfo)
|
2018-03-14 20:01:47 +01:00
|
|
|
|
2018-01-22 23:54:55 +01:00
|
|
|
return &healSequence{
|
2019-06-09 07:14:07 +02:00
|
|
|
bucket: bucket,
|
|
|
|
objPrefix: objPrefix,
|
|
|
|
path: pathJoin(bucket, objPrefix),
|
|
|
|
reportProgress: true,
|
|
|
|
startTime: UTCNow(),
|
|
|
|
clientToken: mustGetUUID(),
|
|
|
|
clientAddress: clientAddr,
|
|
|
|
forceStarted: forceStart,
|
|
|
|
settings: hs,
|
2018-01-22 23:54:55 +01:00
|
|
|
currentStatus: healSequenceStatus{
|
|
|
|
Summary: healNotStartedStatus,
|
|
|
|
HealSettings: hs,
|
|
|
|
NumDisks: numDisks,
|
|
|
|
updateLock: &sync.RWMutex{},
|
|
|
|
},
|
|
|
|
traverseAndHealDoneCh: make(chan error),
|
|
|
|
stopSignalCh: make(chan struct{}),
|
2018-03-14 20:01:47 +01:00
|
|
|
ctx: ctx,
|
2018-01-22 23:54:55 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// isQuitting - determines if the heal sequence is quitting (due to an
|
|
|
|
// external signal)
|
|
|
|
func (h *healSequence) isQuitting() bool {
|
|
|
|
select {
|
|
|
|
case <-h.stopSignalCh:
|
|
|
|
return true
|
|
|
|
default:
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// check if the heal sequence has ended
|
|
|
|
func (h *healSequence) hasEnded() bool {
|
|
|
|
h.currentStatus.updateLock.RLock()
|
|
|
|
summary := h.currentStatus.Summary
|
|
|
|
h.currentStatus.updateLock.RUnlock()
|
|
|
|
return summary == healStoppedStatus || summary == healFinishedStatus
|
|
|
|
}
|
|
|
|
|
|
|
|
// stops the heal sequence - safe to call multiple times.
|
|
|
|
func (h *healSequence) stop() {
|
|
|
|
select {
|
|
|
|
case <-h.stopSignalCh:
|
|
|
|
default:
|
|
|
|
close(h.stopSignalCh)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// pushHealResultItem - pushes a heal result item for consumption in
|
|
|
|
// the heal-status API. It blocks if there are
|
|
|
|
// maxUnconsumedHealResultItems. When it blocks, the heal sequence
|
|
|
|
// routine is effectively paused - this happens when the server has
|
|
|
|
// accumulated the maximum number of heal records per heal
|
|
|
|
// sequence. When the client consumes further records, the heal
|
|
|
|
// sequence automatically resumes. The return value indicates if the
|
|
|
|
// operation succeeded.
|
|
|
|
func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
|
|
|
|
// start a timer to keep an upper time limit to find an empty
|
|
|
|
// slot to add the given heal result - if no slot is found it
|
|
|
|
// means that the server is holding the maximum amount of
|
|
|
|
// heal-results in memory and the client has not consumed it
|
|
|
|
// for too long.
|
|
|
|
unconsumedTimer := time.NewTimer(healUnconsumedTimeout)
|
|
|
|
defer func() {
|
|
|
|
// stop the timeout timer so it is garbage collected.
|
|
|
|
if !unconsumedTimer.Stop() {
|
|
|
|
<-unconsumedTimer.C
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
var itemsLen int
|
|
|
|
for {
|
|
|
|
h.currentStatus.updateLock.Lock()
|
|
|
|
itemsLen = len(h.currentStatus.Items)
|
|
|
|
if itemsLen == maxUnconsumedHealResultItems {
|
|
|
|
// unlock and wait to check again if we can push
|
|
|
|
h.currentStatus.updateLock.Unlock()
|
|
|
|
|
|
|
|
// wait for a second, or quit if an external
|
|
|
|
// stop signal is received or the
|
|
|
|
// unconsumedTimer fires.
|
|
|
|
select {
|
|
|
|
// Check after a second
|
|
|
|
case <-time.After(time.Second):
|
|
|
|
continue
|
|
|
|
|
|
|
|
case <-h.stopSignalCh:
|
|
|
|
// discard result and return.
|
|
|
|
return errHealPushStopNDiscard
|
|
|
|
|
|
|
|
// Timeout if no results consumed for too
|
|
|
|
// long.
|
|
|
|
case <-unconsumedTimer.C:
|
|
|
|
return errHealIdleTimeout
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break
|
|
|
|
}
|
|
|
|
|
|
|
|
// Set the correct result index for the new result item
|
|
|
|
if itemsLen > 0 {
|
|
|
|
r.ResultIndex = 1 + h.currentStatus.Items[itemsLen-1].ResultIndex
|
|
|
|
} else {
|
|
|
|
r.ResultIndex = 1 + h.lastSentResultIndex
|
|
|
|
}
|
|
|
|
|
|
|
|
// append to results
|
|
|
|
h.currentStatus.Items = append(h.currentStatus.Items, r)
|
|
|
|
|
|
|
|
// release lock
|
|
|
|
h.currentStatus.updateLock.Unlock()
|
|
|
|
|
|
|
|
// This is a "safe" point for the heal sequence to quit if
|
2018-06-29 01:02:02 +02:00
|
|
|
// signaled externally.
|
2018-01-22 23:54:55 +01:00
|
|
|
if h.isQuitting() {
|
|
|
|
return errHealStopSignalled
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// healSequenceStart - this is the top-level background heal
|
|
|
|
// routine. It launches another go-routine that actually traverses
|
|
|
|
// on-disk data, checks and heals according to the selected
|
|
|
|
// settings. This go-routine itself, (1) monitors the traversal
|
|
|
|
// routine for completion, and (2) listens for external stop
|
|
|
|
// signals. When either event happens, it sets the finish status for
|
|
|
|
// the heal-sequence.
|
|
|
|
func (h *healSequence) healSequenceStart() {
|
|
|
|
// Set status as running
|
|
|
|
h.currentStatus.updateLock.Lock()
|
|
|
|
h.currentStatus.Summary = healRunningStatus
|
|
|
|
h.currentStatus.StartTime = UTCNow()
|
|
|
|
h.currentStatus.updateLock.Unlock()
|
|
|
|
|
2019-06-09 07:14:07 +02:00
|
|
|
if h.sourceCh == nil {
|
|
|
|
go h.traverseAndHeal()
|
|
|
|
} else {
|
|
|
|
go h.healFromSourceCh()
|
|
|
|
}
|
2018-01-22 23:54:55 +01:00
|
|
|
|
|
|
|
select {
|
|
|
|
case err, ok := <-h.traverseAndHealDoneCh:
|
2018-11-16 23:59:51 +01:00
|
|
|
h.endTime = UTCNow()
|
2018-01-22 23:54:55 +01:00
|
|
|
h.currentStatus.updateLock.Lock()
|
|
|
|
defer h.currentStatus.updateLock.Unlock()
|
|
|
|
// Heal traversal is complete.
|
|
|
|
if ok {
|
|
|
|
// heal traversal had an error.
|
|
|
|
h.currentStatus.Summary = healStoppedStatus
|
|
|
|
h.currentStatus.FailureDetail = err.Error()
|
|
|
|
} else {
|
|
|
|
// heal traversal succeeded.
|
|
|
|
h.currentStatus.Summary = healFinishedStatus
|
|
|
|
}
|
|
|
|
|
|
|
|
case <-h.stopSignalCh:
|
2018-11-16 23:59:51 +01:00
|
|
|
h.endTime = UTCNow()
|
2018-01-22 23:54:55 +01:00
|
|
|
h.currentStatus.updateLock.Lock()
|
|
|
|
h.currentStatus.Summary = healStoppedStatus
|
|
|
|
h.currentStatus.FailureDetail = errHealStopSignalled.Error()
|
|
|
|
h.currentStatus.updateLock.Unlock()
|
|
|
|
|
|
|
|
// drain traverse channel so the traversal
|
|
|
|
// go-routine does not leak.
|
|
|
|
go func() {
|
|
|
|
// Eventually the traversal go-routine closes
|
|
|
|
// the channel and returns, so this go-routine
|
|
|
|
// itself will not leak.
|
|
|
|
<-h.traverseAndHealDoneCh
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-06-09 07:14:07 +02:00
|
|
|
func (h *healSequence) queueHealTask(path string, healType madmin.HealItemType) error {
|
|
|
|
var respCh = make(chan healResult)
|
|
|
|
defer close(respCh)
|
|
|
|
// Send heal request
|
2019-10-28 18:27:49 +01:00
|
|
|
globalBackgroundHealRoutine.queueHealTask(healTask{path: path, responseCh: respCh, opts: h.settings})
|
2019-06-09 07:14:07 +02:00
|
|
|
// Wait for answer and push result to the client
|
|
|
|
res := <-respCh
|
|
|
|
if !h.reportProgress {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
res.result.Type = healType
|
|
|
|
if res.err != nil {
|
|
|
|
// Object might have been deleted, by the time heal
|
|
|
|
// was attempted, we should ignore this object and return success.
|
|
|
|
if isErrObjectNotFound(res.err) {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
// Only report object error
|
|
|
|
if healType != madmin.HealItemObject {
|
|
|
|
return res.err
|
|
|
|
}
|
|
|
|
res.result.Detail = res.err.Error()
|
|
|
|
}
|
|
|
|
return h.pushHealResultItem(res.result)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (h *healSequence) healItemsFromSourceCh() error {
|
2019-06-26 01:42:24 +02:00
|
|
|
h.lastHealActivity = UTCNow()
|
|
|
|
|
2019-06-09 07:14:07 +02:00
|
|
|
// Start healing the config prefix.
|
|
|
|
if err := h.healMinioSysMeta(minioConfigPrefix)(); err != nil {
|
2019-06-26 01:42:24 +02:00
|
|
|
logger.LogIf(h.ctx, err)
|
2019-06-09 07:14:07 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Start healing the bucket config prefix.
|
|
|
|
if err := h.healMinioSysMeta(bucketConfigPrefix)(); err != nil {
|
2019-06-26 01:42:24 +02:00
|
|
|
logger.LogIf(h.ctx, err)
|
2019-06-09 07:14:07 +02:00
|
|
|
}
|
|
|
|
|
2019-12-12 15:02:37 +01:00
|
|
|
// Start healing the background ops prefix.
|
|
|
|
if err := h.healMinioSysMeta(backgroundOpsMetaPrefix)(); err != nil {
|
|
|
|
logger.LogIf(h.ctx, err)
|
|
|
|
}
|
|
|
|
|
2019-06-09 07:14:07 +02:00
|
|
|
for path := range h.sourceCh {
|
2019-06-26 01:42:24 +02:00
|
|
|
|
2019-06-09 07:14:07 +02:00
|
|
|
var itemType madmin.HealItemType
|
2018-01-22 23:54:55 +01:00
|
|
|
switch {
|
2019-10-28 18:27:49 +01:00
|
|
|
case path == nopHeal:
|
|
|
|
continue
|
2019-08-06 21:08:58 +02:00
|
|
|
case path == SlashSeparator:
|
2019-06-09 07:14:07 +02:00
|
|
|
itemType = madmin.HealItemMetadata
|
2019-08-06 21:08:58 +02:00
|
|
|
case !strings.Contains(path, SlashSeparator):
|
2019-06-09 07:14:07 +02:00
|
|
|
itemType = madmin.HealItemBucket
|
|
|
|
default:
|
|
|
|
itemType = madmin.HealItemObject
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := h.queueHealTask(path, itemType); err != nil {
|
2019-06-26 01:42:24 +02:00
|
|
|
logger.LogIf(h.ctx, err)
|
2018-01-22 23:54:55 +01:00
|
|
|
}
|
2019-06-26 01:42:24 +02:00
|
|
|
|
|
|
|
h.scannedItemsCount++
|
|
|
|
h.lastHealActivity = UTCNow()
|
2018-01-22 23:54:55 +01:00
|
|
|
}
|
|
|
|
|
2019-06-09 07:14:07 +02:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (h *healSequence) healFromSourceCh() {
|
2019-10-16 03:35:41 +02:00
|
|
|
h.healItemsFromSourceCh()
|
2019-06-09 07:14:07 +02:00
|
|
|
close(h.traverseAndHealDoneCh)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (h *healSequence) healItems() error {
|
2018-01-22 23:54:55 +01:00
|
|
|
// Start with format healing
|
2019-06-09 07:14:07 +02:00
|
|
|
if err := h.healDiskFormat(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2018-01-22 23:54:55 +01:00
|
|
|
|
2019-02-11 04:53:13 +01:00
|
|
|
// Start healing the config prefix.
|
2019-06-09 07:14:07 +02:00
|
|
|
if err := h.healMinioSysMeta(minioConfigPrefix)(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2019-02-11 04:53:13 +01:00
|
|
|
|
|
|
|
// Start healing the bucket config prefix.
|
2019-06-09 07:14:07 +02:00
|
|
|
if err := h.healMinioSysMeta(bucketConfigPrefix)(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2018-09-29 10:17:01 +02:00
|
|
|
|
2019-12-12 15:02:37 +01:00
|
|
|
// Start healing the background ops prefix.
|
|
|
|
if err := h.healMinioSysMeta(backgroundOpsMetaPrefix)(); err != nil {
|
|
|
|
logger.LogIf(h.ctx, err)
|
|
|
|
}
|
|
|
|
|
2018-01-22 23:54:55 +01:00
|
|
|
// Heal buckets and objects
|
2019-06-09 07:14:07 +02:00
|
|
|
return h.healBuckets()
|
|
|
|
}
|
2018-01-22 23:54:55 +01:00
|
|
|
|
2019-06-09 07:14:07 +02:00
|
|
|
// traverseAndHeal - traverses on-disk data and performs healing
|
|
|
|
// according to settings. At each "safe" point it also checks if an
|
|
|
|
// external quit signal has been received and quits if so. Since the
|
|
|
|
// healing traversal may be mutating on-disk data when an external
|
|
|
|
// quit signal is received, this routine cannot quit immediately and
|
|
|
|
// has to wait until a safe point is reached, such as between scanning
|
|
|
|
// two objects.
|
|
|
|
func (h *healSequence) traverseAndHeal() {
|
|
|
|
if err := h.healItems(); err != nil {
|
|
|
|
if h.isQuitting() {
|
|
|
|
err = errHealStopSignalled
|
|
|
|
}
|
2018-01-22 23:54:55 +01:00
|
|
|
h.traverseAndHealDoneCh <- err
|
|
|
|
}
|
|
|
|
|
|
|
|
close(h.traverseAndHealDoneCh)
|
|
|
|
}
|
|
|
|
|
2019-02-11 04:53:13 +01:00
|
|
|
// healMinioSysMeta - heals all files under a given meta prefix, returns a function
|
|
|
|
// which in-turn heals the respective meta directory path and any files in int.
|
|
|
|
func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
|
|
|
|
return func() error {
|
|
|
|
// Get current object layer instance.
|
2019-11-09 18:27:23 +01:00
|
|
|
objectAPI := newObjectLayerWithoutSafeModeFn()
|
2019-02-11 04:53:13 +01:00
|
|
|
if objectAPI == nil {
|
|
|
|
return errServerNotInitialized
|
2018-09-29 10:17:01 +02:00
|
|
|
}
|
2018-10-01 18:54:26 +02:00
|
|
|
|
2019-02-11 04:53:13 +01:00
|
|
|
// NOTE: Healing on meta is run regardless
|
|
|
|
// of any bucket being selected, this is to ensure that
|
|
|
|
// meta are always upto date and correct.
|
2019-03-14 01:35:09 +01:00
|
|
|
return objectAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, func(bucket string, object string) error {
|
|
|
|
if h.isQuitting() {
|
|
|
|
return errHealStopSignalled
|
2018-10-01 18:54:26 +02:00
|
|
|
}
|
2019-06-09 07:14:07 +02:00
|
|
|
|
|
|
|
herr := h.queueHealTask(pathJoin(bucket, object), madmin.HealItemBucketMetadata)
|
2019-03-14 01:35:09 +01:00
|
|
|
// Object might have been deleted, by the time heal
|
|
|
|
// was attempted we ignore this object an move on.
|
|
|
|
if isErrObjectNotFound(herr) {
|
|
|
|
return nil
|
2018-10-01 18:54:26 +02:00
|
|
|
}
|
2019-06-09 07:14:07 +02:00
|
|
|
return herr
|
2019-03-14 01:35:09 +01:00
|
|
|
})
|
2018-09-29 10:17:01 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-01-22 23:54:55 +01:00
|
|
|
// healDiskFormat - heals format.json, return value indicates if a
|
|
|
|
// failure error occurred.
|
|
|
|
func (h *healSequence) healDiskFormat() error {
|
2018-10-01 18:54:26 +02:00
|
|
|
if h.isQuitting() {
|
|
|
|
return errHealStopSignalled
|
|
|
|
}
|
|
|
|
|
2018-01-22 23:54:55 +01:00
|
|
|
// Get current object layer instance.
|
2019-11-09 18:27:23 +01:00
|
|
|
objectAPI := newObjectLayerWithoutSafeModeFn()
|
2018-01-22 23:54:55 +01:00
|
|
|
if objectAPI == nil {
|
|
|
|
return errServerNotInitialized
|
|
|
|
}
|
|
|
|
|
2019-08-06 21:08:58 +02:00
|
|
|
return h.queueHealTask(SlashSeparator, madmin.HealItemMetadata)
|
2018-01-22 23:54:55 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// healBuckets - check for all buckets heal or just particular bucket.
|
|
|
|
func (h *healSequence) healBuckets() error {
|
2018-10-01 18:54:26 +02:00
|
|
|
if h.isQuitting() {
|
|
|
|
return errHealStopSignalled
|
|
|
|
}
|
|
|
|
|
2018-01-22 23:54:55 +01:00
|
|
|
// 1. If a bucket was specified, heal only the bucket.
|
|
|
|
if h.bucket != "" {
|
|
|
|
return h.healBucket(h.bucket)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get current object layer instance.
|
2019-11-09 18:27:23 +01:00
|
|
|
objectAPI := newObjectLayerWithoutSafeModeFn()
|
2018-01-22 23:54:55 +01:00
|
|
|
if objectAPI == nil {
|
|
|
|
return errServerNotInitialized
|
|
|
|
}
|
|
|
|
|
2018-03-14 20:01:47 +01:00
|
|
|
buckets, err := objectAPI.ListBucketsHeal(h.ctx)
|
2018-01-22 23:54:55 +01:00
|
|
|
if err != nil {
|
2018-11-12 20:07:43 +01:00
|
|
|
return errFnHealFromAPIErr(h.ctx, err)
|
2018-01-22 23:54:55 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, bucket := range buckets {
|
2018-02-16 02:45:57 +01:00
|
|
|
if err = h.healBucket(bucket.Name); err != nil {
|
2018-01-22 23:54:55 +01:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// healBucket - traverses and heals given bucket
|
|
|
|
func (h *healSequence) healBucket(bucket string) error {
|
|
|
|
// Get current object layer instance.
|
2019-11-09 18:27:23 +01:00
|
|
|
objectAPI := newObjectLayerWithoutSafeModeFn()
|
2018-01-22 23:54:55 +01:00
|
|
|
if objectAPI == nil {
|
|
|
|
return errServerNotInitialized
|
|
|
|
}
|
|
|
|
|
2019-06-09 07:14:07 +02:00
|
|
|
if err := h.queueHealTask(bucket, madmin.HealItemBucket); err != nil {
|
2019-02-11 04:53:13 +01:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2018-01-22 23:54:55 +01:00
|
|
|
if !h.settings.Recursive {
|
|
|
|
if h.objPrefix != "" {
|
|
|
|
// Check if an object named as the objPrefix exists,
|
|
|
|
// and if so heal it.
|
2019-06-09 07:14:07 +02:00
|
|
|
_, err := objectAPI.GetObjectInfo(h.ctx, bucket, h.objPrefix, ObjectOptions{})
|
2018-01-22 23:54:55 +01:00
|
|
|
if err == nil {
|
2018-10-01 18:54:26 +02:00
|
|
|
if err = h.healObject(bucket, h.objPrefix); err != nil {
|
2018-01-22 23:54:55 +01:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2019-06-09 07:14:07 +02:00
|
|
|
if err := objectAPI.HealObjects(h.ctx, bucket, h.objPrefix, h.healObject); err != nil {
|
2019-03-14 01:35:09 +01:00
|
|
|
return errFnHealFromAPIErr(h.ctx, err)
|
2018-01-22 23:54:55 +01:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// healObject - heal the given object and record result
|
|
|
|
func (h *healSequence) healObject(bucket, object string) error {
|
|
|
|
if h.isQuitting() {
|
|
|
|
return errHealStopSignalled
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get current object layer instance.
|
2019-11-09 18:27:23 +01:00
|
|
|
objectAPI := newObjectLayerWithoutSafeModeFn()
|
2018-01-22 23:54:55 +01:00
|
|
|
if objectAPI == nil {
|
|
|
|
return errServerNotInitialized
|
|
|
|
}
|
|
|
|
|
2019-06-09 07:14:07 +02:00
|
|
|
return h.queueHealTask(pathJoin(bucket, object), madmin.HealItemObject)
|
2018-01-22 23:54:55 +01:00
|
|
|
}
|