admin: Add Background heal status info API (#7774)

This API returns the information related to the self healing routine.

For the moment, it returns:
- The total number of objects that are scanned
- The last time when an item was scanned

cmd/admin-handlers.go

@@ -785,6 +785,49 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	keepConnLive(w, respCh)
 }
 
+func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "HealBackgroundStatus")
+
+	objectAPI := validateAdminReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	// Check if this setup has an erasure coded backend.
+	if !globalIsXL {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL)
+		return
+	}
+
+	var bgHealStates []madmin.BgHealState
+
+	// Get local heal status first
+	bgHealStates = append(bgHealStates, getLocalBackgroundHealStatus())
+
+	if globalIsDistXL {
+		// Get heal status from other peers
+		peersHealStates := globalNotificationSys.BackgroundHealStatus()
+		bgHealStates = append(bgHealStates, peersHealStates...)
+	}
+
+	// Aggregate healing result
+	var aggregatedHealStateResult = madmin.BgHealState{}
+	for _, state := range bgHealStates {
+		aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
+		if aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
+			aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
+		}
+
+	}
+
+	if err := json.NewEncoder(w).Encode(aggregatedHealStateResult); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
 // GetConfigHandler - GET /minio/admin/v1/config
 // Get config.json of this minio setup.
 func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {

cmd/admin-heal-ops.go

@@ -131,6 +131,19 @@ func (ahs *allHealState) periodicHealSeqsClean() {
 	}
 }
 
+// getHealSequenceByToken - Retrieve a heal sequence by token. The second
+// argument returns if a heal sequence actually exists.
+func (ahs *allHealState) getHealSequenceByToken(token string) (h *healSequence, exists bool) {
+	ahs.Lock()
+	defer ahs.Unlock()
+	for _, healSeq := range ahs.healSeqMap {
+		if healSeq.clientToken == token {
+			return healSeq, true
+		}
+	}
+	return nil, false
+}
+
 // getHealSequence - Retrieve a heal sequence by path. The second
 // argument returns if a heal sequence actually exists.
 func (ahs *allHealState) getHealSequence(path string) (h *healSequence, exists bool) {
@@ -335,6 +348,12 @@ type healSequence struct {
 	// the last result index sent to client
 	lastSentResultIndex int64
 
+	// Number of total items scanned
+	scannedItemsCount int64
+
+	// The time of the last scan/heal activity
+	lastHealActivity time.Time
+
 	// Holds the request-info for logging
 	ctx context.Context
 }
@@ -552,17 +571,20 @@ func (h *healSequence) queueHealTask(path string, healType madmin.HealItemType)
 }
 
 func (h *healSequence) healItemsFromSourceCh() error {
+	h.lastHealActivity = UTCNow()
+
 	// Start healing the config prefix.
 	if err := h.healMinioSysMeta(minioConfigPrefix)(); err != nil {
-		return err
+		logger.LogIf(h.ctx, err)
 	}
 
 	// Start healing the bucket config prefix.
 	if err := h.healMinioSysMeta(bucketConfigPrefix)(); err != nil {
-		return err
+		logger.LogIf(h.ctx, err)
 	}
 
 	for path := range h.sourceCh {
+
 		var itemType madmin.HealItemType
 		switch {
 		case path == "/":
@@ -574,8 +596,11 @@ func (h *healSequence) healItemsFromSourceCh() error {
 		}
 
 		if err := h.queueHealTask(path, itemType); err != nil {
-			return err
+			logger.LogIf(h.ctx, err)
 		}
+
+		h.scannedItemsCount++
+		h.lastHealActivity = UTCNow()
 	}
 
 	return nil

cmd/admin-router.go

@@ -61,6 +61,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 
+		adminV1Router.Methods(http.MethodPost).Path("/background-heal/status").HandlerFunc(httpTraceAll(adminAPI.BackgroundHealStatusHandler))
+
 		/// Health operations
 
 	}

cmd/daily-heal-ops.go

@@ -60,6 +60,18 @@ func newBgHealSequence(numDisks int) *healSequence {
 	}
 }
 
+func getLocalBackgroundHealStatus() madmin.BgHealState {
+	backgroundSequence, ok := globalSweepHealState.getHealSequenceByToken(bgHealingUUID)
+	if !ok {
+		return madmin.BgHealState{}
+	}
+
+	return madmin.BgHealState{
+		ScannedItemsCount: backgroundSequence.scannedItemsCount,
+		LastHealActivity:  backgroundSequence.lastHealActivity,
+	}
+}
+
 func initDailyHeal() {
 	go startDailyHeal()
 }

cmd/notification.go

@@ -34,6 +34,7 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/madmin"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/policy"
 )
@@ -230,6 +231,24 @@ func (sys *NotificationSys) LoadUsers() []NotificationPeerErr {
 	return ng.Wait()
 }
 
+// BackgroundHealStatus - returns background heal status of all peers
+func (sys *NotificationSys) BackgroundHealStatus() []madmin.BgHealState {
+	states := make([]madmin.BgHealState, len(sys.peerClients))
+	for idx, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		st, err := client.BackgroundHealStatus()
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+		} else {
+			states[idx] = st
+		}
+	}
+
+	return states
+}
+
 // StartProfiling - start profiling on remote peers, by initiating a remote RPC.
 func (sys *NotificationSys) StartProfiling(profiler string) []NotificationPeerErr {
 	ng := WithNPeers(len(sys.peerClients))

cmd/peer-rest-client.go

@@ -31,6 +31,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"
 	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/madmin"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/policy"
 )
@@ -422,6 +423,18 @@ func (client *peerRESTClient) SignalService(sig serviceSignal) error {
 	return nil
 }
 
+func (client *peerRESTClient) BackgroundHealStatus() (madmin.BgHealState, error) {
+	respBody, err := client.call(peerRESTMethodBackgroundHealStatus, nil, nil, -1)
+	if err != nil {
+		return madmin.BgHealState{}, err
+	}
+	defer http.DrainBody(respBody)
+
+	state := madmin.BgHealState{}
+	err = gob.NewDecoder(respBody).Decode(&state)
+	return state, err
+}
+
 // Trace - send http trace request to peer nodes
 func (client *peerRESTClient) Trace(doneCh chan struct{}, trcAll bool) (chan []byte, error) {
 	ch := make(chan []byte)

cmd/peer-rest-common.go

@@ -26,6 +26,7 @@ const (
 	peerRESTMethodDrivePerfInfo            = "driveperfinfo"
 	peerRESTMethodDeleteBucket             = "deletebucket"
 	peerRESTMethodSignalService            = "signalservice"
+	peerRESTMethodBackgroundHealStatus     = "backgroundhealstatus"
 	peerRESTMethodGetLocks                 = "getlocks"
 	peerRESTMethodBucketPolicyRemove       = "removebucketpolicy"
 	peerRESTMethodLoadUser                 = "loaduser"

cmd/peer-rest-server.go

@@ -709,6 +709,20 @@ func (s *peerRESTServer) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+func (s *peerRESTServer) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("invalid request"))
+		return
+	}
+
+	ctx := newContext(r, w, "BackgroundHealStatus")
+
+	state := getLocalBackgroundHealStatus()
+
+	defer w.(http.Flusher).Flush()
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(state))
+}
+
 func (s *peerRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
 	w.WriteHeader(http.StatusForbidden)
 	w.Write([]byte(err.Error()))
@@ -755,6 +769,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodReloadFormat).HandlerFunc(httpTraceHdrs(server.ReloadFormatHandler)).Queries(restQueries(peerRESTDryRun)...)
 
 	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodTrace).HandlerFunc(server.TraceHandler)
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodBackgroundHealStatus).HandlerFunc(server.BackgroundHealStatusHandler)
 
 	router.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
 }

pkg/madmin/examples/heal-status.go

@@ -0,0 +1,45 @@
+// +build ignore
+
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package main
+
+import (
+	"log"
+
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
+	// dummy values, please replace them with original values.
+
+	// API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
+	// New returns an MinIO Admin client object.
+	madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	healStatusResult, err := madmClnt.BackgroundHealStatus()
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	log.Printf("Heal status result: %+v\n", healStatusResult)
+}

pkg/madmin/heal-commands.go

@@ -269,3 +269,37 @@ func (adm *AdminClient) Heal(bucket, prefix string, healOpts HealOpts,
 	}
 	return healStart, healTaskStatus, nil
 }
+
+// BgHealState represents the status of the background heal
+type BgHealState struct {
+	ScannedItemsCount int64
+	LastHealActivity  time.Time
+}
+
+// BackgroundHealStatus returns the background heal status of the
+// current server or cluster.
+func (adm *AdminClient) BackgroundHealStatus() (BgHealState, error) {
+	// Execute POST request to background heal status api
+	resp, err := adm.executeMethod("POST", requestData{relPath: "/v1/background-heal/status"})
+	if err != nil {
+		return BgHealState{}, err
+	}
+	defer closeResponse(resp)
+
+	if resp.StatusCode != http.StatusOK {
+		return BgHealState{}, httpRespToErrorResponse(resp)
+	}
+
+	respBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return BgHealState{}, err
+	}
+
+	var healState BgHealState
+
+	err = json.Unmarshal(respBytes, &healState)
+	if err != nil {
+		return BgHealState{}, err
+	}
+	return healState, nil
+}