add support for concurrent heals

cmd/admin-handlers.go

@@ -32,6 +32,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -772,14 +773,25 @@ func (a adminAPIHandlers) HealSetsHandler(w http.ResponseWriter, r *http.Request
 
 	buckets, _ := objectAPI.ListBucketsHeal(ctx)
 	ctx, opts.cancel = context.WithCancel(context.Background())
-	for _, setNumber := range opts.setNumbers {
-		go func(setNumber int) {
-			lbDisks := z.serverSets[0].sets[setNumber].getOnlineDisks()
-			if err := healErasureSet(ctx, setNumber, opts.sleepForIO, opts.sleepDuration, buckets, lbDisks); err != nil {
-				logger.LogIf(ctx, err)
-			}
-		}(setNumber)
-	}
+	go func() {
+		var wg sync.WaitGroup
+		for _, setNumber := range opts.setNumbers {
+			wg.Add(1)
+			go func(setNumber int) {
+				defer wg.Done()
+				lbDisks := z.serverSets[0].sets[setNumber].getOnlineDisks()
+				if err := healErasureSet(ctx, setNumber, opts.sleepForIO, opts.sleepDuration, buckets, lbDisks); err != nil {
+					logger.LogIf(ctx, err)
+				}
+			}(setNumber)
+		}
+		wg.Wait()
+		a.mu.Lock()
+		opts.cancel()
+		delete(a.healSetsMap, opts.taskUUID)
+		a.mu.Unlock()
+		logger.Info("Healing finished for %v", vars[healSetsList])
+	}()
 
 	a.mu.Lock()
 	a.healSetsMap[opts.taskUUID] = opts

cmd/admin-heal-ops.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"runtime"
 	"sync"
 	"time"
 
@@ -420,7 +421,7 @@ func newHealSequence(ctx context.Context, bucket, objPrefix, clientAddr string,
 	clientToken := mustGetUUID()
 
 	return &healSequence{
-		respCh:         make(chan healResult),
+		respCh:         make(chan healResult, runtime.GOMAXPROCS(0)),
 		bucket:         bucket,
 		object:         objPrefix,
 		reportProgress: true,

cmd/background-heal-ops.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"path"
+	"runtime"
 	"time"
 
 	"github.com/minio/minio/pkg/madmin"
@@ -125,7 +126,7 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 
 func newHealRoutine() *healRoutine {
 	return &healRoutine{
-		tasks:  make(chan healTask),
+		tasks:  make(chan healTask, runtime.GOMAXPROCS(0)),
 		doneCh: make(chan struct{}),
 	}
 

cmd/global-heal.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"runtime"
 	"sync"
 	"time"
 
@@ -42,8 +43,8 @@ func newBgHealSequence() *healSequence {
 	}
 
 	return &healSequence{
-		sourceCh:    make(chan healSource),
-		respCh:      make(chan healResult),
+		sourceCh:    make(chan healSource, runtime.GOMAXPROCS(0)),
+		respCh:      make(chan healResult, runtime.GOMAXPROCS(0)),
 		startTime:   UTCNow(),
 		clientToken: bgHealingUUID,
 		// run-background heal with reserved bucket