2016-05-30 06:05:00 +02:00
|
|
|
/*
|
2019-04-09 20:39:42 +02:00
|
|
|
* MinIO Cloud Storage, (C) 2016 MinIO, Inc.
|
2016-05-30 06:05:00 +02:00
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2016-08-19 01:23:42 +02:00
|
|
|
package cmd
|
2016-05-30 06:05:00 +02:00
|
|
|
|
|
|
|
import (
|
2016-06-03 20:33:50 +02:00
|
|
|
"errors"
|
2019-05-14 22:49:10 +02:00
|
|
|
"reflect"
|
2016-05-30 06:05:00 +02:00
|
|
|
"sync"
|
|
|
|
"time"
|
|
|
|
)
|
|
|
|
|
|
|
|
// Global lookup timeout.
|
|
|
|
const (
|
2020-06-17 18:52:07 +02:00
|
|
|
globalLookupTimeout = time.Minute * 30 // 30minutes.
|
|
|
|
treeWalkEntryLimit = 50
|
|
|
|
treeWalkSameEntryLimit = 4
|
2016-05-30 06:05:00 +02:00
|
|
|
)
|
|
|
|
|
2016-06-05 20:55:45 +02:00
|
|
|
// listParams - list object params used for list object map
|
|
|
|
type listParams struct {
|
|
|
|
bucket string
|
|
|
|
recursive bool
|
|
|
|
marker string
|
|
|
|
prefix string
|
|
|
|
}
|
|
|
|
|
|
|
|
// errWalkAbort - returned by doTreeWalk() if it returns prematurely.
|
|
|
|
// doTreeWalk() can return prematurely if
|
|
|
|
// 1) treeWalk is timed out by the timer go-routine.
|
|
|
|
// 2) there is an error during tree walk.
|
2016-06-03 20:33:50 +02:00
|
|
|
var errWalkAbort = errors.New("treeWalk abort")
|
|
|
|
|
2016-06-05 20:55:45 +02:00
|
|
|
// treeWalk - represents the go routine that does the file tree walk.
|
|
|
|
type treeWalk struct {
|
2020-06-17 18:52:07 +02:00
|
|
|
added time.Time
|
2019-04-17 18:52:08 +02:00
|
|
|
resultCh chan TreeWalkResult
|
2016-06-05 20:55:45 +02:00
|
|
|
endWalkCh chan struct{} // To signal when treeWalk go-routine should end.
|
|
|
|
endTimerCh chan<- struct{} // To signal when timer go-routine should end.
|
2016-05-30 06:05:00 +02:00
|
|
|
}
|
|
|
|
|
2019-04-17 18:52:08 +02:00
|
|
|
// TreeWalkPool - pool of treeWalk go routines.
|
2016-06-05 20:55:45 +02:00
|
|
|
// A treeWalk is added to the pool by Set() and removed either by
|
|
|
|
// doing a Release() or if the concerned timer goes off.
|
|
|
|
// treeWalkPool's purpose is to maintain active treeWalk go-routines in a map so that
|
|
|
|
// it can be looked up across related list calls.
|
2019-04-17 18:52:08 +02:00
|
|
|
type TreeWalkPool struct {
|
2020-05-04 07:35:40 +02:00
|
|
|
sync.Mutex
|
2016-06-05 20:55:45 +02:00
|
|
|
pool map[listParams][]treeWalk
|
2016-05-30 06:05:00 +02:00
|
|
|
timeOut time.Duration
|
|
|
|
}
|
|
|
|
|
2019-04-17 18:52:08 +02:00
|
|
|
// NewTreeWalkPool - initialize new tree walk pool.
|
|
|
|
func NewTreeWalkPool(timeout time.Duration) *TreeWalkPool {
|
|
|
|
tPool := &TreeWalkPool{
|
2016-06-05 20:55:45 +02:00
|
|
|
pool: make(map[listParams][]treeWalk),
|
2016-05-30 06:05:00 +02:00
|
|
|
timeOut: timeout,
|
|
|
|
}
|
|
|
|
return tPool
|
|
|
|
}
|
|
|
|
|
2016-06-05 20:55:45 +02:00
|
|
|
// Release - selects a treeWalk from the pool based on the input
|
2019-04-17 18:52:08 +02:00
|
|
|
// listParams, removes it from the pool, and returns the TreeWalkResult
|
2016-06-05 20:55:45 +02:00
|
|
|
// channel.
|
2020-06-17 18:52:07 +02:00
|
|
|
// Returns nil if listParams does not have an associated treeWalk.
|
2020-05-04 07:35:40 +02:00
|
|
|
func (t *TreeWalkPool) Release(params listParams) (resultCh chan TreeWalkResult, endWalkCh chan struct{}) {
|
|
|
|
t.Lock()
|
|
|
|
defer t.Unlock()
|
2016-06-03 20:33:50 +02:00
|
|
|
walks, ok := t.pool[params] // Pick the valid walks.
|
2016-05-30 06:05:00 +02:00
|
|
|
if ok {
|
2016-06-03 20:33:50 +02:00
|
|
|
if len(walks) > 0 {
|
|
|
|
// Pop out the first valid walk entry.
|
|
|
|
walk := walks[0]
|
2020-06-17 18:52:07 +02:00
|
|
|
walks[0] = treeWalk{} // clear references.
|
2016-06-03 20:33:50 +02:00
|
|
|
walks = walks[1:]
|
|
|
|
if len(walks) > 0 {
|
|
|
|
t.pool[params] = walks
|
2016-05-30 06:05:00 +02:00
|
|
|
} else {
|
|
|
|
delete(t.pool, params)
|
|
|
|
}
|
2016-06-05 20:55:45 +02:00
|
|
|
walk.endTimerCh <- struct{}{}
|
|
|
|
return walk.resultCh, walk.endWalkCh
|
2016-05-30 06:05:00 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
// Release return nil if params not found.
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
|
2016-06-05 20:55:45 +02:00
|
|
|
// Set - adds a treeWalk to the treeWalkPool.
|
|
|
|
// Also starts a timer go-routine that ends when:
|
|
|
|
// 1) time.After() expires after t.timeOut seconds.
|
|
|
|
// The expiration is needed so that the treeWalk go-routine resources are freed after a timeout
|
|
|
|
// if the S3 client does only partial listing of objects.
|
2020-06-17 18:52:07 +02:00
|
|
|
// 2) Release() signals the timer go-routine to end on endTimerCh.
|
2016-06-05 20:55:45 +02:00
|
|
|
// During listing the timer should not timeout and end the treeWalk go-routine, hence the
|
|
|
|
// timer go-routine should be ended.
|
2020-05-04 07:35:40 +02:00
|
|
|
func (t *TreeWalkPool) Set(params listParams, resultCh chan TreeWalkResult, endWalkCh chan struct{}) {
|
|
|
|
t.Lock()
|
|
|
|
defer t.Unlock()
|
2016-05-30 06:05:00 +02:00
|
|
|
|
2020-06-17 18:52:07 +02:00
|
|
|
// If we are above the limit delete at least one entry from the pool.
|
|
|
|
if len(t.pool) > treeWalkEntryLimit {
|
|
|
|
age := time.Now()
|
|
|
|
var oldest listParams
|
|
|
|
for k, v := range t.pool {
|
|
|
|
if len(v) == 0 {
|
|
|
|
delete(t.pool, k)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
// The first element is the oldest, so we only check that.
|
|
|
|
if v[0].added.Before(age) {
|
|
|
|
oldest = k
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Invalidate and delete oldest.
|
|
|
|
if walks, ok := t.pool[oldest]; ok {
|
|
|
|
walk := walks[0]
|
|
|
|
walks[0] = treeWalk{} // clear references.
|
|
|
|
walks = walks[1:]
|
|
|
|
if len(walks) > 0 {
|
|
|
|
t.pool[params] = walks
|
|
|
|
} else {
|
|
|
|
delete(t.pool, params)
|
|
|
|
}
|
|
|
|
walk.endTimerCh <- struct{}{}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-06-03 20:33:50 +02:00
|
|
|
// Should be a buffered channel so that Release() never blocks.
|
2016-06-05 20:55:45 +02:00
|
|
|
endTimerCh := make(chan struct{}, 1)
|
|
|
|
walkInfo := treeWalk{
|
2020-06-17 18:52:07 +02:00
|
|
|
added: UTCNow(),
|
2016-06-05 20:55:45 +02:00
|
|
|
resultCh: resultCh,
|
|
|
|
endWalkCh: endWalkCh,
|
|
|
|
endTimerCh: endTimerCh,
|
2016-06-03 20:33:50 +02:00
|
|
|
}
|
|
|
|
// Append new walk info.
|
2020-06-17 18:52:07 +02:00
|
|
|
walks := t.pool[params]
|
|
|
|
if len(walks) < treeWalkSameEntryLimit {
|
|
|
|
t.pool[params] = append(walks, walkInfo)
|
|
|
|
} else {
|
|
|
|
// We are at limit, invalidate oldest, move list down and add new as last.
|
|
|
|
walks[0].endTimerCh <- struct{}{}
|
|
|
|
copy(walks, walks[1:])
|
|
|
|
walks[len(walks)-1] = walkInfo
|
|
|
|
}
|
2016-05-30 06:05:00 +02:00
|
|
|
|
2016-06-05 20:55:45 +02:00
|
|
|
// Timer go-routine which times out after t.timeOut seconds.
|
|
|
|
go func(endTimerCh <-chan struct{}) {
|
2016-05-30 06:05:00 +02:00
|
|
|
select {
|
|
|
|
// Wait until timeOut
|
|
|
|
case <-time.After(t.timeOut):
|
2016-06-05 20:55:45 +02:00
|
|
|
// Timeout has expired. Remove the treeWalk from treeWalkPool and
|
|
|
|
// end the treeWalk go-routine.
|
2020-05-04 07:35:40 +02:00
|
|
|
t.Lock()
|
2016-06-05 20:55:45 +02:00
|
|
|
walks, ok := t.pool[params]
|
2016-06-03 20:33:50 +02:00
|
|
|
if ok {
|
2019-05-14 22:49:10 +02:00
|
|
|
// Trick of filtering without allocating
|
|
|
|
// https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating
|
|
|
|
nwalks := walks[:0]
|
2016-06-03 20:33:50 +02:00
|
|
|
// Look for walkInfo, remove it from the walks list.
|
2019-05-14 22:49:10 +02:00
|
|
|
for _, walk := range walks {
|
|
|
|
if !reflect.DeepEqual(walk, walkInfo) {
|
|
|
|
nwalks = append(nwalks, walk)
|
2016-06-03 20:33:50 +02:00
|
|
|
}
|
|
|
|
}
|
2019-05-14 22:49:10 +02:00
|
|
|
if len(nwalks) == 0 {
|
2016-06-05 20:55:45 +02:00
|
|
|
// No more treeWalk go-routines associated with listParams
|
|
|
|
// hence remove map entry.
|
2016-06-03 20:33:50 +02:00
|
|
|
delete(t.pool, params)
|
2016-06-05 20:55:45 +02:00
|
|
|
} else {
|
|
|
|
// There are more treeWalk go-routines associated with listParams
|
|
|
|
// hence save the list in the map.
|
2019-05-14 22:49:10 +02:00
|
|
|
t.pool[params] = nwalks
|
2016-06-03 20:33:50 +02:00
|
|
|
}
|
2016-05-30 06:05:00 +02:00
|
|
|
}
|
2016-06-05 20:55:45 +02:00
|
|
|
// Signal the treeWalk go-routine to die.
|
|
|
|
close(endWalkCh)
|
2020-05-04 07:35:40 +02:00
|
|
|
t.Unlock()
|
2016-06-05 20:55:45 +02:00
|
|
|
case <-endTimerCh:
|
2016-05-30 06:05:00 +02:00
|
|
|
return
|
|
|
|
}
|
2016-06-05 20:55:45 +02:00
|
|
|
}(endTimerCh)
|
2016-05-30 06:05:00 +02:00
|
|
|
}
|