2016-05-05 21:51:56 +02:00
|
|
|
/*
|
|
|
|
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
|
|
|
"sort"
|
|
|
|
"strings"
|
|
|
|
)
|
|
|
|
|
2016-07-08 07:10:27 +02:00
|
|
|
// list of all errors that can be ignored in tree walk operation.
|
|
|
|
var walkResultIgnoredErrs = []error{
|
|
|
|
errFileNotFound,
|
|
|
|
errVolumeNotFound,
|
|
|
|
errDiskNotFound,
|
|
|
|
errDiskAccessDenied,
|
|
|
|
errFaultyDisk,
|
|
|
|
}
|
|
|
|
|
2016-05-05 21:51:56 +02:00
|
|
|
// Tree walk result carries results of tree walking.
|
2016-06-05 20:55:45 +02:00
|
|
|
type treeWalkResult struct {
|
2016-05-25 18:22:39 +02:00
|
|
|
entry string
|
|
|
|
err error
|
|
|
|
end bool
|
2016-05-05 21:51:56 +02:00
|
|
|
}
|
|
|
|
|
2016-07-04 10:49:27 +02:00
|
|
|
// "listDir" function of type listDirFunc returned by listDirFactory() - explained below.
|
|
|
|
type listDirFunc func(bucket, prefixDir, prefixEntry string) (entries []string, err error)
|
|
|
|
|
|
|
|
// Returns function "listDir" of the type listDirFunc.
|
|
|
|
// isLeaf - is used by listDir function to check if an entry is a leaf or non-leaf entry.
|
|
|
|
// disks - used for doing disk.ListDir(). FS passes single disk argument, XL passes a list of disks.
|
|
|
|
func listDirFactory(isLeaf func(string, string) bool, disks ...StorageAPI) listDirFunc {
|
|
|
|
// listDir - lists all the entries at a given prefix and given entry in the prefix.
|
|
|
|
// isLeaf is used to detect if an entry is a leaf entry. There are four scenarios where isLeaf
|
|
|
|
// should behave differently:
|
|
|
|
// 1. FS backend object listing - isLeaf is true if the entry has a trailing "/"
|
|
|
|
// 2. FS backend multipart listing - isLeaf is true if the entry is a directory and contains uploads.json
|
|
|
|
// 3. XL backend object listing - isLeaf is true if the entry is a directory and contains xl.json
|
|
|
|
// 4. XL backend multipart listing - isLeaf is true if the entry is a directory and contains uploads.json
|
|
|
|
listDir := func(bucket, prefixDir, prefixEntry string) (entries []string, err error) {
|
|
|
|
for _, disk := range disks {
|
|
|
|
if disk == nil {
|
2016-06-03 07:49:27 +02:00
|
|
|
continue
|
|
|
|
}
|
2016-07-04 10:49:27 +02:00
|
|
|
entries, err = disk.ListDir(bucket, prefixDir)
|
2016-07-08 07:10:27 +02:00
|
|
|
if err == nil {
|
|
|
|
// Skip the entries which do not match the prefixEntry.
|
|
|
|
for i, entry := range entries {
|
|
|
|
if !strings.HasPrefix(entry, prefixEntry) {
|
|
|
|
entries[i] = ""
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if isLeaf(bucket, pathJoin(prefixDir, entry)) {
|
|
|
|
entries[i] = strings.TrimSuffix(entry, slashSeparator)
|
|
|
|
}
|
2016-07-04 10:49:27 +02:00
|
|
|
}
|
2016-07-08 07:10:27 +02:00
|
|
|
sort.Strings(entries)
|
|
|
|
// Skip the empty strings
|
|
|
|
for len(entries) > 0 && entries[0] == "" {
|
|
|
|
entries = entries[1:]
|
2016-07-04 10:49:27 +02:00
|
|
|
}
|
2016-07-08 07:10:27 +02:00
|
|
|
return entries, nil
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
2016-07-08 07:10:27 +02:00
|
|
|
// For any reason disk was deleted or goes offline, continue
|
|
|
|
// and list from other disks if possible.
|
|
|
|
if isErrIgnored(err, walkResultIgnoredErrs) {
|
|
|
|
continue
|
2016-07-04 10:49:27 +02:00
|
|
|
}
|
2016-07-08 07:10:27 +02:00
|
|
|
break
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
2016-07-04 10:49:27 +02:00
|
|
|
// Return error at the end.
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return listDir
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
|
|
|
|
2016-07-04 10:49:27 +02:00
|
|
|
// treeWalk walks directory tree recursively pushing treeWalkResult into the channel as and when it encounters files.
|
|
|
|
func doTreeWalk(bucket, prefixDir, entryPrefixMatch, marker string, recursive bool, listDir listDirFunc, resultCh chan treeWalkResult, endWalkCh chan struct{}, isEnd bool) error {
|
2016-05-05 21:51:56 +02:00
|
|
|
// Example:
|
|
|
|
// if prefixDir="one/two/three/" and marker="four/five.txt" treeWalk is recursively
|
|
|
|
// called with prefixDir="one/two/three/four/" and marker="five.txt"
|
|
|
|
|
|
|
|
var markerBase, markerDir string
|
|
|
|
if marker != "" {
|
|
|
|
// Ex: if marker="four/five.txt", markerDir="four/" markerBase="five.txt"
|
|
|
|
markerSplit := strings.SplitN(marker, slashSeparator, 2)
|
|
|
|
markerDir = markerSplit[0]
|
|
|
|
if len(markerSplit) == 2 {
|
|
|
|
markerDir += slashSeparator
|
|
|
|
markerBase = markerSplit[1]
|
|
|
|
}
|
|
|
|
}
|
2016-07-04 10:49:27 +02:00
|
|
|
entries, err := listDir(bucket, prefixDir, entryPrefixMatch)
|
2016-05-05 21:51:56 +02:00
|
|
|
if err != nil {
|
2016-05-30 06:05:00 +02:00
|
|
|
select {
|
2016-06-05 20:55:45 +02:00
|
|
|
case <-endWalkCh:
|
2016-06-03 20:33:50 +02:00
|
|
|
return errWalkAbort
|
2016-06-05 20:55:45 +02:00
|
|
|
case resultCh <- treeWalkResult{err: err}:
|
2016-06-03 20:33:50 +02:00
|
|
|
return err
|
2016-05-30 06:05:00 +02:00
|
|
|
}
|
2016-05-05 21:51:56 +02:00
|
|
|
}
|
2016-06-03 20:33:50 +02:00
|
|
|
// For an empty list return right here.
|
2016-05-05 21:51:56 +02:00
|
|
|
if len(entries) == 0 {
|
2016-06-03 20:33:50 +02:00
|
|
|
return nil
|
2016-05-05 21:51:56 +02:00
|
|
|
}
|
2016-05-21 05:48:47 +02:00
|
|
|
|
2016-05-05 21:51:56 +02:00
|
|
|
// example:
|
|
|
|
// If markerDir="four/" Search() returns the index of "four/" in the sorted
|
|
|
|
// entries list so we skip all the entries till "four/"
|
2016-05-07 11:08:03 +02:00
|
|
|
idx := sort.Search(len(entries), func(i int) bool {
|
2016-05-21 05:48:47 +02:00
|
|
|
return entries[i] >= markerDir
|
2016-05-07 11:08:03 +02:00
|
|
|
})
|
2016-05-05 21:51:56 +02:00
|
|
|
entries = entries[idx:]
|
2016-06-03 20:33:50 +02:00
|
|
|
// For an empty list after search through the entries, return right here.
|
2016-05-30 06:05:00 +02:00
|
|
|
if len(entries) == 0 {
|
2016-06-03 20:33:50 +02:00
|
|
|
return nil
|
2016-05-30 06:05:00 +02:00
|
|
|
}
|
2016-05-05 21:51:56 +02:00
|
|
|
for i, entry := range entries {
|
|
|
|
if i == 0 && markerDir == entry {
|
|
|
|
if !recursive {
|
|
|
|
// Skip as the marker would already be listed in the previous listing.
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if recursive && !strings.HasSuffix(entry, slashSeparator) {
|
|
|
|
// We should not skip for recursive listing and if markerDir is a directory
|
|
|
|
// for ex. if marker is "four/five.txt" markerDir will be "four/" which
|
2016-05-31 01:51:59 +02:00
|
|
|
// should not be skipped, instead it will need to be treeWalk()'ed into.
|
2016-05-05 21:51:56 +02:00
|
|
|
|
|
|
|
// Skip if it is a file though as it would be listed in previous listing.
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if recursive && strings.HasSuffix(entry, slashSeparator) {
|
|
|
|
// If the entry is a directory, we will need recurse into it.
|
|
|
|
markerArg := ""
|
|
|
|
if entry == markerDir {
|
|
|
|
// We need to pass "five.txt" as marker only if we are
|
|
|
|
// recursing into "four/"
|
|
|
|
markerArg = markerBase
|
|
|
|
}
|
|
|
|
prefixMatch := "" // Valid only for first level treeWalk and empty for subdirectories.
|
2016-06-03 20:33:50 +02:00
|
|
|
// markIsEnd is passed to this entry's treeWalk() so that treeWalker.end can be marked
|
|
|
|
// true at the end of the treeWalk stream.
|
|
|
|
markIsEnd := i == len(entries)-1 && isEnd
|
2016-07-04 10:49:27 +02:00
|
|
|
if tErr := doTreeWalk(bucket, pathJoin(prefixDir, entry), prefixMatch, markerArg, recursive, listDir, resultCh, endWalkCh, markIsEnd); tErr != nil {
|
2016-06-03 20:33:50 +02:00
|
|
|
return tErr
|
2016-05-05 21:51:56 +02:00
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
2016-06-03 20:33:50 +02:00
|
|
|
// EOF is set if we are at last entry and the caller indicated we at the end.
|
|
|
|
isEOF := ((i == len(entries)-1) && isEnd)
|
2016-05-30 06:05:00 +02:00
|
|
|
select {
|
2016-06-05 20:55:45 +02:00
|
|
|
case <-endWalkCh:
|
2016-06-03 20:33:50 +02:00
|
|
|
return errWalkAbort
|
2016-06-05 20:55:45 +02:00
|
|
|
case resultCh <- treeWalkResult{entry: pathJoin(prefixDir, entry), end: isEOF}:
|
2016-05-30 06:05:00 +02:00
|
|
|
}
|
|
|
|
}
|
2016-06-03 20:33:50 +02:00
|
|
|
|
|
|
|
// Everything is listed.
|
|
|
|
return nil
|
2016-05-05 21:51:56 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Initiate a new treeWalk in a goroutine.
|
2016-07-04 10:49:27 +02:00
|
|
|
func startTreeWalk(bucket, prefix, marker string, recursive bool, listDir listDirFunc, endWalkCh chan struct{}) chan treeWalkResult {
|
2016-05-05 21:51:56 +02:00
|
|
|
// Example 1
|
|
|
|
// If prefix is "one/two/three/" and marker is "one/two/three/four/five.txt"
|
|
|
|
// treeWalk is called with prefixDir="one/two/three/" and marker="four/five.txt"
|
|
|
|
// and entryPrefixMatch=""
|
|
|
|
|
|
|
|
// Example 2
|
|
|
|
// if prefix is "one/two/th" and marker is "one/two/three/four/five.txt"
|
|
|
|
// treeWalk is called with prefixDir="one/two/" and marker="three/four/five.txt"
|
|
|
|
// and entryPrefixMatch="th"
|
|
|
|
|
2016-06-05 20:55:45 +02:00
|
|
|
resultCh := make(chan treeWalkResult, maxObjectList)
|
2016-05-05 21:51:56 +02:00
|
|
|
entryPrefixMatch := prefix
|
|
|
|
prefixDir := ""
|
|
|
|
lastIndex := strings.LastIndex(prefix, slashSeparator)
|
|
|
|
if lastIndex != -1 {
|
|
|
|
entryPrefixMatch = prefix[lastIndex+1:]
|
|
|
|
prefixDir = prefix[:lastIndex+1]
|
|
|
|
}
|
|
|
|
marker = strings.TrimPrefix(marker, prefixDir)
|
2016-06-03 20:33:50 +02:00
|
|
|
go func() {
|
|
|
|
isEnd := true // Indication to start walking the tree with end as true.
|
2016-07-04 10:49:27 +02:00
|
|
|
doTreeWalk(bucket, prefixDir, entryPrefixMatch, marker, recursive, listDir, resultCh, endWalkCh, isEnd)
|
2016-06-05 20:55:45 +02:00
|
|
|
close(resultCh)
|
2016-06-03 20:33:50 +02:00
|
|
|
}()
|
2016-06-05 20:55:45 +02:00
|
|
|
return resultCh
|
2016-05-05 21:51:56 +02:00
|
|
|
}
|