2016-08-17 20:36:33 +02:00
|
|
|
/*
|
|
|
|
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2016-08-19 01:23:42 +02:00
|
|
|
package cmd
|
2016-08-17 20:36:33 +02:00
|
|
|
|
|
|
|
import (
|
|
|
|
"sort"
|
|
|
|
"strings"
|
|
|
|
)
|
|
|
|
|
2016-10-15 04:57:40 +02:00
|
|
|
func listDirHealFactory(isLeaf isLeafFunc, disks ...StorageAPI) listDirFunc {
|
2016-08-17 20:36:33 +02:00
|
|
|
// Returns sorted merged entries from all the disks.
|
2016-10-15 04:57:40 +02:00
|
|
|
listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool, err error) {
|
2016-08-17 20:36:33 +02:00
|
|
|
for _, disk := range disks {
|
2016-10-15 04:57:40 +02:00
|
|
|
if disk == nil {
|
|
|
|
continue
|
|
|
|
}
|
2016-08-17 20:36:33 +02:00
|
|
|
var entries []string
|
|
|
|
var newEntries []string
|
|
|
|
entries, err = disk.ListDir(bucket, prefixDir)
|
|
|
|
if err != nil {
|
|
|
|
continue
|
|
|
|
}
|
2016-10-15 04:57:40 +02:00
|
|
|
// Listing needs to be sorted.
|
|
|
|
sort.Strings(entries)
|
|
|
|
|
|
|
|
// Filter entries that have the prefix prefixEntry.
|
|
|
|
entries = filterMatchingPrefix(entries, prefixEntry)
|
2016-08-17 20:36:33 +02:00
|
|
|
|
2016-10-15 04:57:40 +02:00
|
|
|
// isLeaf() check has to happen here so that trailing "/" for objects can be removed.
|
2016-08-17 20:36:33 +02:00
|
|
|
for i, entry := range entries {
|
2016-10-15 04:57:40 +02:00
|
|
|
if isLeaf(bucket, pathJoin(prefixDir, entry)) {
|
|
|
|
entries[i] = strings.TrimSuffix(entry, slashSeparator)
|
2016-08-17 20:36:33 +02:00
|
|
|
}
|
|
|
|
}
|
2016-10-15 04:57:40 +02:00
|
|
|
// Sort again after removing trailing "/" for objects as the previous sort
|
|
|
|
// does not hold good anymore.
|
|
|
|
sort.Strings(entries)
|
|
|
|
if len(mergedEntries) == 0 {
|
2016-08-17 20:36:33 +02:00
|
|
|
// For the first successful disk.ListDir()
|
2016-10-15 04:57:40 +02:00
|
|
|
mergedEntries = entries
|
|
|
|
sort.Strings(mergedEntries)
|
2016-08-17 20:36:33 +02:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
// find elements in entries which are not in mergedentries
|
|
|
|
for _, entry := range entries {
|
2016-10-15 04:57:40 +02:00
|
|
|
idx := sort.SearchStrings(mergedEntries, entry)
|
|
|
|
if mergedEntries[idx] == entry {
|
2016-08-17 20:36:33 +02:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
newEntries = append(newEntries, entry)
|
|
|
|
}
|
|
|
|
if len(newEntries) > 0 {
|
|
|
|
// Merge the entries and sort it.
|
2016-10-15 04:57:40 +02:00
|
|
|
mergedEntries = append(mergedEntries, newEntries...)
|
|
|
|
sort.Strings(mergedEntries)
|
2016-08-17 20:36:33 +02:00
|
|
|
}
|
|
|
|
}
|
2016-10-15 04:57:40 +02:00
|
|
|
return mergedEntries, false, nil
|
2016-08-17 20:36:33 +02:00
|
|
|
}
|
|
|
|
return listDir
|
|
|
|
}
|
|
|
|
|
|
|
|
// listObjectsHeal - wrapper function implemented over file tree walk.
|
|
|
|
func (xl xlObjects) listObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
|
|
|
|
// Default is recursive, if delimiter is set then list non recursive.
|
|
|
|
recursive := true
|
|
|
|
if delimiter == slashSeparator {
|
|
|
|
recursive = false
|
|
|
|
}
|
|
|
|
|
|
|
|
// "heal" true for listObjectsHeal() and false for listObjects()
|
|
|
|
heal := true
|
|
|
|
walkResultCh, endWalkCh := xl.listPool.Release(listParams{bucket, recursive, marker, prefix, heal})
|
|
|
|
if walkResultCh == nil {
|
|
|
|
endWalkCh = make(chan struct{})
|
2016-10-15 04:57:40 +02:00
|
|
|
isLeaf := xl.isObject
|
|
|
|
listDir := listDirHealFactory(isLeaf, xl.storageDisks...)
|
2016-08-17 20:36:33 +02:00
|
|
|
walkResultCh = startTreeWalk(bucket, prefix, marker, recursive, listDir, nil, endWalkCh)
|
|
|
|
}
|
|
|
|
|
|
|
|
var objInfos []ObjectInfo
|
|
|
|
var eof bool
|
|
|
|
var nextMarker string
|
|
|
|
for i := 0; i < maxKeys; {
|
|
|
|
walkResult, ok := <-walkResultCh
|
|
|
|
if !ok {
|
|
|
|
// Closed channel.
|
|
|
|
eof = true
|
|
|
|
break
|
|
|
|
}
|
|
|
|
// For any walk error return right away.
|
|
|
|
if walkResult.err != nil {
|
|
|
|
// File not found is a valid case.
|
|
|
|
if walkResult.err == errFileNotFound {
|
|
|
|
return ListObjectsInfo{}, nil
|
|
|
|
}
|
|
|
|
return ListObjectsInfo{}, toObjectErr(walkResult.err, bucket, prefix)
|
|
|
|
}
|
|
|
|
entry := walkResult.entry
|
|
|
|
var objInfo ObjectInfo
|
|
|
|
if strings.HasSuffix(entry, slashSeparator) {
|
|
|
|
// Object name needs to be full path.
|
|
|
|
objInfo.Bucket = bucket
|
|
|
|
objInfo.Name = entry
|
|
|
|
objInfo.IsDir = true
|
|
|
|
} else {
|
|
|
|
objInfo.Bucket = bucket
|
|
|
|
objInfo.Name = entry
|
|
|
|
}
|
|
|
|
nextMarker = objInfo.Name
|
|
|
|
objInfos = append(objInfos, objInfo)
|
|
|
|
i++
|
|
|
|
if walkResult.end == true {
|
|
|
|
eof = true
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
params := listParams{bucket, recursive, nextMarker, prefix, heal}
|
|
|
|
if !eof {
|
|
|
|
xl.listPool.Set(params, walkResultCh, endWalkCh)
|
|
|
|
}
|
|
|
|
|
|
|
|
result := ListObjectsInfo{IsTruncated: !eof}
|
|
|
|
for _, objInfo := range objInfos {
|
|
|
|
result.NextMarker = objInfo.Name
|
|
|
|
if objInfo.IsDir {
|
|
|
|
result.Prefixes = append(result.Prefixes, objInfo.Name)
|
|
|
|
continue
|
|
|
|
}
|
2016-08-31 20:39:08 +02:00
|
|
|
|
2016-09-28 03:28:46 +02:00
|
|
|
// get a random ID for lock instrumentation.
|
2016-08-31 20:39:08 +02:00
|
|
|
opsID := getOpsID()
|
|
|
|
|
2016-08-24 19:13:01 +02:00
|
|
|
// Check if the current object needs healing
|
2016-08-31 20:39:08 +02:00
|
|
|
nsMutex.RLock(bucket, objInfo.Name, opsID)
|
2016-08-24 19:13:01 +02:00
|
|
|
partsMetadata, errs := readAllXLMetadata(xl.storageDisks, bucket, objInfo.Name)
|
|
|
|
if xlShouldHeal(partsMetadata, errs) {
|
|
|
|
result.Objects = append(result.Objects, ObjectInfo{
|
|
|
|
Name: objInfo.Name,
|
|
|
|
ModTime: objInfo.ModTime,
|
|
|
|
Size: objInfo.Size,
|
|
|
|
IsDir: false,
|
|
|
|
})
|
|
|
|
}
|
2016-08-31 20:39:08 +02:00
|
|
|
nsMutex.RUnlock(bucket, objInfo.Name, opsID)
|
2016-08-17 20:36:33 +02:00
|
|
|
}
|
|
|
|
return result, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// ListObjects - list all objects at prefix, delimited by '/'.
|
|
|
|
func (xl xlObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
|
|
|
|
// Verify if bucket is valid.
|
|
|
|
if !IsValidBucketName(bucket) {
|
2016-08-25 18:39:01 +02:00
|
|
|
return ListObjectsInfo{}, traceError(BucketNameInvalid{Bucket: bucket})
|
2016-08-17 20:36:33 +02:00
|
|
|
}
|
|
|
|
// Verify if bucket exists.
|
|
|
|
if !xl.isBucketExist(bucket) {
|
2016-08-25 18:39:01 +02:00
|
|
|
return ListObjectsInfo{}, traceError(BucketNotFound{Bucket: bucket})
|
2016-08-17 20:36:33 +02:00
|
|
|
}
|
|
|
|
if !IsValidObjectPrefix(prefix) {
|
2016-08-25 18:39:01 +02:00
|
|
|
return ListObjectsInfo{}, traceError(ObjectNameInvalid{Bucket: bucket, Object: prefix})
|
2016-08-17 20:36:33 +02:00
|
|
|
}
|
|
|
|
// Verify if delimiter is anything other than '/', which we do not support.
|
|
|
|
if delimiter != "" && delimiter != slashSeparator {
|
2016-08-25 18:39:01 +02:00
|
|
|
return ListObjectsInfo{}, traceError(UnsupportedDelimiter{
|
2016-08-17 20:36:33 +02:00
|
|
|
Delimiter: delimiter,
|
2016-08-25 18:39:01 +02:00
|
|
|
})
|
2016-08-17 20:36:33 +02:00
|
|
|
}
|
|
|
|
// Verify if marker has prefix.
|
|
|
|
if marker != "" {
|
|
|
|
if !strings.HasPrefix(marker, prefix) {
|
2016-08-25 18:39:01 +02:00
|
|
|
return ListObjectsInfo{}, traceError(InvalidMarkerPrefixCombination{
|
2016-08-17 20:36:33 +02:00
|
|
|
Marker: marker,
|
|
|
|
Prefix: prefix,
|
2016-08-25 18:39:01 +02:00
|
|
|
})
|
2016-08-17 20:36:33 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// With max keys of zero we have reached eof, return right here.
|
|
|
|
if maxKeys == 0 {
|
|
|
|
return ListObjectsInfo{}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// For delimiter and prefix as '/' we do not list anything at all
|
|
|
|
// since according to s3 spec we stop at the 'delimiter' along
|
|
|
|
// with the prefix. On a flat namespace with 'prefix' as '/'
|
|
|
|
// we don't have any entries, since all the keys are of form 'keyName/...'
|
|
|
|
if delimiter == slashSeparator && prefix == slashSeparator {
|
|
|
|
return ListObjectsInfo{}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Over flowing count - reset to maxObjectList.
|
|
|
|
if maxKeys < 0 || maxKeys > maxObjectList {
|
|
|
|
maxKeys = maxObjectList
|
|
|
|
}
|
|
|
|
|
|
|
|
// Initiate a list operation, if successful filter and return quickly.
|
|
|
|
listObjInfo, err := xl.listObjectsHeal(bucket, prefix, marker, delimiter, maxKeys)
|
|
|
|
if err == nil {
|
|
|
|
// We got the entries successfully return.
|
|
|
|
return listObjInfo, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Return error at the end.
|
|
|
|
return ListObjectsInfo{}, toObjectErr(err, bucket, prefix)
|
|
|
|
}
|