2016-05-31 01:51:59 +02:00
|
|
|
/*
|
2019-04-09 20:39:42 +02:00
|
|
|
* MinIO Cloud Storage, (C) 2016 MinIO, Inc.
|
2016-05-31 01:51:59 +02:00
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2016-08-19 01:23:42 +02:00
|
|
|
package cmd
|
2016-05-21 05:48:47 +02:00
|
|
|
|
2018-02-01 19:47:49 +01:00
|
|
|
import (
|
2018-03-14 20:01:47 +01:00
|
|
|
"context"
|
2018-02-01 19:47:49 +01:00
|
|
|
"sort"
|
|
|
|
)
|
2017-11-25 20:58:29 +01:00
|
|
|
|
2017-01-27 00:39:22 +01:00
|
|
|
// Returns function "listDir" of the type listDirFunc.
|
2018-02-01 19:47:49 +01:00
|
|
|
// disks - used for doing disk.ListDir()
|
2019-05-02 07:06:57 +02:00
|
|
|
func listDirFactory(ctx context.Context, disks ...StorageAPI) ListDirFunc {
|
2018-02-01 19:47:49 +01:00
|
|
|
// Returns sorted merged entries from all the disks.
|
2019-05-02 07:06:57 +02:00
|
|
|
listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string) {
|
2017-01-27 00:39:22 +01:00
|
|
|
for _, disk := range disks {
|
|
|
|
if disk == nil {
|
|
|
|
continue
|
|
|
|
}
|
2018-02-01 19:47:49 +01:00
|
|
|
var entries []string
|
|
|
|
var newEntries []string
|
2018-07-28 00:32:19 +02:00
|
|
|
var err error
|
2019-04-23 23:54:28 +02:00
|
|
|
entries, err = disk.ListDir(bucket, prefixDir, -1, xlMetaJSONFile)
|
2017-03-20 19:09:05 +01:00
|
|
|
if err != nil {
|
2018-07-28 00:32:19 +02:00
|
|
|
continue
|
2017-01-27 00:39:22 +01:00
|
|
|
}
|
2017-03-20 19:09:05 +01:00
|
|
|
|
2018-02-01 19:47:49 +01:00
|
|
|
// Find elements in entries which are not in mergedEntries
|
|
|
|
for _, entry := range entries {
|
|
|
|
idx := sort.SearchStrings(mergedEntries, entry)
|
|
|
|
// if entry is already present in mergedEntries don't add.
|
|
|
|
if idx < len(mergedEntries) && mergedEntries[idx] == entry {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
newEntries = append(newEntries, entry)
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(newEntries) > 0 {
|
|
|
|
// Merge the entries and sort it.
|
|
|
|
mergedEntries = append(mergedEntries, newEntries...)
|
|
|
|
sort.Strings(mergedEntries)
|
|
|
|
}
|
2017-01-27 00:39:22 +01:00
|
|
|
}
|
2019-05-02 07:06:57 +02:00
|
|
|
return filterMatchingPrefix(mergedEntries, prefixEntry)
|
2017-01-27 00:39:22 +01:00
|
|
|
}
|
|
|
|
return listDir
|
|
|
|
}
|
|
|
|
|
2016-05-31 01:51:59 +02:00
|
|
|
// listObjects - wrapper function implemented over file tree walk.
|
2018-04-06 00:04:40 +02:00
|
|
|
func (xl xlObjects) listObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
|
2016-05-21 05:48:47 +02:00
|
|
|
// Default is recursive, if delimiter is set then list non recursive.
|
|
|
|
recursive := true
|
2019-08-06 21:08:58 +02:00
|
|
|
if delimiter == SlashSeparator {
|
2016-05-21 05:48:47 +02:00
|
|
|
recursive = false
|
|
|
|
}
|
|
|
|
|
2019-06-09 07:14:07 +02:00
|
|
|
walkResultCh, endWalkCh := xl.listPool.Release(listParams{bucket, recursive, marker, prefix, false})
|
2016-06-05 20:55:45 +02:00
|
|
|
if walkResultCh == nil {
|
|
|
|
endWalkCh = make(chan struct{})
|
2019-05-02 07:06:57 +02:00
|
|
|
listDir := listDirFactory(ctx, xl.getLoadBalancedDisks()...)
|
|
|
|
walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, endWalkCh)
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
2016-05-30 06:05:00 +02:00
|
|
|
|
2016-05-21 05:48:47 +02:00
|
|
|
var objInfos []ObjectInfo
|
|
|
|
var eof bool
|
|
|
|
var nextMarker string
|
|
|
|
for i := 0; i < maxKeys; {
|
2018-02-10 00:19:30 +01:00
|
|
|
|
2016-06-05 20:55:45 +02:00
|
|
|
walkResult, ok := <-walkResultCh
|
2016-05-21 05:48:47 +02:00
|
|
|
if !ok {
|
|
|
|
// Closed channel.
|
|
|
|
eof = true
|
|
|
|
break
|
|
|
|
}
|
2016-05-25 18:22:39 +02:00
|
|
|
entry := walkResult.entry
|
|
|
|
var objInfo ObjectInfo
|
2019-12-06 08:16:06 +01:00
|
|
|
if HasSuffix(entry, SlashSeparator) {
|
2016-05-25 18:22:39 +02:00
|
|
|
// Object name needs to be full path.
|
|
|
|
objInfo.Bucket = bucket
|
|
|
|
objInfo.Name = entry
|
|
|
|
objInfo.IsDir = true
|
|
|
|
} else {
|
|
|
|
// Set the Mode to a "regular" file.
|
|
|
|
var err error
|
2018-04-06 00:04:40 +02:00
|
|
|
objInfo, err = xl.getObjectInfo(ctx, bucket, entry)
|
2016-05-25 18:22:39 +02:00
|
|
|
if err != nil {
|
2018-02-16 02:45:57 +01:00
|
|
|
// Ignore errFileNotFound as the object might have got
|
|
|
|
// deleted in the interim period of listing and getObjectInfo(),
|
|
|
|
// ignore quorum error as it might be an entry from an outdated disk.
|
2019-04-23 23:54:28 +02:00
|
|
|
if IsErrIgnored(err, []error{
|
|
|
|
errFileNotFound,
|
|
|
|
errXLReadQuorum,
|
|
|
|
}...) {
|
2016-06-27 07:10:22 +02:00
|
|
|
continue
|
|
|
|
}
|
2017-06-22 04:53:09 +02:00
|
|
|
return loi, toObjectErr(err, bucket, prefix)
|
2016-05-25 18:22:39 +02:00
|
|
|
}
|
|
|
|
}
|
2016-05-21 05:48:47 +02:00
|
|
|
nextMarker = objInfo.Name
|
|
|
|
objInfos = append(objInfos, objInfo)
|
2016-05-30 06:05:00 +02:00
|
|
|
i++
|
2016-08-16 16:57:14 +02:00
|
|
|
if walkResult.end {
|
2016-05-21 05:48:47 +02:00
|
|
|
eof = true
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
2016-05-30 06:05:00 +02:00
|
|
|
|
2019-06-09 07:14:07 +02:00
|
|
|
params := listParams{bucket, recursive, nextMarker, prefix, false}
|
2016-05-21 05:48:47 +02:00
|
|
|
if !eof {
|
2016-06-05 20:55:45 +02:00
|
|
|
xl.listPool.Set(params, walkResultCh, endWalkCh)
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
|
|
|
|
2018-12-20 22:30:25 +01:00
|
|
|
result := ListObjectsInfo{}
|
2016-05-21 05:48:47 +02:00
|
|
|
for _, objInfo := range objInfos {
|
2019-08-06 21:08:58 +02:00
|
|
|
if objInfo.IsDir && delimiter == SlashSeparator {
|
2016-06-01 07:10:55 +02:00
|
|
|
result.Prefixes = append(result.Prefixes, objInfo.Name)
|
|
|
|
continue
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
2016-11-13 20:48:02 +01:00
|
|
|
result.Objects = append(result.Objects, objInfo)
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
2018-12-20 22:30:25 +01:00
|
|
|
|
|
|
|
if !eof {
|
|
|
|
result.IsTruncated = true
|
|
|
|
if len(objInfos) > 0 {
|
|
|
|
result.NextMarker = objInfos[len(objInfos)-1].Name
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-05-21 05:48:47 +02:00
|
|
|
return result, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// ListObjects - list all objects at prefix, delimited by '/'.
|
2018-03-14 20:01:47 +01:00
|
|
|
func (xl xlObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
|
2018-04-06 00:04:40 +02:00
|
|
|
if err := checkListObjsArgs(ctx, bucket, prefix, marker, delimiter, xl); err != nil {
|
2017-06-22 04:53:09 +02:00
|
|
|
return loi, err
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// With max keys of zero we have reached eof, return right here.
|
|
|
|
if maxKeys == 0 {
|
2017-06-22 04:53:09 +02:00
|
|
|
return loi, nil
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
|
|
|
|
2018-02-10 00:19:30 +01:00
|
|
|
// Marker is set validate pre-condition.
|
|
|
|
if marker != "" {
|
|
|
|
// Marker not common with prefix is not implemented.Send an empty response
|
2019-12-06 08:16:06 +01:00
|
|
|
if !HasPrefix(marker, prefix) {
|
2018-02-10 00:19:30 +01:00
|
|
|
return ListObjectsInfo{}, e
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-05-28 00:43:51 +02:00
|
|
|
// For delimiter and prefix as '/' we do not list anything at all
|
|
|
|
// since according to s3 spec we stop at the 'delimiter' along
|
|
|
|
// with the prefix. On a flat namespace with 'prefix' as '/'
|
|
|
|
// we don't have any entries, since all the keys are of form 'keyName/...'
|
2019-08-06 21:08:58 +02:00
|
|
|
if delimiter == SlashSeparator && prefix == SlashSeparator {
|
2017-06-22 04:53:09 +02:00
|
|
|
return loi, nil
|
2016-05-28 00:43:51 +02:00
|
|
|
}
|
|
|
|
|
2016-05-21 05:48:47 +02:00
|
|
|
// Over flowing count - reset to maxObjectList.
|
|
|
|
if maxKeys < 0 || maxKeys > maxObjectList {
|
|
|
|
maxKeys = maxObjectList
|
|
|
|
}
|
|
|
|
|
|
|
|
// Initiate a list operation, if successful filter and return quickly.
|
2018-04-06 00:04:40 +02:00
|
|
|
listObjInfo, err := xl.listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
|
2016-05-21 05:48:47 +02:00
|
|
|
if err == nil {
|
|
|
|
// We got the entries successfully return.
|
|
|
|
return listObjInfo, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Return error at the end.
|
2017-06-22 04:53:09 +02:00
|
|
|
return loi, toObjectErr(err, bucket, prefix)
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|