2016-02-18 09:38:58 +01:00
|
|
|
/*
|
|
|
|
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
package fs
|
|
|
|
|
|
|
|
import (
|
|
|
|
"io"
|
|
|
|
"os"
|
|
|
|
"path/filepath"
|
|
|
|
"sort"
|
|
|
|
"strings"
|
|
|
|
"time"
|
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
2016-03-22 11:18:22 +01:00
|
|
|
// listObjectsLimit - maximum list objects limit.
|
2016-02-18 09:38:58 +01:00
|
|
|
listObjectsLimit = 1000
|
|
|
|
)
|
|
|
|
|
2016-03-22 11:18:22 +01:00
|
|
|
// isDirEmpty - returns whether given directory is empty or not.
|
2016-03-12 04:47:44 +01:00
|
|
|
func isDirEmpty(dirname string) (status bool, err error) {
|
2016-02-18 09:38:58 +01:00
|
|
|
f, err := os.Open(dirname)
|
|
|
|
if err == nil {
|
|
|
|
defer f.Close()
|
|
|
|
if _, err = f.Readdirnames(1); err == io.EOF {
|
|
|
|
status = true
|
|
|
|
err = nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2016-03-22 11:18:22 +01:00
|
|
|
// isDirExist - returns whether given directory is exist or not.
|
2016-03-12 04:47:44 +01:00
|
|
|
func isDirExist(dirname string) (status bool, err error) {
|
2016-02-18 09:38:58 +01:00
|
|
|
fi, err := os.Lstat(dirname)
|
|
|
|
if err == nil {
|
|
|
|
status = fi.IsDir()
|
|
|
|
}
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2016-03-22 11:18:22 +01:00
|
|
|
// byName implements sort.Interface for sorting os.FileInfo list.
|
2016-02-18 09:38:58 +01:00
|
|
|
type byName []os.FileInfo
|
|
|
|
|
2016-03-17 01:30:22 +01:00
|
|
|
func (f byName) Len() int {
|
|
|
|
return len(f)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f byName) Swap(i, j int) {
|
|
|
|
f[i], f[j] = f[j], f[i]
|
|
|
|
}
|
|
|
|
|
2016-02-18 09:38:58 +01:00
|
|
|
func (f byName) Less(i, j int) bool {
|
|
|
|
n1 := f[i].Name()
|
|
|
|
if f[i].IsDir() {
|
|
|
|
n1 = n1 + string(os.PathSeparator)
|
|
|
|
}
|
|
|
|
|
|
|
|
n2 := f[j].Name()
|
|
|
|
if f[j].IsDir() {
|
|
|
|
n2 = n2 + string(os.PathSeparator)
|
|
|
|
}
|
|
|
|
|
|
|
|
return n1 < n2
|
|
|
|
}
|
|
|
|
|
2016-03-17 01:30:22 +01:00
|
|
|
// ObjectInfo - object info.
|
2016-02-18 09:38:58 +01:00
|
|
|
type ObjectInfo struct {
|
2016-03-12 01:31:24 +01:00
|
|
|
Bucket string
|
2016-02-18 09:38:58 +01:00
|
|
|
Name string
|
|
|
|
ModifiedTime time.Time
|
2016-03-12 01:31:24 +01:00
|
|
|
ContentType string
|
|
|
|
MD5Sum string
|
2016-02-18 09:38:58 +01:00
|
|
|
Size int64
|
|
|
|
IsDir bool
|
|
|
|
Err error
|
|
|
|
}
|
|
|
|
|
2016-03-17 01:30:22 +01:00
|
|
|
// Using sort.Search() internally to jump to the file entry containing the prefix.
|
|
|
|
func searchFileInfos(fileInfos []os.FileInfo, x string) int {
|
|
|
|
processFunc := func(i int) bool {
|
|
|
|
return fileInfos[i].Name() >= x
|
|
|
|
}
|
|
|
|
return sort.Search(len(fileInfos), processFunc)
|
|
|
|
}
|
|
|
|
|
2016-03-22 11:18:22 +01:00
|
|
|
// readDir - read 'scanDir' directory. It returns list of ObjectInfo.
|
2016-03-17 01:30:22 +01:00
|
|
|
// Each object name is appended with 'namePrefix'.
|
|
|
|
func readDir(scanDir, namePrefix, queryPrefix string, isFirst bool) (objInfos []ObjectInfo) {
|
2016-02-18 09:38:58 +01:00
|
|
|
f, err := os.Open(scanDir)
|
|
|
|
if err != nil {
|
|
|
|
objInfos = append(objInfos, ObjectInfo{Err: err})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
fis, err := f.Readdir(-1)
|
|
|
|
if err != nil {
|
|
|
|
f.Close()
|
|
|
|
objInfos = append(objInfos, ObjectInfo{Err: err})
|
|
|
|
return
|
|
|
|
}
|
2016-03-17 01:30:22 +01:00
|
|
|
// Close the directory.
|
2016-02-18 09:38:58 +01:00
|
|
|
f.Close()
|
2016-03-12 01:31:24 +01:00
|
|
|
// Sort files by Name.
|
2016-02-18 09:38:58 +01:00
|
|
|
sort.Sort(byName(fis))
|
|
|
|
|
2016-03-17 01:30:22 +01:00
|
|
|
var prefixIndex int
|
|
|
|
// Searching for entries with objectName containing prefix.
|
|
|
|
// Binary search is used for efficient search.
|
|
|
|
if queryPrefix != "" && isFirst {
|
|
|
|
prefixIndex = searchFileInfos(fis, queryPrefix)
|
|
|
|
if prefixIndex == len(fis) {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
if !strings.HasPrefix(fis[prefixIndex].Name(), queryPrefix) {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
fis = fis[prefixIndex:]
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
// Populate []ObjectInfo from []FileInfo.
|
2016-02-18 09:38:58 +01:00
|
|
|
for _, fi := range fis {
|
|
|
|
name := fi.Name()
|
2016-03-17 01:30:22 +01:00
|
|
|
if queryPrefix != "" && isFirst {
|
|
|
|
// If control is here then there is a queryPrefix, and there are objects which satisfies the prefix.
|
|
|
|
// Since the result is sorted, the object names which satisfies query prefix would be stored one after the other.
|
|
|
|
// Push the objectInfo only if its contains the prefix.
|
|
|
|
// This ensures that the channel containing object Info would only has objects with the given queryPrefix.
|
|
|
|
if !strings.HasPrefix(name, queryPrefix) {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
2016-03-12 01:31:24 +01:00
|
|
|
size := fi.Size()
|
|
|
|
modTime := fi.ModTime()
|
|
|
|
isDir := fi.Mode().IsDir()
|
|
|
|
|
|
|
|
// Add prefix if name prefix exists.
|
2016-02-18 09:38:58 +01:00
|
|
|
if namePrefix != "" {
|
|
|
|
name = namePrefix + "/" + name
|
|
|
|
}
|
|
|
|
|
2016-03-12 01:31:24 +01:00
|
|
|
// For directories explicitly end with '/'.
|
|
|
|
if isDir {
|
2016-02-18 09:38:58 +01:00
|
|
|
name += "/"
|
2016-03-22 11:18:22 +01:00
|
|
|
// size is set to '0' for directories explicitly.
|
2016-03-17 01:30:22 +01:00
|
|
|
size = 0
|
2016-03-12 01:31:24 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
|
|
|
|
// Handle symlink by doing an additional stat and follow the link.
|
|
|
|
st, e := os.Stat(filepath.Join(scanDir, name))
|
|
|
|
if e != nil {
|
|
|
|
objInfos = append(objInfos, ObjectInfo{Err: err})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
size = st.Size()
|
|
|
|
modTime = st.ModTime()
|
|
|
|
isDir = st.Mode().IsDir()
|
|
|
|
// For directories explicitly end with '/'.
|
|
|
|
if isDir {
|
|
|
|
name += "/"
|
2016-03-22 11:18:22 +01:00
|
|
|
// size is set to '0' for directories explicitly.
|
|
|
|
size = 0
|
2016-03-12 01:31:24 +01:00
|
|
|
}
|
2016-02-18 09:38:58 +01:00
|
|
|
}
|
|
|
|
|
2016-03-12 01:31:24 +01:00
|
|
|
// Populate []ObjectInfo.
|
2016-02-18 09:38:58 +01:00
|
|
|
objInfos = append(objInfos, ObjectInfo{
|
|
|
|
Name: name,
|
2016-03-12 01:31:24 +01:00
|
|
|
ModifiedTime: modTime,
|
|
|
|
MD5Sum: "", // TODO
|
|
|
|
Size: size,
|
|
|
|
IsDir: isDir,
|
2016-02-18 09:38:58 +01:00
|
|
|
})
|
2016-03-17 01:30:22 +01:00
|
|
|
|
2016-02-18 09:38:58 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2016-03-17 01:30:22 +01:00
|
|
|
// ObjectInfoChannel - object info channel.
|
2016-02-18 09:38:58 +01:00
|
|
|
type ObjectInfoChannel struct {
|
|
|
|
ch <-chan ObjectInfo
|
|
|
|
objInfo *ObjectInfo
|
|
|
|
closed bool
|
|
|
|
timeoutCh <-chan struct{}
|
|
|
|
timedOut bool
|
|
|
|
}
|
|
|
|
|
|
|
|
func (oic *ObjectInfoChannel) Read() (ObjectInfo, bool) {
|
|
|
|
if oic.closed {
|
|
|
|
return ObjectInfo{}, false
|
|
|
|
}
|
|
|
|
|
|
|
|
if oic.objInfo == nil {
|
2016-03-17 01:30:22 +01:00
|
|
|
// First read.
|
2016-02-18 09:38:58 +01:00
|
|
|
if oi, ok := <-oic.ch; ok {
|
|
|
|
oic.objInfo = &oi
|
|
|
|
} else {
|
|
|
|
oic.closed = true
|
|
|
|
return ObjectInfo{}, false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
retObjInfo := *oic.objInfo
|
|
|
|
status := true
|
|
|
|
oic.objInfo = nil
|
|
|
|
|
2016-03-17 01:30:22 +01:00
|
|
|
// Read once more to know whether it was last read.
|
2016-02-18 09:38:58 +01:00
|
|
|
if oi, ok := <-oic.ch; ok {
|
|
|
|
oic.objInfo = &oi
|
|
|
|
} else {
|
|
|
|
oic.closed = true
|
|
|
|
}
|
|
|
|
|
|
|
|
return retObjInfo, status
|
|
|
|
}
|
|
|
|
|
2016-03-17 01:30:22 +01:00
|
|
|
// IsClosed - return whether channel is closed or not.
|
2016-02-18 09:38:58 +01:00
|
|
|
func (oic ObjectInfoChannel) IsClosed() bool {
|
|
|
|
if oic.objInfo != nil {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
return oic.closed
|
|
|
|
}
|
|
|
|
|
2016-03-17 01:30:22 +01:00
|
|
|
// IsTimedOut - return whether channel is closed due to timeout.
|
2016-02-18 09:38:58 +01:00
|
|
|
func (oic ObjectInfoChannel) IsTimedOut() bool {
|
|
|
|
if oic.timedOut {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
select {
|
|
|
|
case _, ok := <-oic.timeoutCh:
|
|
|
|
if ok {
|
|
|
|
oic.timedOut = true
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
default:
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-03-22 11:18:22 +01:00
|
|
|
// treeWalk - walk into 'scanDir' recursively when 'recursive' is true.
|
2016-03-17 01:30:22 +01:00
|
|
|
// It uses 'bucketDir' to get name prefix for object name.
|
|
|
|
func treeWalk(scanDir, bucketDir string, recursive bool, queryPrefix string) ObjectInfoChannel {
|
2016-02-18 09:38:58 +01:00
|
|
|
objectInfoCh := make(chan ObjectInfo, listObjectsLimit)
|
|
|
|
timeoutCh := make(chan struct{}, 1)
|
|
|
|
|
2016-03-17 01:30:22 +01:00
|
|
|
// goroutine - retrieves directory entries, makes ObjectInfo and sends into the channel.
|
2016-02-18 09:38:58 +01:00
|
|
|
go func() {
|
|
|
|
defer close(objectInfoCh)
|
|
|
|
defer close(timeoutCh)
|
|
|
|
|
2016-03-22 11:18:22 +01:00
|
|
|
// send function - returns true if ObjectInfo is sent.
|
2016-03-17 01:30:22 +01:00
|
|
|
// Within (time.Second * 15) else false on time-out.
|
2016-02-18 09:38:58 +01:00
|
|
|
send := func(oi ObjectInfo) bool {
|
|
|
|
timer := time.After(time.Second * 15)
|
|
|
|
select {
|
|
|
|
case objectInfoCh <- oi:
|
|
|
|
return true
|
|
|
|
case <-timer:
|
|
|
|
timeoutCh <- struct{}{}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
namePrefix := strings.Replace(filepath.ToSlash(scanDir), filepath.ToSlash(bucketDir), "", 1)
|
|
|
|
if strings.HasPrefix(namePrefix, "/") {
|
2016-03-17 01:30:22 +01:00
|
|
|
// Remove forward slash ("/") from beginning.
|
2016-02-18 09:38:58 +01:00
|
|
|
namePrefix = namePrefix[1:]
|
|
|
|
}
|
2016-03-17 01:30:22 +01:00
|
|
|
// The last argument (isFisrt), is set to `true` only during the first run of the function.
|
|
|
|
// This makes sure that the sub-directories inside the prefixDir are recursed
|
|
|
|
// without being asserted for prefix in the object name.
|
|
|
|
isFirst := true
|
|
|
|
for objInfos := readDir(scanDir, namePrefix, queryPrefix, isFirst); len(objInfos) > 0; {
|
2016-02-18 09:38:58 +01:00
|
|
|
var objInfo ObjectInfo
|
|
|
|
objInfo, objInfos = objInfos[0], objInfos[1:]
|
|
|
|
if !send(objInfo) {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
if objInfo.IsDir && recursive {
|
|
|
|
scanDir := filepath.Join(bucketDir, filepath.FromSlash(objInfo.Name))
|
|
|
|
namePrefix = strings.Replace(filepath.ToSlash(scanDir), filepath.ToSlash(bucketDir), "", 1)
|
2016-03-17 01:30:22 +01:00
|
|
|
|
2016-02-18 09:38:58 +01:00
|
|
|
if strings.HasPrefix(namePrefix, "/") {
|
|
|
|
/* remove beginning "/" */
|
|
|
|
namePrefix = namePrefix[1:]
|
|
|
|
}
|
2016-03-17 01:30:22 +01:00
|
|
|
// The last argument is set to false in the further calls to readdir.
|
|
|
|
isFirst = false
|
|
|
|
objInfos = append(readDir(scanDir, namePrefix, queryPrefix, isFirst), objInfos...)
|
2016-02-18 09:38:58 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
return ObjectInfoChannel{ch: objectInfoCh, timeoutCh: timeoutCh}
|
|
|
|
}
|