re-implement the cache purging routine to avoid using ioutil.ReadDir which can lead to high allocations when there are cache directories with lots of content, or when cache is installed in memory constrainted environments. Instead rely on a callback function where we are not using memory no-more than 8KiB per cycle. Precursor for this change refer #9425, original issue pointed by Caleb Case <caleb@storj.io>
148 lines
3.6 KiB
Go
148 lines
3.6 KiB
Go
// +build plan9 solaris
|
|
|
|
/*
|
|
* MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package cmd
|
|
|
|
import (
|
|
"io"
|
|
"os"
|
|
"path"
|
|
"strings"
|
|
|
|
"github.com/minio/minio/cmd/logger"
|
|
)
|
|
|
|
// Return all the entries at the directory dirPath.
|
|
func readDir(dirPath string) (entries []string, err error) {
|
|
return readDirN(dirPath, -1)
|
|
}
|
|
|
|
// readDir applies the filter function on each entries at dirPath, doesn't recurse into
|
|
// the directory itself.
|
|
func readDirFilterFn(dirPath string, filter func(name string, typ os.FileMode) error) error {
|
|
d, err := os.Open(dirPath)
|
|
if err != nil {
|
|
// File is really not found.
|
|
if os.IsNotExist(err) {
|
|
return errFileNotFound
|
|
}
|
|
|
|
// File path cannot be verified since one of the parents is a file.
|
|
if strings.Contains(err.Error(), "not a directory") {
|
|
return errFileNotFound
|
|
}
|
|
return err
|
|
}
|
|
defer d.Close()
|
|
|
|
maxEntries := 1000
|
|
for {
|
|
// Read up to max number of entries.
|
|
fis, err := d.Readdir(maxEntries)
|
|
if err != nil {
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
return err
|
|
}
|
|
for _, fi := range fis {
|
|
if err = filter(fi.Name(), fi.Mode()); err == errDoneForNow {
|
|
// filtering requested to return by caller.
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Return N entries at the directory dirPath. If count is -1, return all entries
|
|
func readDirN(dirPath string, count int) (entries []string, err error) {
|
|
d, err := os.Open(dirPath)
|
|
if err != nil {
|
|
// File is really not found.
|
|
if os.IsNotExist(err) {
|
|
return nil, errFileNotFound
|
|
}
|
|
|
|
// File path cannot be verified since one of the parents is a file.
|
|
if strings.Contains(err.Error(), "not a directory") {
|
|
return nil, errFileNotFound
|
|
}
|
|
return nil, err
|
|
}
|
|
defer d.Close()
|
|
|
|
maxEntries := 1000
|
|
if count > 0 && count < maxEntries {
|
|
maxEntries = count
|
|
}
|
|
|
|
done := false
|
|
remaining := count
|
|
|
|
for !done {
|
|
// Read up to max number of entries.
|
|
fis, err := d.Readdir(maxEntries)
|
|
if err != nil {
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
return nil, err
|
|
}
|
|
if count > 0 {
|
|
if remaining <= len(fis) {
|
|
fis = fis[:remaining]
|
|
done = true
|
|
}
|
|
}
|
|
for _, fi := range fis {
|
|
// Stat symbolic link and follow to get the final value.
|
|
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
|
|
var st os.FileInfo
|
|
st, err = os.Stat(path.Join(dirPath, fi.Name()))
|
|
if err != nil {
|
|
reqInfo := (&logger.ReqInfo{}).AppendTags("path", path.Join(dirPath, fi.Name()))
|
|
ctx := logger.SetReqInfo(GlobalContext, reqInfo)
|
|
logger.LogIf(ctx, err)
|
|
continue
|
|
}
|
|
// Append to entries if symbolic link exists and is valid.
|
|
if st.IsDir() {
|
|
entries = append(entries, fi.Name()+SlashSeparator)
|
|
} else if st.Mode().IsRegular() {
|
|
entries = append(entries, fi.Name())
|
|
}
|
|
if count > 0 {
|
|
remaining--
|
|
}
|
|
continue
|
|
}
|
|
if fi.Mode().IsDir() {
|
|
// Append SlashSeparator instead of "\" so that sorting is achieved as expected.
|
|
entries = append(entries, fi.Name()+SlashSeparator)
|
|
} else if fi.Mode().IsRegular() {
|
|
entries = append(entries, fi.Name())
|
|
}
|
|
if count > 0 {
|
|
remaining--
|
|
}
|
|
}
|
|
}
|
|
return entries, nil
|
|
}
|