minio/cmd/os-readdir_windows.go
Harshavardhana 289e1d8b2a
fix: reduce crawler memory usage by orders of magnitude (#11556)
currently crawler waits for an entire readdir call to
return until it processes usage, lifecycle, replication
and healing - instead we should pass the applicator all
the way down to avoid building any special stack for all
the contents in a single directory.

This allows for

- no need to remember the entire list of entries per directory
  before applying the required functions
- no need to wait for entire readdir() call to finish before
  applying the required functions
2021-02-17 15:34:42 -08:00

133 lines
3.2 KiB
Go

// +build windows
/*
* MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"io"
"os"
"syscall"
)
// Return all the entries at the directory dirPath.
func readDir(dirPath string) (entries []string, err error) {
return readDirN(dirPath, -1)
}
// readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into
// the directory itself, if the dirPath doesn't exist this function doesn't return
// an error.
func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error) error {
f, err := os.Open(dirPath)
if err != nil {
if osErrToFileErr(err) == errFileNotFound {
return nil
}
return osErrToFileErr(err)
}
defer f.Close()
data := &syscall.Win32finddata{}
for {
e := syscall.FindNextFile(syscall.Handle(f.Fd()), data)
if e != nil {
if e == syscall.ERROR_NO_MORE_FILES {
break
} else {
if isSysErrPathNotFound(e) {
return nil
}
return osErrToFileErr(&os.PathError{
Op: "FindNextFile",
Path: dirPath,
Err: e,
})
}
}
name := syscall.UTF16ToString(data.FileName[0:])
if name == "" || name == "." || name == ".." { // Useless names
continue
}
if data.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0 {
continue
}
var typ os.FileMode = 0 // regular file
if data.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 {
typ = os.ModeDir
}
if e = filter(name, typ); e == errDoneForNow {
// filtering requested to return by caller.
return nil
}
}
return nil
}
// Return N entries at the directory dirPath. If count is -1, return all entries
func readDirN(dirPath string, count int) (entries []string, err error) {
f, err := os.Open(dirPath)
if err != nil {
return nil, osErrToFileErr(err)
}
defer f.Close()
// Check if file or dir. This is the quickest way.
_, err = f.Seek(0, io.SeekStart)
if err == nil {
return nil, errFileNotFound
}
data := &syscall.Win32finddata{}
handle := syscall.Handle(f.Fd())
for count != 0 {
e := syscall.FindNextFile(handle, data)
if e != nil {
if e == syscall.ERROR_NO_MORE_FILES {
break
} else {
return nil, osErrToFileErr(&os.PathError{
Op: "FindNextFile",
Path: dirPath,
Err: e,
})
}
}
name := syscall.UTF16ToString(data.FileName[0:])
if name == "" || name == "." || name == ".." { // Useless names
continue
}
switch {
case data.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0:
continue
case data.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0:
entries = append(entries, name+SlashSeparator)
default:
entries = append(entries, name)
}
count--
}
return entries, nil
}
func globalSync() {
// no-op on windows
}