maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

fix: check for O_DIRECT support for reads and writes (#11331)

Browse source 
In-case user enables O_DIRECT for reads and backend does
not support it we shall proceed to turn it off instead
and print a warning. This validation avoids any unexpected
downtimes that users may incur.
This commit is contained in:
Harshavardhana 2021-01-22 15:38:21 -08:00 committed by GitHub
parent 1b453728a3
commit 43f973c4cf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 72 additions and 136 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmd/config/storageclass/storage-class.go
View file

@ -64,7 +64,7 @@ const (
defaultRRSParity = minParityDisks
// Default DMA value
defaultDMA = DMAWrite
defaultDMA = DMAReadWrite
)
// DefaultKVS - default storage class config

2
cmd/fs-v1.go
View file

@ -123,7 +123,7 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
}
var err error
if fsPath, err = getValidPath(fsPath, false); err != nil {
if fsPath, err = getValidPath(fsPath); err != nil {
if err == errMinDiskSize {
return nil, config.ErrUnableToWriteInBackend(err).Hint(err.Error())
}

4
cmd/storage-rest_test.go
View file

@ -82,8 +82,8 @@ func testStorageAPIListVols(t *testing.T, storage StorageAPI) {
expectedResult []VolInfo
expectErr bool
}{
{nil, []VolInfo{}, false},
{[]string{"foo"}, []VolInfo{{Name: "foo"}}, false},
{nil, []VolInfo{{Name: ".minio.sys"}}, false},
{[]string{"foo"}, []VolInfo{{Name: ".minio.sys"}, {Name: "foo"}}, false},
}
for i, testCase := range testCases {

130
cmd/xl-storage.go
View file

@ -53,10 +53,9 @@ import (
)
const (
nullVersionID = "null"
diskMinTotalSpace = 900 * humanize.MiByte // Min 900MiB total space.
blockSizeLarge = 1 * humanize.MiByte // Default r/w block size for larger objects.
blockSizeSmall = 128 * humanize.KiByte // Default r/w block size for smaller objects.
nullVersionID = "null"
blockSizeLarge = 1 * humanize.MiByte // Default r/w block size for larger objects.
blockSizeSmall = 128 * humanize.KiByte // Default r/w block size for smaller objects.
// On regular files bigger than this;
readAheadSize = 16 << 20
@ -103,6 +102,8 @@ type xlStorage struct {
rootDisk bool
readODirectSupported bool
diskID string
formatFileInfo os.FileInfo
@ -155,7 +156,7 @@ func checkPathLength(pathName string) error {
return nil
}
func getValidPath(path string, requireDirectIO bool) (string, error) {
func getValidPath(path string) (string, error) {
if path == "" {
return path, errInvalidArgument
}
@ -173,7 +174,7 @@ func getValidPath(path string, requireDirectIO bool) (string, error) {
}
if osIsNotExist(err) {
// Disk not found create it.
if err = os.MkdirAll(path, 0777); err != nil {
if err = reliableMkdirAll(path, 0777); err != nil {
return path, err
}
}
@ -181,42 +182,6 @@ func getValidPath(path string, requireDirectIO bool) (string, error) {
return path, errDiskNotDir
}
// check if backend is writable.
var rnd [8]byte
_, _ = rand.Read(rnd[:])
fn := pathJoin(path, ".writable-check-"+hex.EncodeToString(rnd[:])+".tmp")
defer os.Remove(fn)
var file *os.File
if requireDirectIO {
// only erasure coding needs direct-io support
file, err = disk.OpenFileDirectIO(fn, os.O_CREATE|os.O_EXCL, 0666)
} else {
file, err = os.OpenFile(fn, os.O_CREATE|os.O_EXCL, 0666)
}
// open file in direct I/O and use default umask, this also verifies
// if direct i/o failed.
if err != nil {
if isSysErrInvalidArg(err) {
// O_DIRECT not supported
return path, errUnsupportedDisk
}
return path, osErrToFileErr(err)
}
file.Close()
di, err := getDiskInfo(path)
if err != nil {
return path, err
}
if err = checkDiskMinTotal(di); err != nil {
return path, err
}
return path, nil
}
@ -245,7 +210,7 @@ func newLocalXLStorage(path string) (*xlStorage, error) {
func newXLStorage(ep Endpoint) (*xlStorage, error) {
path := ep.Path
var err error
if path, err = getValidPath(path, true); err != nil {
if path, err = getValidPath(path); err != nil {
return nil, err
}
@ -279,6 +244,35 @@ func newXLStorage(ep Endpoint) (*xlStorage, error) {
rootDisk: rootDisk,
}
// Create all necessary bucket folders if possible.
if err = p.MakeVolBulk(context.TODO(), minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, dataUsageBucket); err != nil {
return nil, err
}
// Check if backend is writable and supports O_DIRECT
var rnd [8]byte
_, _ = rand.Read(rnd[:])
tmpFile := ".writable-check-" + hex.EncodeToString(rnd[:]) + ".tmp"
if err = p.CreateFile(GlobalContext, minioMetaTmpBucket, tmpFile, 1, strings.NewReader("0")); err != nil {
return p, err
}
defer os.Remove(pathJoin(p.diskPath, minioMetaTmpBucket, tmpFile))
volumeDir, err := p.getVolDir(minioMetaTmpBucket)
if err != nil {
return p, err
}
// Check if backend is readable, and optionally supports O_DIRECT.
if _, err = p.readAllData(volumeDir, pathJoin(volumeDir, tmpFile), true); err != nil {
if err != errUnsupportedDisk {
return p, err
}
// error is unsupported disk, turn-off directIO for reads
logger.Info(fmt.Sprintf("Drive %s does not support O_DIRECT for reads, proceeding to use the drive without O_DIRECT", ep))
p.readODirectSupported = false
}
// Success.
return p, nil
}
@ -301,17 +295,6 @@ func getDiskInfo(diskPath string) (di disk.Info, err error) {
return di, err
}
// check if disk total has minimum required size.
func checkDiskMinTotal(di disk.Info) (err error) {
// Remove 5% from total space for cumulative disk space
// used for journalling, inodes etc.
totalDiskSpace := float64(di.Total) * diskFillFraction
if int64(totalDiskSpace) <= diskMinTotalSpace {
return errMinDiskSize
}
return nil
}
// Implements stringer compatible interface.
func (s *xlStorage) String() string {
return s.diskPath
@ -583,11 +566,11 @@ func (s *xlStorage) SetDiskID(id string) {
// storage rest server for remote disks.
}
func (s *xlStorage) MakeVolBulk(ctx context.Context, volumes ...string) (err error) {
func (s *xlStorage) MakeVolBulk(ctx context.Context, volumes ...string) error {
for _, volume := range volumes {
if err = s.MakeVol(ctx, volume); err != nil {
if osIsPermission(err) {
return errVolumeAccessDenied
if err := s.MakeVol(ctx, volume); err != nil {
if errors.Is(err, errDiskAccessDenied) {
return errDiskAccessDenied
}
}
}
@ -595,7 +578,7 @@ func (s *xlStorage) MakeVolBulk(ctx context.Context, volumes ...string) (err err
}
// Make a volume entry.
func (s *xlStorage) MakeVol(ctx context.Context, volume string) (err error) {
func (s *xlStorage) MakeVol(ctx context.Context, volume string) error {
if !isValidVolname(volume) {
return errInvalidArgument
}
@ -614,7 +597,7 @@ func (s *xlStorage) MakeVol(ctx context.Context, volume string) (err error) {
// Volume does not exist we proceed to create.
if osIsNotExist(err) {
// Make a volume entry, with mode 0777 mkdir honors system umask.
err = os.MkdirAll(volumeDir, 0777)
err = reliableMkdirAll(volumeDir, 0777)
}
if osIsPermission(err) {
return errDiskAccessDenied
@ -1138,7 +1121,10 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str
// - object size lesser than 32KiB
// - object has maximum of 1 parts
if fi.TransitionStatus == "" && fi.DataDir != "" && fi.Size <= smallFileThreshold && len(fi.Parts) == 1 {
fi.Data, err = s.readAllData(volumeDir, pathJoin(volumeDir, path, fi.DataDir, fmt.Sprintf("part.%d", fi.Parts[0].Number)))
// Enable O_DIRECT optionally only if drive supports it.
requireDirectIO := globalStorageClass.GetDMA() == storageclass.DMAReadWrite && s.readODirectSupported
partPath := fmt.Sprintf("part.%d", fi.Parts[0].Number)
fi.Data, err = s.readAllData(volumeDir, pathJoin(volumeDir, path, fi.DataDir, partPath), requireDirectIO)
if err != nil {
return FileInfo{}, err
}
@ -1148,9 +1134,9 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str
return fi, nil
}
func (s *xlStorage) readAllData(volumeDir, filePath string) (buf []byte, err error) {
func (s *xlStorage) readAllData(volumeDir, filePath string, requireDirectIO bool) (buf []byte, err error) {
var f *os.File
if globalStorageClass.GetDMA() == storageclass.DMAReadWrite {
if requireDirectIO {
f, err = disk.OpenFileDirectIO(filePath, os.O_RDONLY, 0666)
} else {
f, err = os.Open(filePath)
@ -1176,7 +1162,7 @@ func (s *xlStorage) readAllData(volumeDir, filePath string) (buf []byte, err err
} else if isSysErrTooManyFiles(err) {
return nil, errTooManyOpenFiles
} else if isSysErrInvalidArg(err) {
return nil, errFileNotFound
return nil, errUnsupportedDisk
}
return nil, err
}
@ -1185,7 +1171,11 @@ func (s *xlStorage) readAllData(volumeDir, filePath string) (buf []byte, err err
rd := &odirectReader{f, nil, nil, true, true, s, nil}
defer rd.Close()
return ioutil.ReadAll(rd)
buf, err = ioutil.ReadAll(rd)
if err != nil {
err = osErrToFileErr(err)
}
return buf, err
}
// ReadAll reads from r until an error or EOF and returns the data it read.
@ -1233,7 +1223,7 @@ func (s *xlStorage) ReadAll(ctx context.Context, volume string, path string) (bu
} else if isSysErrTooManyFiles(err) {
return nil, errTooManyOpenFiles
} else if isSysErrInvalidArg(err) {
return nil, errFileNotFound
return nil, errUnsupportedDisk
}
return nil, err
}
@ -1486,7 +1476,7 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
var file *os.File
// O_DIRECT only supported if offset is zero
if offset == 0 && globalStorageClass.GetDMA() == storageclass.DMAReadWrite {
if offset == 0 && globalStorageClass.GetDMA() == storageclass.DMAReadWrite && s.readODirectSupported {
file, err = disk.OpenFileDirectIO(filePath, os.O_RDONLY, 0666)
} else {
// Open the file for reading.
@ -1508,6 +1498,8 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
return nil, errFaultyDisk
case isSysErrTooManyFiles(err):
return nil, errTooManyOpenFiles
case isSysErrInvalidArg(err):
return nil, errUnsupportedDisk
default:
return nil, err
}
@ -1525,7 +1517,7 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
}
atomic.AddInt32(&s.activeIOCount, 1)
if offset == 0 && globalStorageClass.GetDMA() == storageclass.DMAReadWrite {
if offset == 0 && globalStorageClass.GetDMA() == storageclass.DMAReadWrite && s.readODirectSupported {
if length <= smallFileThreshold {
return &odirectReader{file, nil, nil, true, true, s, nil}, nil
}
@ -2145,7 +2137,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir,
legacyDataPath := pathJoin(dstVolumeDir, dstPath, legacyDataDir)
// legacy data dir means its old content, honor system umask.
if err = os.MkdirAll(legacyDataPath, 0777); err != nil {
if err = reliableMkdirAll(legacyDataPath, 0777); err != nil {
return osErrToFileErr(err)
}

70
cmd/xl-storage_test.go
View file

@ -31,7 +31,6 @@ import (
"testing"
"github.com/minio/minio/cmd/config/storageclass"
"github.com/minio/minio/pkg/disk"
)
func TestCheckPathLength(t *testing.T) {
@ -128,10 +127,6 @@ func newXLStorageTestSetup() (*xlStorageDiskIDCheck, string, error) {
if err != nil {
return nil, "", err
}
err = storage.MakeVol(context.Background(), minioMetaBucket)
if err != nil {
return nil, "", err
}
// Create a sample format.json file
err = storage.WriteAll(context.Background(), minioMetaBucket, formatConfigFile, []byte(`{"version":"1","format":"xl","id":"592a41c2-b7cc-4130-b883-c4b5cb15965b","xl":{"version":"3","this":"da017d62-70e3-45f1-8a1a-587707e69ad1","sets":[["e07285a6-8c73-4962-89c6-047fb939f803","33b8d431-482d-4376-b63c-626d229f0a29","cff6513a-4439-4dc1-bcaa-56c9e880c352","da017d62-70e3-45f1-8a1a-587707e69ad1","9c9f21d5-1f15-4737-bce6-835faa0d9626","0a59b346-1424-4fc2-9fa2-a2e80541d0c1","7924a3dc-b69a-4971-9a2e-014966d6aebb","4d2b8dd9-4e48-444b-bdca-c89194b26042"]],"distributionAlgo":"CRCMOD"}}`))
if err != nil {
@ -258,7 +253,7 @@ func TestXLStorageReadVersion(t *testing.T) {
// create xlStorage test setup
xlStorage, path, err := newXLStorageTestSetup()
if err != nil {
t.Fatalf("Unable to create xlStorage test setup, %s", err)
t.Fatalf("Unable to cfgreate xlStorage test setup, %s", err)
}
defer os.RemoveAll(path)
@ -539,7 +534,7 @@ func TestXLStorageMakeVol(t *testing.T) {
// Initialize xlStorage storage layer for permission denied error.
_, err = newLocalXLStorage(permDeniedDir)
if err != nil && err != errFileAccessDenied {
if err != nil && err != errDiskAccessDenied {
t.Fatalf("Unable to initialize xlStorage, %s", err)
}
@ -638,7 +633,7 @@ func TestXLStorageDeleteVol(t *testing.T) {
// Initialize xlStorage storage layer for permission denied error.
_, err = newLocalXLStorage(permDeniedDir)
if err != nil && err != errFileAccessDenied {
if err != nil && err != errDiskAccessDenied {
t.Fatalf("Unable to initialize xlStorage, %s", err)
}
@ -890,7 +885,7 @@ func TestXLStorageListDir(t *testing.T) {
// Initialize xlStorage storage layer for permission denied error.
_, err = newLocalXLStorage(permDeniedDir)
if err != nil && err != errFileAccessDenied {
if err != nil && err != errDiskAccessDenied {
t.Fatalf("Unable to initialize xlStorage, %s", err)
}
@ -1014,7 +1009,7 @@ func TestXLStorageDeleteFile(t *testing.T) {
// Initialize xlStorage storage layer for permission denied error.
_, err = newLocalXLStorage(permDeniedDir)
if err != nil && err != errFileAccessDenied {
if err != nil && err != errDiskAccessDenied {
t.Fatalf("Unable to initialize xlStorage, %s", err)
}
@ -1221,7 +1216,7 @@ func TestXLStorageReadFile(t *testing.T) {
// Initialize xlStorage storage layer for permission denied error.
_, err = newLocalXLStorage(permDeniedDir)
if err != nil && err != errFileAccessDenied {
if err != nil && err != errDiskAccessDenied {
t.Fatalf("Unable to initialize xlStorage, %s", err)
}
@ -1391,7 +1386,7 @@ func TestXLStorageAppendFile(t *testing.T) {
var xlStoragePermStorage StorageAPI
// Initialize xlStorage storage layer for permission denied error.
_, err = newLocalXLStorage(permDeniedDir)
if err != nil && err != errFileAccessDenied {
if err != nil && err != errDiskAccessDenied {
t.Fatalf("Unable to initialize xlStorage, %s", err)
}
@ -1824,54 +1819,3 @@ func TestXLStorageVerifyFile(t *testing.T) {
t.Fatal("expected to fail bitrot check")
}
}
// Checks for restrictions for min total disk space and inodes.
func TestCheckDiskTotalMin(t *testing.T) {
testCases := []struct {
diskInfo disk.Info
err error
}{
// Test 1 - when fstype is nfs.
{
diskInfo: disk.Info{
Total: diskMinTotalSpace * 3,
FSType: "NFS",
},
err: nil,
},
// Test 2 - when fstype is xfs and total inodes are less than 10k.
{
diskInfo: disk.Info{
Total: diskMinTotalSpace * 3,
FSType: "XFS",
Files: 9999,
},
err: nil,
},
// Test 3 - when fstype is btrfs and total inodes is empty.
{
diskInfo: disk.Info{
Total: diskMinTotalSpace * 3,
FSType: "BTRFS",
Files: 0,
},
err: nil,
},
// Test 4 - when fstype is xfs and total disk space is really small.
{
diskInfo: disk.Info{
Total: diskMinTotalSpace - diskMinTotalSpace/1024,
FSType: "XFS",
Files: 9999,
},
err: errMinDiskSize,
},
}
// Validate all cases.
for i, test := range testCases {
if err := checkDiskMinTotal(test.diskInfo); test.err != err {
t.Errorf("Test %d: Expected error %s, got %s", i+1, test.err, err)
}
}
}