maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

fs: Add proper volume and path validation.

Browse source
This commit is contained in:
Harshavardhana 2016-04-13 11:32:47 -07:00
parent caa35f68fa
commit 8457af5708
8 changed files with 235 additions and 138 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

56
fs-utils.go Normal file
View file

@ -0,0 +1,56 @@
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"regexp"
"unicode/utf8"
)
// validVolname regexp.
var validVolname = regexp.MustCompile(`^.{3,63}$`)
// isValidVolname verifies a volname name in accordance with object
// layer requirements.
func isValidVolname(volname string) bool {
return validVolname.MatchString(volname)
}
// Keeping this as lower bound value supporting Linux, Darwin and Windows operating systems.
const pathMax = 4096
// isValidPath verifies if a path name is in accordance with FS limitations.
func isValidPath(path string) bool {
// TODO: Make this FSType or Operating system specific.
if len(path) > pathMax || len(path) == 0 {
return false
}
if !utf8.ValidString(path) {
return false
}
return true
}
// isValidPrefix verifies where the prefix is a valid path.
func isValidPrefix(prefix string) bool {
// Prefix can be empty.
if prefix == "" {
return true
}
// Verify if prefix is a valid path.
return isValidPath(prefix)
}

241
fs.go
View file

@ -121,26 +121,63 @@ func checkDiskFree(diskPath string, minFreeDisk int64) (err error) {
return nil
}
// checkVolumeArg - will convert incoming volume names to
// corresponding valid volume names on the backend in a platform
// compatible way for all operating systems. If volume is not found
// an error is generated.
func (s fsStorage) checkVolumeArg(volume string) (string, error) {
if !isValidVolname(volume) {
return "", errInvalidArgument
}
volumeDir := filepath.Join(s.diskPath, volume)
_, err := os.Stat(volumeDir)
if err == nil {
return volumeDir, nil
}
if os.IsNotExist(err) {
var volumes []os.FileInfo
volumes, err = ioutil.ReadDir(s.diskPath)
if err != nil {
return volumeDir, errVolumeNotFound
}
for _, vol := range volumes {
if vol.IsDir() {
// Verify if lowercase version of the volume
// is equal to the incoming volume, then use the proper name.
if strings.ToLower(vol.Name()) == volume {
volumeDir = filepath.Join(s.diskPath, vol.Name())
return volumeDir, nil
}
}
}
return volumeDir, errVolumeNotFound
} else if os.IsPermission(err) {
return volumeDir, errVolumeAccessDenied
}
return volumeDir, err
}
// Make a volume entry.
func (s fsStorage) MakeVol(volume string) (err error) {
if volume == "" {
return errInvalidArgument
}
if err = checkDiskFree(s.diskPath, s.minFreeDisk); err != nil {
return err
}
volumeDir := getVolumeDir(s.diskPath, volume)
if _, err = os.Stat(volumeDir); err == nil {
volumeDir, err := s.checkVolumeArg(volume)
if err == nil {
// Volume already exists, return error.
return errVolumeExists
}
// Make a volume entry.
if err = os.Mkdir(volumeDir, 0700); err != nil {
return err
// Validate if disk is free.
if e := checkDiskFree(s.diskPath, s.minFreeDisk); e != nil {
return e
}
return nil
// If volume not found create it.
if err == errVolumeNotFound {
// Make a volume entry.
return os.Mkdir(volumeDir, 0700)
}
// For all other errors return here.
return err
}
// removeDuplicateVols - remove duplicate volumes.
@ -175,9 +212,15 @@ func (s fsStorage) ListVols() (volsInfo []VolInfo, err error) {
// If not directory, ignore all file types.
continue
}
// Volname on case sensitive fs backends can come in as
// capitalized, but object layer cannot consume it
// directly. Convert it as we see fit.
volName := strings.ToLower(file.Name())
// Modtime is used as created time.
createdTime := file.ModTime()
volInfo := VolInfo{
Name: file.Name(),
Created: file.ModTime(),
Name: volName,
Created: createdTime,
}
volsInfo = append(volsInfo, volInfo)
}
@ -186,30 +229,13 @@ func (s fsStorage) ListVols() (volsInfo []VolInfo, err error) {
return volsInfo, nil
}
// getVolumeDir - will convert incoming volume names to
// corresponding valid volume names on the backend in a platform
// compatible way for all operating systems.
func getVolumeDir(diskPath, volume string) string {
volumes, e := ioutil.ReadDir(diskPath)
if e != nil {
return volume
}
for _, vol := range volumes {
// Verify if lowercase version of the volume
// is equal to the incoming volume, then use the proper name.
if strings.ToLower(vol.Name()) == volume {
return filepath.Join(diskPath, vol.Name())
}
}
return filepath.Join(diskPath, volume)
}
// StatVol - get volume info.
func (s fsStorage) StatVol(volume string) (volInfo VolInfo, err error) {
if volume == "" {
return VolInfo{}, errInvalidArgument
// Verify if volume is valid and it exists.
volumeDir, err := s.checkVolumeArg(volume)
if err != nil {
return VolInfo{}, err
}
volumeDir := getVolumeDir(s.diskPath, volume)
// Stat a volume entry.
var st os.FileInfo
st, err = os.Stat(volumeDir)
@ -219,18 +245,23 @@ func (s fsStorage) StatVol(volume string) (volInfo VolInfo, err error) {
}
return VolInfo{}, err
}
// Modtime is used as created time since operating systems lack a
// portable way of knowing the actual created time of a directory.
createdTime := st.ModTime()
return VolInfo{
Name: st.Name(),
Created: st.ModTime(),
Name: volume,
Created: createdTime,
}, nil
}
// DeleteVol - delete a volume.
func (s fsStorage) DeleteVol(volume string) error {
if volume == "" {
return errInvalidArgument
// Verify if volume is valid and it exists.
volumeDir, err := s.checkVolumeArg(volume)
if err != nil {
return err
}
err := os.Remove(getVolumeDir(s.diskPath, volume))
err = os.Remove(volumeDir)
if err != nil && os.IsNotExist(err) {
return errVolumeNotFound
}
@ -281,23 +312,13 @@ var specialPrefixes = []string{
// List operation.
func (s fsStorage) ListFiles(volume, prefix, marker string, recursive bool, count int) ([]FileInfo, bool, error) {
if volume == "" {
return nil, true, errInvalidArgument
// Verify if volume is valid and it exists.
volumeDir, err := s.checkVolumeArg(volume)
if err != nil {
return nil, true, err
}
var fileInfos []FileInfo
volumeDir := getVolumeDir(s.diskPath, volume)
// Verify if volume directory exists
if exists, err := isDirExist(volumeDir); !exists {
if err == nil {
return nil, true, errVolumeNotFound
} else if os.IsNotExist(err) {
return nil, true, errVolumeNotFound
} else {
return nil, true, err
}
}
if marker != "" {
// Verify if marker has prefix.
if marker != "" && !strings.HasPrefix(marker, prefix) {
@ -323,6 +344,7 @@ func (s fsStorage) ListFiles(volume, prefix, marker string, recursive bool, coun
// Prefix does not exist, not an error just respond empty list response.
return nil, true, nil
} else if strings.Contains(err.Error(), "not a directory") {
// Prefix exists as a file.
return nil, true, nil
}
// Rest errors should be treated as failure.
@ -375,26 +397,18 @@ func (s fsStorage) ListFiles(volume, prefix, marker string, recursive bool, coun
// ReadFile - read a file at a given offset.
func (s fsStorage) ReadFile(volume string, path string, offset int64) (readCloser io.ReadCloser, err error) {
if volume == "" || path == "" {
return nil, errInvalidArgument
volumeDir, err := s.checkVolumeArg(volume)
if err != nil {
return nil, err
}
volumeDir := getVolumeDir(s.diskPath, volume)
// Verify if volume directory exists
var exists bool
if exists, err = isDirExist(volumeDir); !exists {
if err == nil {
return nil, errVolumeNotFound
} else if os.IsNotExist(err) {
return nil, errVolumeNotFound
} else {
return nil, err
}
}
filePath := filepath.Join(volumeDir, path)
filePath := filepath.Join(volumeDir, filepath.FromSlash(path))
file, err := os.Open(filePath)
if err != nil {
if os.IsNotExist(err) {
return nil, errFileNotFound
} else if os.IsPermission(err) {
return nil, errFileAccessDenied
}
return nil, err
}
@ -416,22 +430,12 @@ func (s fsStorage) ReadFile(volume string, path string, offset int64) (readClose
// CreateFile - create a file at path.
func (s fsStorage) CreateFile(volume, path string) (writeCloser io.WriteCloser, err error) {
if volume == "" || path == "" {
return nil, errInvalidArgument
volumeDir, err := s.checkVolumeArg(volume)
if err != nil {
return nil, err
}
if e := checkDiskFree(s.diskPath, s.minFreeDisk); e != nil {
return nil, e
}
volumeDir := getVolumeDir(s.diskPath, volume)
// Verify if volume directory exists
if exists, err := isDirExist(volumeDir); !exists {
if err == nil {
return nil, errVolumeNotFound
} else if os.IsNotExist(err) {
return nil, errVolumeNotFound
} else {
return nil, err
}
if err := checkDiskFree(s.diskPath, s.minFreeDisk); err != nil {
return nil, err
}
filePath := filepath.Join(volumeDir, path)
// Verify if the file already exists and is not of regular type.
@ -445,33 +449,26 @@ func (s fsStorage) CreateFile(volume, path string) (writeCloser io.WriteCloser,
// StatFile - get file info.
func (s fsStorage) StatFile(volume, path string) (file FileInfo, err error) {
if volume == "" || path == "" {
return FileInfo{}, errInvalidArgument
}
volumeDir := getVolumeDir(s.diskPath, volume)
// Verify if volume directory exists
var exists bool
if exists, err = isDirExist(volumeDir); !exists {
if err == nil {
return FileInfo{}, errVolumeNotFound
} else if os.IsNotExist(err) {
return FileInfo{}, errVolumeNotFound
} else {
return FileInfo{}, err
}
volumeDir, err := s.checkVolumeArg(volume)
if err != nil {
return FileInfo{}, err
}
filePath := filepath.Join(volumeDir, path)
filePath := filepath.Join(volumeDir, filepath.FromSlash(path))
st, err := os.Stat(filePath)
if err != nil {
// File is really not found.
if os.IsNotExist(err) {
return FileInfo{}, errFileNotFound
}
// File path cannot be verified since one of the parents is a file.
if strings.Contains(err.Error(), "not a directory") {
return FileInfo{}, errIsNotRegular
}
// Return all errors here.
return FileInfo{}, err
}
// If its a directory its not a regular file.
if st.Mode().IsDir() {
return FileInfo{}, errIsNotRegular
}
@ -486,49 +483,55 @@ func (s fsStorage) StatFile(volume, path string) (file FileInfo, err error) {
}
// deleteFile - delete file path if its empty.
func deleteFile(basePath, deletePath, volume, path string) error {
func deleteFile(basePath, deletePath string) error {
if basePath == deletePath {
return nil
}
// Verify if the path exists.
pathSt, e := os.Stat(deletePath)
if e != nil {
return e
pathSt, err := os.Stat(deletePath)
if err != nil {
if os.IsNotExist(err) {
return errFileNotFound
} else if os.IsPermission(err) {
return errFileAccessDenied
}
return err
}
if pathSt.IsDir() {
// Verify if directory is empty.
empty, e := isDirEmpty(deletePath)
if e != nil {
return e
empty, err := isDirEmpty(deletePath)
if err != nil {
return err
}
if !empty {
return nil
}
}
// Attempt to remove path.
if e := os.Remove(deletePath); e != nil {
return e
if err := os.Remove(deletePath); err != nil {
return err
}
// Recursively go down the next path and delete again.
if e := deleteFile(basePath, filepath.Dir(deletePath), volume, path); e != nil {
return e
if err := deleteFile(basePath, filepath.Dir(deletePath)); err != nil {
return err
}
return nil
}
// DeleteFile - delete a file at path.
func (s fsStorage) DeleteFile(volume, path string) error {
if volume == "" || path == "" {
return errInvalidArgument
volumeDir, err := s.checkVolumeArg(volume)
if err != nil {
return err
}
volumeDir := getVolumeDir(s.diskPath, volume)
// Following code is needed so that we retain "/" suffix if any in
// path argument. Do not use filepath.Join() since it would strip
// off any suffixes.
filePath := s.diskPath + string(os.PathSeparator) + volume + string(os.PathSeparator) + path
// path argument.
filePath := filepath.Join(volumeDir, filepath.FromSlash(path))
if strings.HasSuffix(filepath.FromSlash(path), string(os.PathSeparator)) {
filePath = filePath + string(os.PathSeparator)
}
// Delete file and delete parent directory as well if its empty.
return deleteFile(volumeDir, filePath, volume, path)
return deleteFile(volumeDir, filePath)
}

33
object-api-multipart.go
View file

@ -22,7 +22,6 @@ import (
"fmt"
"io"
"path"
"path/filepath"
"strconv"
"strings"
@ -111,10 +110,13 @@ func (o objectAPI) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarke
}
result.IsTruncated = true
newMaxUploads := 0
prefixPath := bucket + slashPathSeparator + prefix // do not use filepath.Join so that we retain trailing '/' if any
prefixPath := path.Join(bucket, prefix)
if strings.HasSuffix(prefix, slashPathSeparator) {
// Add back the slash separator removed after 'path.Join'.
prefixPath = prefixPath + slashPathSeparator
}
if recursive {
keyMarkerPath := filepath.Join(keyMarker, uploadIDMarker)
keyMarkerPath := path.Join(keyMarker, uploadIDMarker)
outerLoop:
for {
fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, prefixPath, keyMarkerPath, recursive, maxUploads-newMaxUploads)
@ -123,17 +125,17 @@ func (o objectAPI) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarke
}
for _, fi := range fileInfos {
keyMarkerPath = fi.Name
fileName := filepath.Base(fi.Name)
fileName := path.Base(fi.Name)
if strings.Contains(fileName, ".") {
// fileName contains partnumber and md5sum info, skip this.
continue
}
result.Uploads = append(result.Uploads, uploadMetadata{
Object: filepath.Dir(fi.Name),
Object: path.Dir(fi.Name),
UploadID: fileName,
Initiated: fi.ModTime,
})
result.NextKeyMarker = filepath.Dir(fi.Name)
result.NextKeyMarker = path.Dir(fi.Name)
result.NextUploadIDMarker = fileName
newMaxUploads++
if newMaxUploads == maxUploads {
@ -244,7 +246,7 @@ func (o objectAPI) NewMultipartUpload(bucket, object string) (string, *probe.Err
return "", probe.NewError(e)
}
uploadID := uuid.String()
uploadIDFile := filepath.Join(bucket, object, uploadID)
uploadIDFile := path.Join(bucket, object, uploadID)
if _, e = o.storage.StatFile(minioMetaVolume, uploadIDFile); e != nil {
if e != errFileNotFound {
return "", probe.NewError(e)
@ -266,7 +268,7 @@ func (o objectAPI) NewMultipartUpload(bucket, object string) (string, *probe.Err
}
func (o objectAPI) isUploadIDExist(bucket, object, uploadID string) (bool, error) {
st, e := o.storage.StatFile(minioMetaVolume, filepath.Join(bucket, object, uploadID))
st, e := o.storage.StatFile(minioMetaVolume, path.Join(bucket, object, uploadID))
if e != nil {
if e == errFileNotFound {
return false, nil
@ -291,7 +293,7 @@ func (o objectAPI) PutObjectPart(bucket, object, uploadID string, partID int, si
}
partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, partID, md5Hex)
fileWriter, e := o.storage.CreateFile(minioMetaVolume, filepath.Join(bucket, object, partSuffix))
fileWriter, e := o.storage.CreateFile(minioMetaVolume, path.Join(bucket, object, partSuffix))
if e != nil {
if e == errVolumeNotFound {
return "", probe.NewError(BucketNotFound{
@ -356,7 +358,7 @@ func (o objectAPI) ListObjectParts(bucket, object, uploadID string, partNumberMa
marker := ""
nextPartNumberMarker := 0
if partNumberMarker > 0 {
fileInfos, _, e := o.storage.ListFiles(minioMetaVolume, filepath.Join(bucket, object, uploadID)+"."+strconv.Itoa(partNumberMarker)+".", "", false, 1)
fileInfos, _, e := o.storage.ListFiles(minioMetaVolume, path.Join(bucket, object, uploadID)+"."+strconv.Itoa(partNumberMarker)+".", "", false, 1)
if e != nil {
return result, probe.NewError(e)
}
@ -365,12 +367,12 @@ func (o objectAPI) ListObjectParts(bucket, object, uploadID string, partNumberMa
}
marker = fileInfos[0].Name
}
fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, filepath.Join(bucket, object, uploadID)+".", marker, false, maxParts)
fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, path.Join(bucket, object, uploadID)+".", marker, false, maxParts)
if e != nil {
return result, probe.NewError(InvalidPart{})
}
for _, fileInfo := range fileInfos {
fileName := filepath.Base(fileInfo.Name)
fileName := path.Base(fileInfo.Name)
splitResult := strings.Split(fileName, ".")
partNum, e := strconv.Atoi(splitResult[1])
if e != nil {
@ -415,7 +417,7 @@ func (o objectAPI) CompleteMultipartUpload(bucket string, object string, uploadI
for _, part := range parts {
partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, part.PartNumber, part.ETag)
var fileReader io.ReadCloser
fileReader, e = o.storage.ReadFile(minioMetaVolume, filepath.Join(bucket, object, partSuffix), 0)
fileReader, e = o.storage.ReadFile(minioMetaVolume, path.Join(bucket, object, partSuffix), 0)
if e != nil {
return ObjectInfo{}, probe.NewError(e)
}
@ -456,7 +458,8 @@ func (o objectAPI) removeMultipartUpload(bucket, object, uploadID string) *probe
}
marker := ""
for {
fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, filepath.Join(bucket, object, uploadID), marker, false, 1000)
uploadIDFile := path.Join(bucket, object, uploadID)
fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, uploadIDFile, marker, false, 1000)
if e != nil {
return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
}

7
object-api.go
View file

@ -80,10 +80,15 @@ func (o objectAPI) ListBuckets() ([]BucketInfo, *probe.Error) {
return nil, probe.NewError(e)
}
for _, vol := range vols {
// StorageAPI can send volume names which are incompatible
// with buckets, handle it and skip them.
if !IsValidBucketName(vol.Name) {
continue
}
bucketInfos = append(bucketInfos, BucketInfo{vol.Name, vol.Created})
bucketInfos = append(bucketInfos, BucketInfo{
Name: vol.Name,
Created: vol.Created,
})
}
return bucketInfos, nil
}

1
object-handlers.go
View file

@ -1012,7 +1012,6 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
return
}
fmt.Println(string(completeMultipartBytes))
complMultipartUpload := &completeMultipartUpload{}
if e = xml.Unmarshal(completeMultipartBytes, complMultipartUpload); e != nil {
errorIf(probe.NewError(e), "XML Unmarshal failed", nil)

26
object-utils.go
View file

@ -18,6 +18,7 @@ package main
import (
"regexp"
"strings"
"unicode/utf8"
)
@ -41,7 +42,27 @@ func IsValidBucketName(bucket string) bool {
// IsValidObjectName verifies an object name in accordance with Amazon's
// requirements. It cannot exceed 1024 characters and must be a valid UTF8
// string.
// See: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
//
// See:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
//
// You should avoid the following characters in a key name because of
// significant special handling for consistency across all
// applications.
//
// Rejects strings with following characters.
//
// - Backslash ("\")
// - Left curly brace ("{")
// - Caret ("^")
// - Right curly brace ("}")
// - Grave accent / back tick ("`")
// - Right square bracket ("]")
// - Left square bracket ("[")
// - Tilde ("~")
// - 'Greater Than' symbol (">")
// - 'Less Than' symbol ("<")
// - Vertical bar / pipe ("|")
func IsValidObjectName(object string) bool {
if len(object) > 1024 || len(object) == 0 {
return false
@ -49,7 +70,8 @@ func IsValidObjectName(object string) bool {
if !utf8.ValidString(object) {
return false
}
return true
// Reject unsupported characters in object name.
return !strings.ContainsAny(object, "`^*{}[]|\\\"'")
}
// IsValidObjectPrefix verifies whether the prefix is a valid object name.

1
server_test.go
View file

@ -1172,6 +1172,7 @@ func (s *MyAPISuite) TestValidateObjectMultipartUploadID(c *C) {
c.Assert(err, IsNil)
response, err = client.Do(request)
c.Assert(err, IsNil)
c.Assert(response.StatusCode, Equals, http.StatusOK)
decoder := xml.NewDecoder(response.Body)

8
storage-errors.go
View file

@ -32,3 +32,11 @@ var errIsNotRegular = errors.New("Not a regular file type.")
// errVolumeNotFound - cannot find the volume.
var errVolumeNotFound = errors.New("Volume not found.")
// errVolumeAccessDenied - cannot access volume, insufficient
// permissions.
var errVolumeAccessDenied = errors.New("Volume access denied.")
// errVolumeAccessDenied - cannot access file, insufficient
// permissions.
var errFileAccessDenied = errors.New("File access denied.")