2018-10-05 02:44:06 +02:00
/ *
2019-04-09 20:39:42 +02:00
* MinIO Cloud Storage , ( C ) 2018 MinIO , Inc .
2018-10-05 02:44:06 +02:00
*
* Licensed under the Apache License , Version 2.0 ( the "License" ) ;
* you may not use this file except in compliance with the License .
* You may obtain a copy of the License at
*
* http : //www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing , software
* distributed under the License is distributed on an "AS IS" BASIS ,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
* See the License for the specific language governing permissions and
* limitations under the License .
* /
package cmd
import (
2020-02-02 03:11:29 +01:00
"bufio"
2019-03-18 21:07:58 +01:00
"encoding/gob"
"encoding/hex"
2019-02-14 00:29:46 +01:00
"errors"
2018-10-05 02:44:06 +02:00
"fmt"
"io"
2020-03-19 00:19:29 +01:00
"io/ioutil"
2019-03-18 21:07:58 +01:00
"net/http"
2019-12-26 07:05:54 +01:00
"os/user"
2018-10-05 02:44:06 +02:00
"path"
"strconv"
2019-12-24 01:31:03 +01:00
"strings"
2018-10-05 02:44:06 +02:00
"time"
2020-01-31 03:59:22 +01:00
jwtreq "github.com/dgrijalva/jwt-go/request"
2018-10-05 02:44:06 +02:00
"github.com/gorilla/mux"
2019-10-04 19:35:33 +02:00
"github.com/minio/minio/cmd/config"
2019-07-03 07:34:32 +02:00
xhttp "github.com/minio/minio/cmd/http"
2020-01-31 03:59:22 +01:00
xjwt "github.com/minio/minio/cmd/jwt"
2018-10-05 02:44:06 +02:00
"github.com/minio/minio/cmd/logger"
)
2019-10-25 19:37:53 +02:00
var errDiskStale = errors . New ( "disk stale" )
2019-02-14 00:29:46 +01:00
2018-10-05 02:44:06 +02:00
// To abstract a disk over network.
type storageRESTServer struct {
2020-06-13 05:04:01 +02:00
storage * xlStorage
2018-10-05 02:44:06 +02:00
}
func ( s * storageRESTServer ) writeErrorResponse ( w http . ResponseWriter , err error ) {
2020-06-17 23:49:26 +02:00
if errors . Is ( err , errDiskStale ) {
w . WriteHeader ( http . StatusPreconditionFailed )
} else {
w . WriteHeader ( http . StatusForbidden )
}
2018-10-05 02:44:06 +02:00
w . Write ( [ ] byte ( err . Error ( ) ) )
2019-03-18 06:20:26 +01:00
w . ( http . Flusher ) . Flush ( )
2018-10-05 02:44:06 +02:00
}
2019-04-18 08:16:27 +02:00
// DefaultSkewTime - skew time is 15 minutes between minio peers.
const DefaultSkewTime = 15 * time . Minute
2019-02-14 00:29:46 +01:00
// Authenticates storage client's requests and validates for skewed time.
func storageServerRequestValidate ( r * http . Request ) error {
2020-01-31 03:59:22 +01:00
token , err := jwtreq . AuthorizationHeaderExtractor . ExtractToken ( r )
2019-04-03 21:16:19 +02:00
if err != nil {
2020-01-31 03:59:22 +01:00
if err == jwtreq . ErrNoTokenInRequest {
return errNoAuthToken
}
2019-02-14 00:29:46 +01:00
return err
2019-02-12 22:24:14 +01:00
}
2020-01-31 03:59:22 +01:00
claims := xjwt . NewStandardClaims ( )
if err = xjwt . ParseWithStandardClaims ( token , claims , [ ] byte ( globalActiveCred . SecretKey ) ) ; err != nil {
return errAuthentication
}
owner := claims . AccessKey == globalActiveCred . AccessKey || claims . Subject == globalActiveCred . AccessKey
if ! owner {
return errAuthentication
}
if claims . Audience != r . URL . Query ( ) . Encode ( ) {
return errAuthentication
}
2018-10-05 02:44:06 +02:00
requestTimeStr := r . Header . Get ( "X-Minio-Time" )
requestTime , err := time . Parse ( time . RFC3339 , requestTimeStr )
if err != nil {
2019-02-14 00:29:46 +01:00
return err
2018-10-05 02:44:06 +02:00
}
utcNow := UTCNow ( )
delta := requestTime . Sub ( utcNow )
if delta < 0 {
2020-01-31 03:59:22 +01:00
delta *= - 1
2018-10-05 02:44:06 +02:00
}
if delta > DefaultSkewTime {
2019-02-14 00:29:46 +01:00
return fmt . Errorf ( "client time %v is too apart with server time %v" , requestTime , utcNow )
}
2020-01-31 03:59:22 +01:00
2019-02-14 00:29:46 +01:00
return nil
}
// IsValid - To authenticate and verify the time difference.
func ( s * storageRESTServer ) IsValid ( w http . ResponseWriter , r * http . Request ) bool {
if err := storageServerRequestValidate ( r ) ; err != nil {
s . writeErrorResponse ( w , err )
return false
}
2019-10-25 19:37:53 +02:00
diskID := r . URL . Query ( ) . Get ( storageRESTDiskID )
if diskID == "" {
// Request sent empty disk-id, we allow the request
// as the peer might be coming up and trying to read format.json
// or create format.json
return true
2018-10-05 02:44:06 +02:00
}
2020-03-27 22:48:30 +01:00
storedDiskID , err := s . storage . GetDiskID ( )
2019-10-25 19:37:53 +02:00
if err == nil && diskID == storedDiskID {
// If format.json is available and request sent the right disk-id, we allow the request
return true
2019-02-14 00:29:46 +01:00
}
2019-10-25 19:37:53 +02:00
s . writeErrorResponse ( w , errDiskStale )
return false
2019-02-14 00:29:46 +01:00
}
2020-06-17 23:49:26 +02:00
// HealthHandler handler checks if disk is stale
func ( s * storageRESTServer ) HealthHandler ( w http . ResponseWriter , r * http . Request ) {
s . IsValid ( w , r )
}
2018-10-05 02:44:06 +02:00
// DiskInfoHandler - returns disk info.
func ( s * storageRESTServer ) DiskInfoHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
info , err := s . storage . DiskInfo ( )
if err != nil {
2020-07-13 18:51:07 +02:00
info . Error = err . Error ( )
2018-10-05 02:44:06 +02:00
}
defer w . ( http . Flusher ) . Flush ( )
gob . NewEncoder ( w ) . Encode ( info )
}
2019-12-12 15:02:37 +01:00
func ( s * storageRESTServer ) CrawlAndGetDataUsageHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
w . Header ( ) . Set ( xhttp . ContentType , "text/event-stream" )
2020-03-19 00:19:29 +01:00
var cache dataUsageCache
2020-06-12 19:28:21 +02:00
err := cache . deserialize ( r . Body )
2019-12-12 15:02:37 +01:00
if err != nil {
2020-03-19 00:19:29 +01:00
logger . LogIf ( r . Context ( ) , err )
2019-12-12 15:02:37 +01:00
s . writeErrorResponse ( w , err )
return
}
2020-03-19 00:19:29 +01:00
done := keepHTTPResponseAlive ( w )
usageInfo , err := s . storage . CrawlAndGetDataUsage ( r . Context ( ) , cache )
2020-05-12 05:41:38 +02:00
done ( err )
2020-03-19 00:19:29 +01:00
if err != nil {
return
}
w . Write ( usageInfo . serialize ( ) )
2019-12-12 15:02:37 +01:00
w . ( http . Flusher ) . Flush ( )
}
2018-10-05 02:44:06 +02:00
// MakeVolHandler - make a volume.
func ( s * storageRESTServer ) MakeVolHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
err := s . storage . MakeVol ( volume )
if err != nil {
s . writeErrorResponse ( w , err )
}
}
2019-12-24 01:31:03 +01:00
// MakeVolBulkHandler - create multiple volumes as a bulk operation.
func ( s * storageRESTServer ) MakeVolBulkHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volumes := strings . Split ( vars [ storageRESTVolumes ] , "," )
err := s . storage . MakeVolBulk ( volumes ... )
if err != nil {
s . writeErrorResponse ( w , err )
}
}
2018-10-05 02:44:06 +02:00
// ListVolsHandler - list volumes.
func ( s * storageRESTServer ) ListVolsHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
infos , err := s . storage . ListVols ( )
if err != nil {
s . writeErrorResponse ( w , err )
return
}
gob . NewEncoder ( w ) . Encode ( & infos )
2019-10-01 22:12:15 +02:00
w . ( http . Flusher ) . Flush ( )
2018-10-05 02:44:06 +02:00
}
// StatVolHandler - stat a volume.
func ( s * storageRESTServer ) StatVolHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
info , err := s . storage . StatVol ( volume )
if err != nil {
s . writeErrorResponse ( w , err )
return
}
gob . NewEncoder ( w ) . Encode ( info )
2019-10-01 22:12:15 +02:00
w . ( http . Flusher ) . Flush ( )
2018-10-05 02:44:06 +02:00
}
// DeleteVolumeHandler - delete a volume.
func ( s * storageRESTServer ) DeleteVolHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
2020-03-28 05:52:59 +01:00
forceDelete := vars [ storageRESTForceDelete ] == "true"
err := s . storage . DeleteVol ( volume , forceDelete )
2018-10-05 02:44:06 +02:00
if err != nil {
s . writeErrorResponse ( w , err )
}
}
2019-01-17 13:58:18 +01:00
// AppendFileHandler - append data from the request to the file specified.
func ( s * storageRESTServer ) AppendFileHandler ( w http . ResponseWriter , r * http . Request ) {
2018-10-05 02:44:06 +02:00
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
filePath := vars [ storageRESTFilePath ]
2019-01-17 13:58:18 +01:00
buf := make ( [ ] byte , r . ContentLength )
_ , err := io . ReadFull ( r . Body , buf )
2018-10-05 02:44:06 +02:00
if err != nil {
s . writeErrorResponse ( w , err )
return
}
2019-01-17 13:58:18 +01:00
err = s . storage . AppendFile ( volume , filePath , buf )
2018-10-05 02:44:06 +02:00
if err != nil {
s . writeErrorResponse ( w , err )
}
}
2019-01-17 13:58:18 +01:00
// CreateFileHandler - fallocate() space for a file and copy the contents from the request.
func ( s * storageRESTServer ) CreateFileHandler ( w http . ResponseWriter , r * http . Request ) {
2018-10-05 02:44:06 +02:00
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
filePath := vars [ storageRESTFilePath ]
2019-01-17 13:58:18 +01:00
fileSizeStr := vars [ storageRESTLength ]
fileSize , err := strconv . Atoi ( fileSizeStr )
2018-10-05 02:44:06 +02:00
if err != nil {
s . writeErrorResponse ( w , err )
return
}
2019-01-17 13:58:18 +01:00
err = s . storage . CreateFile ( volume , filePath , int64 ( fileSize ) , r . Body )
2018-10-05 02:44:06 +02:00
if err != nil {
s . writeErrorResponse ( w , err )
}
}
2020-06-13 05:04:01 +02:00
// DeleteVersion delete updated metadata.
func ( s * storageRESTServer ) DeleteVersionHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
filePath := vars [ storageRESTFilePath ]
2020-06-27 01:49:49 +02:00
if r . ContentLength < 0 {
s . writeErrorResponse ( w , errInvalidArgument )
return
}
2020-06-23 19:20:31 +02:00
var fi FileInfo
2020-06-27 01:49:49 +02:00
if err := gob . NewDecoder ( r . Body ) . Decode ( & fi ) ; err != nil {
2020-06-23 19:20:31 +02:00
s . writeErrorResponse ( w , err )
return
}
err := s . storage . DeleteVersion ( volume , filePath , fi )
2020-06-13 05:04:01 +02:00
if err != nil {
s . writeErrorResponse ( w , err )
}
}
2020-06-27 01:49:49 +02:00
// ReadVersion read metadata of versionID
2020-06-13 05:04:01 +02:00
func ( s * storageRESTServer ) ReadVersionHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
filePath := vars [ storageRESTFilePath ]
versionID := vars [ storageRESTVersionID ]
fi , err := s . storage . ReadVersion ( volume , filePath , versionID )
if err != nil {
s . writeErrorResponse ( w , err )
return
}
gob . NewEncoder ( w ) . Encode ( fi )
w . ( http . Flusher ) . Flush ( )
}
// WriteMetadata write new updated metadata.
func ( s * storageRESTServer ) WriteMetadataHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
filePath := vars [ storageRESTFilePath ]
if r . ContentLength < 0 {
s . writeErrorResponse ( w , errInvalidArgument )
return
}
var fi FileInfo
err := gob . NewDecoder ( r . Body ) . Decode ( & fi )
if err != nil {
s . writeErrorResponse ( w , err )
return
}
err = s . storage . WriteMetadata ( volume , filePath , fi )
if err != nil {
s . writeErrorResponse ( w , err )
}
}
2018-11-14 15:18:35 +01:00
// WriteAllHandler - write to file all content.
func ( s * storageRESTServer ) WriteAllHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
filePath := vars [ storageRESTFilePath ]
if r . ContentLength < 0 {
s . writeErrorResponse ( w , errInvalidArgument )
return
}
2019-05-22 22:47:15 +02:00
err := s . storage . WriteAll ( volume , filePath , io . LimitReader ( r . Body , r . ContentLength ) )
2018-11-14 15:18:35 +01:00
if err != nil {
s . writeErrorResponse ( w , err )
}
}
2020-06-13 05:04:01 +02:00
// CheckPartsHandler - check if a file metadata exists.
func ( s * storageRESTServer ) CheckPartsHandler ( w http . ResponseWriter , r * http . Request ) {
2018-10-05 02:44:06 +02:00
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
filePath := vars [ storageRESTFilePath ]
2020-06-13 05:04:01 +02:00
if r . ContentLength < 0 {
s . writeErrorResponse ( w , errInvalidArgument )
return
}
var fi FileInfo
if err := gob . NewDecoder ( r . Body ) . Decode ( & fi ) ; err != nil {
2018-10-05 02:44:06 +02:00
s . writeErrorResponse ( w , err )
return
}
2020-06-13 05:04:01 +02:00
if err := s . storage . CheckParts ( volume , filePath , fi ) ; err != nil {
s . writeErrorResponse ( w , err )
}
}
// CheckFileHandler - check if a file metadata exists.
func ( s * storageRESTServer ) CheckFileHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
filePath := vars [ storageRESTFilePath ]
if err := s . storage . CheckFile ( volume , filePath ) ; err != nil {
s . writeErrorResponse ( w , err )
}
2018-10-05 02:44:06 +02:00
}
// ReadAllHandler - read all the contents of a file.
func ( s * storageRESTServer ) ReadAllHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
filePath := vars [ storageRESTFilePath ]
buf , err := s . storage . ReadAll ( volume , filePath )
if err != nil {
s . writeErrorResponse ( w , err )
return
}
2019-07-03 07:34:32 +02:00
w . Header ( ) . Set ( xhttp . ContentLength , strconv . Itoa ( len ( buf ) ) )
2018-10-05 02:44:06 +02:00
w . Write ( buf )
2019-03-18 06:20:26 +01:00
w . ( http . Flusher ) . Flush ( )
2018-10-05 02:44:06 +02:00
}
// ReadFileHandler - read section of a file.
func ( s * storageRESTServer ) ReadFileHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
filePath := vars [ storageRESTFilePath ]
offset , err := strconv . Atoi ( vars [ storageRESTOffset ] )
if err != nil {
s . writeErrorResponse ( w , err )
return
}
length , err := strconv . Atoi ( vars [ storageRESTLength ] )
if err != nil {
s . writeErrorResponse ( w , err )
return
}
if offset < 0 || length < 0 {
s . writeErrorResponse ( w , errInvalidArgument )
return
}
var verifier * BitrotVerifier
if vars [ storageRESTBitrotAlgo ] != "" {
hashStr := vars [ storageRESTBitrotHash ]
var hash [ ] byte
hash , err = hex . DecodeString ( hashStr )
if err != nil {
s . writeErrorResponse ( w , err )
return
}
verifier = NewBitrotVerifier ( BitrotAlgorithmFromString ( vars [ storageRESTBitrotAlgo ] ) , hash )
}
buf := make ( [ ] byte , length )
_ , err = s . storage . ReadFile ( volume , filePath , int64 ( offset ) , buf , verifier )
if err != nil {
s . writeErrorResponse ( w , err )
return
}
2019-07-03 07:34:32 +02:00
w . Header ( ) . Set ( xhttp . ContentLength , strconv . Itoa ( len ( buf ) ) )
2018-10-05 02:44:06 +02:00
w . Write ( buf )
2019-03-18 06:20:26 +01:00
w . ( http . Flusher ) . Flush ( )
2018-10-05 02:44:06 +02:00
}
2019-01-17 13:58:18 +01:00
// ReadFileHandler - read section of a file.
func ( s * storageRESTServer ) ReadFileStreamHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
filePath := vars [ storageRESTFilePath ]
offset , err := strconv . Atoi ( vars [ storageRESTOffset ] )
if err != nil {
s . writeErrorResponse ( w , err )
return
}
length , err := strconv . Atoi ( vars [ storageRESTLength ] )
if err != nil {
s . writeErrorResponse ( w , err )
return
}
2019-03-18 06:20:26 +01:00
2019-01-17 13:58:18 +01:00
rc , err := s . storage . ReadFileStream ( volume , filePath , int64 ( offset ) , int64 ( length ) )
if err != nil {
s . writeErrorResponse ( w , err )
return
}
defer rc . Close ( )
2019-09-26 08:08:24 +02:00
2019-07-03 07:34:32 +02:00
w . Header ( ) . Set ( xhttp . ContentLength , strconv . Itoa ( length ) )
2019-03-18 06:20:26 +01:00
2019-01-17 13:58:18 +01:00
io . Copy ( w , rc )
2019-03-18 06:20:26 +01:00
w . ( http . Flusher ) . Flush ( )
2019-01-17 13:58:18 +01:00
}
2020-06-13 05:04:01 +02:00
// WalkHandler - remote caller to start walking at a requested directory path.
func ( s * storageRESTServer ) WalkSplunkHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
dirPath := vars [ storageRESTDirPath ]
markerPath := vars [ storageRESTMarkerPath ]
w . Header ( ) . Set ( xhttp . ContentType , "text/event-stream" )
encoder := gob . NewEncoder ( w )
2019-05-14 22:49:10 +02:00
2020-06-13 05:04:01 +02:00
fch , err := s . storage . WalkSplunk ( volume , dirPath , markerPath , r . Context ( ) . Done ( ) )
2019-05-14 22:49:10 +02:00
if err != nil {
2020-06-13 05:04:01 +02:00
s . writeErrorResponse ( w , err )
return
2019-05-14 22:49:10 +02:00
}
2020-06-13 05:04:01 +02:00
for fi := range fch {
encoder . Encode ( & fi )
2019-05-14 22:49:10 +02:00
}
}
2020-06-13 05:04:01 +02:00
// WalkVersionsHandler - remote caller to start walking at a requested directory path.
func ( s * storageRESTServer ) WalkVersionsHandler ( w http . ResponseWriter , r * http . Request ) {
2020-03-23 03:23:47 +01:00
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
dirPath := vars [ storageRESTDirPath ]
markerPath := vars [ storageRESTMarkerPath ]
2020-06-13 05:04:01 +02:00
recursive , err := strconv . ParseBool ( vars [ storageRESTRecursive ] )
if err != nil {
s . writeErrorResponse ( w , err )
return
}
2020-03-23 03:23:47 +01:00
w . Header ( ) . Set ( xhttp . ContentType , "text/event-stream" )
encoder := gob . NewEncoder ( w )
2020-06-13 05:04:01 +02:00
fch , err := s . storage . WalkVersions ( volume , dirPath , markerPath , recursive , r . Context ( ) . Done ( ) )
2020-03-23 03:23:47 +01:00
if err != nil {
s . writeErrorResponse ( w , err )
return
}
for fi := range fch {
2020-07-02 19:29:50 +02:00
logger . LogIf ( r . Context ( ) , encoder . Encode ( & fi ) )
2020-03-23 03:23:47 +01:00
}
}
2019-05-14 22:49:10 +02:00
// WalkHandler - remote caller to start walking at a requested directory path.
func ( s * storageRESTServer ) WalkHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
dirPath := vars [ storageRESTDirPath ]
markerPath := vars [ storageRESTMarkerPath ]
recursive , err := strconv . ParseBool ( vars [ storageRESTRecursive ] )
if err != nil {
s . writeErrorResponse ( w , err )
return
}
2020-03-22 06:10:13 +01:00
w . Header ( ) . Set ( xhttp . ContentType , "text/event-stream" )
encoder := gob . NewEncoder ( w )
2020-06-13 05:04:01 +02:00
fch , err := s . storage . Walk ( volume , dirPath , markerPath , recursive , r . Context ( ) . Done ( ) )
2019-05-14 22:49:10 +02:00
if err != nil {
2020-03-23 03:23:47 +01:00
s . writeErrorResponse ( w , err )
2019-05-14 22:49:10 +02:00
return
}
for fi := range fch {
2020-07-02 19:29:50 +02:00
logger . LogIf ( r . Context ( ) , encoder . Encode ( & fi ) )
2019-05-14 22:49:10 +02:00
}
}
2018-10-05 02:44:06 +02:00
// ListDirHandler - list a directory.
func ( s * storageRESTServer ) ListDirHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
dirPath := vars [ storageRESTDirPath ]
count , err := strconv . Atoi ( vars [ storageRESTCount ] )
if err != nil {
s . writeErrorResponse ( w , err )
return
}
2020-03-22 06:10:13 +01:00
2020-06-13 05:04:01 +02:00
entries , err := s . storage . ListDir ( volume , dirPath , count )
2018-10-05 02:44:06 +02:00
if err != nil {
s . writeErrorResponse ( w , err )
return
}
gob . NewEncoder ( w ) . Encode ( & entries )
2019-10-01 22:12:15 +02:00
w . ( http . Flusher ) . Flush ( )
2018-10-05 02:44:06 +02:00
}
// DeleteFileHandler - delete a file.
func ( s * storageRESTServer ) DeleteFileHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
filePath := vars [ storageRESTFilePath ]
err := s . storage . DeleteFile ( volume , filePath )
if err != nil {
s . writeErrorResponse ( w , err )
}
}
2020-06-13 05:04:01 +02:00
// DeleteVersionsErrsResp - collection of delete errors
// for bulk version deletes
type DeleteVersionsErrsResp struct {
2019-10-01 04:01:28 +02:00
Errs [ ] error
}
2020-06-13 05:04:01 +02:00
// DeleteVersionsHandler - delete a set of a versions.
func ( s * storageRESTServer ) DeleteVersionsHandler ( w http . ResponseWriter , r * http . Request ) {
2019-05-13 21:25:49 +02:00
if ! s . IsValid ( w , r ) {
return
}
2020-06-13 05:04:01 +02:00
2019-05-13 21:25:49 +02:00
vars := r . URL . Query ( )
volume := vars . Get ( storageRESTVolume )
2020-02-02 03:11:29 +01:00
2020-06-13 05:04:01 +02:00
totalVersions , err := strconv . Atoi ( vars . Get ( storageRESTTotalVersions ) )
if err != nil {
2020-02-02 03:11:29 +01:00
s . writeErrorResponse ( w , err )
return
}
2019-05-13 21:25:49 +02:00
2020-06-13 05:04:01 +02:00
versions := make ( [ ] FileInfo , totalVersions )
decoder := gob . NewDecoder ( r . Body )
for i := 0 ; i < totalVersions ; i ++ {
if err := decoder . Decode ( & versions [ i ] ) ; err != nil {
s . writeErrorResponse ( w , err )
return
}
}
dErrsResp := & DeleteVersionsErrsResp { Errs : make ( [ ] error , totalVersions ) }
2020-03-22 06:10:13 +01:00
2020-03-11 16:56:36 +01:00
w . Header ( ) . Set ( xhttp . ContentType , "text/event-stream" )
encoder := gob . NewEncoder ( w )
2020-03-19 00:19:29 +01:00
done := keepHTTPResponseAlive ( w )
2020-06-13 05:04:01 +02:00
errs := s . storage . DeleteVersions ( volume , versions )
2020-05-12 05:41:38 +02:00
done ( nil )
2020-06-13 05:04:01 +02:00
for idx := range versions {
if errs [ idx ] != nil {
dErrsResp . Errs [ idx ] = StorageErr ( errs [ idx ] . Error ( ) )
2019-10-01 04:01:28 +02:00
}
}
2020-03-11 16:56:36 +01:00
encoder . Encode ( dErrsResp )
w . ( http . Flusher ) . Flush ( )
}
2020-06-13 05:04:01 +02:00
// RenameDataHandler - renames a meta object and data dir to destination.
func ( s * storageRESTServer ) RenameDataHandler ( w http . ResponseWriter , r * http . Request ) {
2020-03-11 16:56:36 +01:00
if ! s . IsValid ( w , r ) {
return
}
2020-06-13 05:04:01 +02:00
vars := mux . Vars ( r )
srcVolume := vars [ storageRESTSrcVolume ]
srcFilePath := vars [ storageRESTSrcPath ]
dataDir := vars [ storageRESTDataDir ]
dstVolume := vars [ storageRESTDstVolume ]
dstFilePath := vars [ storageRESTDstPath ]
err := s . storage . RenameData ( srcVolume , srcFilePath , dataDir , dstVolume , dstFilePath )
if err != nil {
2020-03-11 16:56:36 +01:00
s . writeErrorResponse ( w , err )
}
2019-05-13 21:25:49 +02:00
}
2018-10-05 02:44:06 +02:00
// RenameFileHandler - rename a file.
func ( s * storageRESTServer ) RenameFileHandler ( w http . ResponseWriter , r * http . Request ) {
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
srcVolume := vars [ storageRESTSrcVolume ]
srcFilePath := vars [ storageRESTSrcPath ]
dstVolume := vars [ storageRESTDstVolume ]
dstFilePath := vars [ storageRESTDstPath ]
err := s . storage . RenameFile ( srcVolume , srcFilePath , dstVolume , dstFilePath )
if err != nil {
s . writeErrorResponse ( w , err )
}
}
2020-03-19 00:19:29 +01:00
// keepHTTPResponseAlive can be used to avoid timeouts with long storage
2019-12-12 15:02:37 +01:00
// operations, such as bitrot verification or data usage crawling.
2020-03-19 00:19:29 +01:00
// Every 10 seconds a space character is sent.
// The returned function should always be called to release resources.
2020-05-12 05:41:38 +02:00
// An optional error can be sent which will be picked as text only error,
// without its original type by the receiver.
2020-03-19 00:19:29 +01:00
// waitForHTTPResponse should be used to the receiving side.
2020-05-12 05:41:38 +02:00
func keepHTTPResponseAlive ( w http . ResponseWriter ) func ( error ) {
doneCh := make ( chan error )
2019-07-08 22:51:18 +02:00
go func ( ) {
2020-03-19 00:19:29 +01:00
defer close ( doneCh )
2019-07-08 22:51:18 +02:00
ticker := time . NewTicker ( time . Second * 10 )
for {
select {
case <- ticker . C :
2020-05-12 05:41:38 +02:00
// Response not ready, write a filler byte.
w . Write ( [ ] byte { 32 } )
2019-07-08 22:51:18 +02:00
w . ( http . Flusher ) . Flush ( )
2020-05-12 05:41:38 +02:00
case err := <- doneCh :
if err != nil {
w . Write ( [ ] byte { 1 } )
w . Write ( [ ] byte ( err . Error ( ) ) )
} else {
w . Write ( [ ] byte { 0 } )
}
2019-07-08 22:51:18 +02:00
ticker . Stop ( )
return
}
}
} ( )
2020-05-12 05:41:38 +02:00
return func ( err error ) {
if doneCh == nil {
return
}
2020-03-19 00:19:29 +01:00
// Indicate we are ready to write.
2020-05-12 05:41:38 +02:00
doneCh <- err
2020-03-19 00:19:29 +01:00
// Wait for channel to be closed so we don't race on writes.
<- doneCh
2020-05-12 05:41:38 +02:00
// Clear so we can be called multiple times without crashing.
doneCh = nil
2020-03-19 00:19:29 +01:00
}
}
// waitForHTTPResponse will wait for responses where keepHTTPResponseAlive
// has been used.
// The returned reader contains the payload.
func waitForHTTPResponse ( respBody io . Reader ) ( io . Reader , error ) {
reader := bufio . NewReader ( respBody )
for {
b , err := reader . ReadByte ( )
if err != nil {
return nil , err
}
2020-05-12 05:41:38 +02:00
// Check if we have a response ready or a filler byte.
switch b {
case 0 :
return reader , nil
case 1 :
errorText , err := ioutil . ReadAll ( reader )
if err != nil {
return nil , err
2020-03-19 00:19:29 +01:00
}
2020-05-12 05:41:38 +02:00
return nil , errors . New ( string ( errorText ) )
case 32 :
continue
default :
return nil , fmt . Errorf ( "unexpected filler byte: %d" , b )
2020-03-19 00:19:29 +01:00
}
}
2019-07-08 22:51:18 +02:00
}
// VerifyFileResp - VerifyFile()'s response.
type VerifyFileResp struct {
Err error
}
2020-06-13 05:04:01 +02:00
// VerifyFileHandler - Verify all part of file for bitrot errors.
func ( s * storageRESTServer ) VerifyFileHandler ( w http . ResponseWriter , r * http . Request ) {
2019-07-08 22:51:18 +02:00
if ! s . IsValid ( w , r ) {
return
}
vars := mux . Vars ( r )
volume := vars [ storageRESTVolume ]
filePath := vars [ storageRESTFilePath ]
2020-06-13 05:04:01 +02:00
if r . ContentLength < 0 {
s . writeErrorResponse ( w , errInvalidArgument )
2019-07-13 01:29:44 +02:00
return
}
2020-06-13 05:04:01 +02:00
var fi FileInfo
err := gob . NewDecoder ( r . Body ) . Decode ( & fi )
2019-07-08 22:51:18 +02:00
if err != nil {
s . writeErrorResponse ( w , err )
return
}
2020-06-13 05:04:01 +02:00
2019-07-11 22:19:25 +02:00
w . Header ( ) . Set ( xhttp . ContentType , "text/event-stream" )
2019-10-01 22:12:15 +02:00
encoder := gob . NewEncoder ( w )
2020-03-19 00:19:29 +01:00
done := keepHTTPResponseAlive ( w )
2020-06-13 05:04:01 +02:00
err = s . storage . VerifyFile ( volume , filePath , fi )
2020-05-12 05:41:38 +02:00
done ( nil )
2019-10-01 22:12:15 +02:00
vresp := & VerifyFileResp { }
if err != nil {
2020-01-15 03:45:17 +01:00
vresp . Err = StorageErr ( err . Error ( ) )
2019-10-01 22:12:15 +02:00
}
encoder . Encode ( vresp )
w . ( http . Flusher ) . Flush ( )
2019-07-08 22:51:18 +02:00
}
2018-10-05 02:44:06 +02:00
// registerStorageRPCRouter - register storage rpc router.
2019-11-20 02:42:27 +01:00
func registerStorageRESTHandlers ( router * mux . Router , endpointZones EndpointZones ) {
for _ , ep := range endpointZones {
for _ , endpoint := range ep . Endpoints {
if ! endpoint . IsLocal {
continue
}
2020-06-13 05:04:01 +02:00
storage , err := newXLStorage ( endpoint . Path , endpoint . Host )
2019-11-20 02:42:27 +01:00
if err != nil {
2020-05-08 01:12:16 +02:00
if err == errMinDiskSize {
logger . Fatal ( config . ErrUnableToWriteInBackend ( err ) . Hint ( err . Error ( ) ) , "Unable to initialize backend" )
} else if err == errUnsupportedDisk {
hint := fmt . Sprintf ( "'%s' does not support O_DIRECT flags, refusing to use" , endpoint . Path )
logger . Fatal ( config . ErrUnsupportedBackend ( err ) . Hint ( hint ) , "Unable to initialize backend" )
}
2019-12-26 07:05:54 +01:00
// Show a descriptive error with a hint about how to fix it.
var username string
if u , err := user . Current ( ) ; err == nil {
username = u . Username
} else {
username = "<your-username>"
}
2020-06-17 00:32:29 +02:00
hint := fmt . Sprintf ( "Run the following command to add the convenient permissions: `sudo chown -R %s %s && sudo chmod u+rxw %s`" , username , endpoint . Path , endpoint . Path )
2020-06-13 05:04:01 +02:00
logger . Fatal ( config . ErrUnableToWriteInBackend ( err ) . Hint ( hint ) , "Unable to initialize posix backend" )
2019-11-20 02:42:27 +01:00
}
2018-10-05 02:44:06 +02:00
2019-11-20 02:42:27 +01:00
server := & storageRESTServer { storage : storage }
subrouter := router . PathPrefix ( path . Join ( storageRESTPrefix , endpoint . Path ) ) . Subrouter ( )
2020-06-17 23:49:26 +02:00
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodHealth ) . HandlerFunc ( httpTraceHdrs ( server . HealthHandler ) )
2019-11-20 02:42:27 +01:00
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodDiskInfo ) . HandlerFunc ( httpTraceHdrs ( server . DiskInfoHandler ) )
2019-12-12 15:02:37 +01:00
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodCrawlAndGetDataUsage ) . HandlerFunc ( httpTraceHdrs ( server . CrawlAndGetDataUsageHandler ) )
2019-11-20 02:42:27 +01:00
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodMakeVol ) . HandlerFunc ( httpTraceHdrs ( server . MakeVolHandler ) ) . Queries ( restQueries ( storageRESTVolume ) ... )
2019-12-24 01:31:03 +01:00
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodMakeVolBulk ) . HandlerFunc ( httpTraceHdrs ( server . MakeVolBulkHandler ) ) . Queries ( restQueries ( storageRESTVolumes ) ... )
2019-11-20 02:42:27 +01:00
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodStatVol ) . HandlerFunc ( httpTraceHdrs ( server . StatVolHandler ) ) . Queries ( restQueries ( storageRESTVolume ) ... )
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodDeleteVol ) . HandlerFunc ( httpTraceHdrs ( server . DeleteVolHandler ) ) . Queries ( restQueries ( storageRESTVolume ) ... )
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodListVols ) . HandlerFunc ( httpTraceHdrs ( server . ListVolsHandler ) )
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodAppendFile ) . HandlerFunc ( httpTraceHdrs ( server . AppendFileHandler ) ) .
Queries ( restQueries ( storageRESTVolume , storageRESTFilePath ) ... )
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodWriteAll ) . HandlerFunc ( httpTraceHdrs ( server . WriteAllHandler ) ) .
Queries ( restQueries ( storageRESTVolume , storageRESTFilePath ) ... )
2020-06-13 05:04:01 +02:00
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodWriteMetadata ) . HandlerFunc ( httpTraceHdrs ( server . WriteMetadataHandler ) ) .
Queries ( restQueries ( storageRESTVolume , storageRESTFilePath ) ... )
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodDeleteVersion ) . HandlerFunc ( httpTraceHdrs ( server . DeleteVersionHandler ) ) .
2020-06-27 01:49:49 +02:00
Queries ( restQueries ( storageRESTVolume , storageRESTFilePath ) ... )
2020-06-13 05:04:01 +02:00
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodReadVersion ) . HandlerFunc ( httpTraceHdrs ( server . ReadVersionHandler ) ) .
Queries ( restQueries ( storageRESTVolume , storageRESTFilePath , storageRESTVersionID ) ... )
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodRenameData ) . HandlerFunc ( httpTraceHdrs ( server . RenameDataHandler ) ) .
Queries ( restQueries ( storageRESTSrcVolume , storageRESTSrcPath , storageRESTDataDir ,
storageRESTDstVolume , storageRESTDstPath ) ... )
2019-11-20 02:42:27 +01:00
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodCreateFile ) . HandlerFunc ( httpTraceHdrs ( server . CreateFileHandler ) ) .
Queries ( restQueries ( storageRESTVolume , storageRESTFilePath , storageRESTLength ) ... )
2020-06-13 05:04:01 +02:00
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodCheckFile ) . HandlerFunc ( httpTraceHdrs ( server . CheckFileHandler ) ) .
2019-11-20 02:42:27 +01:00
Queries ( restQueries ( storageRESTVolume , storageRESTFilePath ) ... )
2020-06-13 05:04:01 +02:00
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodCheckParts ) . HandlerFunc ( httpTraceHdrs ( server . CheckPartsHandler ) ) .
Queries ( restQueries ( storageRESTVolume , storageRESTFilePath ) ... )
2019-11-20 02:42:27 +01:00
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodReadAll ) . HandlerFunc ( httpTraceHdrs ( server . ReadAllHandler ) ) .
Queries ( restQueries ( storageRESTVolume , storageRESTFilePath ) ... )
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodReadFile ) . HandlerFunc ( httpTraceHdrs ( server . ReadFileHandler ) ) .
Queries ( restQueries ( storageRESTVolume , storageRESTFilePath , storageRESTOffset , storageRESTLength , storageRESTBitrotAlgo , storageRESTBitrotHash ) ... )
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodReadFileStream ) . HandlerFunc ( httpTraceHdrs ( server . ReadFileStreamHandler ) ) .
Queries ( restQueries ( storageRESTVolume , storageRESTFilePath , storageRESTOffset , storageRESTLength ) ... )
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodListDir ) . HandlerFunc ( httpTraceHdrs ( server . ListDirHandler ) ) .
2020-06-13 05:04:01 +02:00
Queries ( restQueries ( storageRESTVolume , storageRESTDirPath , storageRESTCount ) ... )
2019-11-20 02:42:27 +01:00
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodWalk ) . HandlerFunc ( httpTraceHdrs ( server . WalkHandler ) ) .
2020-06-13 05:04:01 +02:00
Queries ( restQueries ( storageRESTVolume , storageRESTDirPath , storageRESTMarkerPath , storageRESTRecursive ) ... )
2020-03-23 03:23:47 +01:00
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodWalkSplunk ) . HandlerFunc ( httpTraceHdrs ( server . WalkSplunkHandler ) ) .
Queries ( restQueries ( storageRESTVolume , storageRESTDirPath , storageRESTMarkerPath ) ... )
2020-06-13 05:04:01 +02:00
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodWalkVersions ) . HandlerFunc ( httpTraceHdrs ( server . WalkVersionsHandler ) ) .
2020-07-02 19:29:50 +02:00
Queries ( restQueries ( storageRESTVolume , storageRESTDirPath , storageRESTMarkerPath , storageRESTRecursive ) ... )
2020-06-13 05:04:01 +02:00
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodDeleteVersions ) . HandlerFunc ( httpTraceHdrs ( server . DeleteVersionsHandler ) ) .
Queries ( restQueries ( storageRESTVolume , storageRESTTotalVersions ) ... )
2019-11-20 02:42:27 +01:00
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodDeleteFile ) . HandlerFunc ( httpTraceHdrs ( server . DeleteFileHandler ) ) .
Queries ( restQueries ( storageRESTVolume , storageRESTFilePath ) ... )
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodRenameFile ) . HandlerFunc ( httpTraceHdrs ( server . RenameFileHandler ) ) .
Queries ( restQueries ( storageRESTSrcVolume , storageRESTSrcPath , storageRESTDstVolume , storageRESTDstPath ) ... )
2020-06-13 05:04:01 +02:00
subrouter . Methods ( http . MethodPost ) . Path ( storageRESTVersionPrefix + storageRESTMethodVerifyFile ) . HandlerFunc ( httpTraceHdrs ( server . VerifyFileHandler ) ) .
Queries ( restQueries ( storageRESTVolume , storageRESTFilePath ) ... )
2019-11-20 02:42:27 +01:00
}
2018-10-05 02:44:06 +02:00
}
}