2017-04-12 00:44:27 +02:00
|
|
|
/*
|
2019-09-12 20:06:12 +02:00
|
|
|
* MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
|
2017-04-12 00:44:27 +02:00
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
package cmd
|
|
|
|
|
|
|
|
import (
|
2019-04-19 19:26:44 +02:00
|
|
|
"context"
|
2017-04-12 00:44:27 +02:00
|
|
|
"fmt"
|
|
|
|
"net"
|
|
|
|
"net/url"
|
|
|
|
"path"
|
2017-08-16 00:10:50 +02:00
|
|
|
"path/filepath"
|
2017-11-12 03:39:53 +01:00
|
|
|
"runtime"
|
2017-04-12 00:44:27 +02:00
|
|
|
"strconv"
|
|
|
|
"strings"
|
2019-04-19 19:26:44 +02:00
|
|
|
"time"
|
2017-04-12 00:44:27 +02:00
|
|
|
|
2019-04-19 19:26:44 +02:00
|
|
|
humanize "github.com/dustin/go-humanize"
|
2019-05-30 01:35:12 +02:00
|
|
|
"github.com/minio/minio-go/v6/pkg/set"
|
2019-10-04 19:35:33 +02:00
|
|
|
"github.com/minio/minio/cmd/config"
|
2018-04-06 00:04:40 +02:00
|
|
|
"github.com/minio/minio/cmd/logger"
|
2019-10-04 19:35:33 +02:00
|
|
|
"github.com/minio/minio/pkg/env"
|
2017-08-16 00:10:50 +02:00
|
|
|
"github.com/minio/minio/pkg/mountinfo"
|
2017-04-12 00:44:27 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
// EndpointType - enum for endpoint type.
|
|
|
|
type EndpointType int
|
|
|
|
|
|
|
|
const (
|
|
|
|
// PathEndpointType - path style endpoint type enum.
|
|
|
|
PathEndpointType EndpointType = iota + 1
|
|
|
|
|
|
|
|
// URLEndpointType - URL style endpoint type enum.
|
|
|
|
URLEndpointType
|
2019-04-19 19:26:44 +02:00
|
|
|
|
|
|
|
retryInterval = 5 // In Seconds.
|
2017-04-12 00:44:27 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
// Endpoint - any type of endpoint.
|
|
|
|
type Endpoint struct {
|
|
|
|
*url.URL
|
2020-01-10 11:35:06 +01:00
|
|
|
IsLocal bool
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
func (endpoint Endpoint) String() string {
|
|
|
|
if endpoint.Host == "" {
|
|
|
|
return endpoint.Path
|
|
|
|
}
|
|
|
|
|
|
|
|
return endpoint.URL.String()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Type - returns type of endpoint.
|
|
|
|
func (endpoint Endpoint) Type() EndpointType {
|
|
|
|
if endpoint.Host == "" {
|
|
|
|
return PathEndpointType
|
|
|
|
}
|
|
|
|
|
|
|
|
return URLEndpointType
|
|
|
|
}
|
|
|
|
|
2019-11-20 02:42:27 +01:00
|
|
|
// HTTPS - returns true if secure for URLEndpointType.
|
|
|
|
func (endpoint Endpoint) HTTPS() bool {
|
2017-12-04 07:47:12 +01:00
|
|
|
return endpoint.Scheme == "https"
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
|
|
|
|
2019-04-19 19:26:44 +02:00
|
|
|
// UpdateIsLocal - resolves the host and updates if it is local or not.
|
2019-11-26 20:42:10 +01:00
|
|
|
func (endpoint *Endpoint) UpdateIsLocal() (err error) {
|
2019-04-19 19:26:44 +02:00
|
|
|
if !endpoint.IsLocal {
|
2019-11-26 20:42:10 +01:00
|
|
|
endpoint.IsLocal, err = isLocalHost(endpoint.Hostname(), endpoint.Port(), globalMinioPort)
|
2019-04-19 19:26:44 +02:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-04-12 00:44:27 +02:00
|
|
|
// NewEndpoint - returns new endpoint based on given arguments.
|
2017-06-22 04:53:09 +02:00
|
|
|
func NewEndpoint(arg string) (ep Endpoint, e error) {
|
2017-04-12 00:44:27 +02:00
|
|
|
// isEmptyPath - check whether given path is not empty.
|
|
|
|
isEmptyPath := func(path string) bool {
|
2019-08-06 21:08:58 +02:00
|
|
|
return path == "" || path == SlashSeparator || path == `\`
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
if isEmptyPath(arg) {
|
2017-06-22 04:53:09 +02:00
|
|
|
return ep, fmt.Errorf("empty or root endpoint is not supported")
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
var isLocal bool
|
2019-04-19 19:26:44 +02:00
|
|
|
var host string
|
2017-04-12 00:44:27 +02:00
|
|
|
u, err := url.Parse(arg)
|
|
|
|
if err == nil && u.Host != "" {
|
|
|
|
// URL style of endpoint.
|
|
|
|
// Valid URL style endpoint is
|
|
|
|
// - Scheme field must contain "http" or "https"
|
|
|
|
// - All field should be empty except Host and Path.
|
|
|
|
if !((u.Scheme == "http" || u.Scheme == "https") &&
|
2019-02-13 13:59:36 +01:00
|
|
|
u.User == nil && u.Opaque == "" && !u.ForceQuery && u.RawQuery == "" && u.Fragment == "") {
|
2017-06-22 04:53:09 +02:00
|
|
|
return ep, fmt.Errorf("invalid URL endpoint format")
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
|
|
|
|
2019-04-19 19:26:44 +02:00
|
|
|
var port string
|
2017-08-16 00:10:50 +02:00
|
|
|
host, port, err = net.SplitHostPort(u.Host)
|
2017-04-12 00:44:27 +02:00
|
|
|
if err != nil {
|
|
|
|
if !strings.Contains(err.Error(), "missing port in address") {
|
2019-12-02 18:28:01 +01:00
|
|
|
return ep, fmt.Errorf("invalid URL endpoint format: %w", err)
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
host = u.Host
|
|
|
|
} else {
|
|
|
|
var p int
|
|
|
|
p, err = strconv.Atoi(port)
|
|
|
|
if err != nil {
|
2017-06-22 04:53:09 +02:00
|
|
|
return ep, fmt.Errorf("invalid URL endpoint format: invalid port number")
|
2017-04-12 00:44:27 +02:00
|
|
|
} else if p < 1 || p > 65535 {
|
2017-06-22 04:53:09 +02:00
|
|
|
return ep, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
|
|
|
}
|
2018-12-14 08:37:46 +01:00
|
|
|
if i := strings.Index(host, "%"); i > -1 {
|
|
|
|
host = host[:i]
|
|
|
|
}
|
2017-04-12 00:44:27 +02:00
|
|
|
|
|
|
|
if host == "" {
|
2017-06-22 04:53:09 +02:00
|
|
|
return ep, fmt.Errorf("invalid URL endpoint format: empty host name")
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// As this is path in the URL, we should use path package, not filepath package.
|
|
|
|
// On MS Windows, filepath.Clean() converts into Windows path style ie `/foo` becomes `\foo`
|
|
|
|
u.Path = path.Clean(u.Path)
|
|
|
|
if isEmptyPath(u.Path) {
|
2017-06-22 04:53:09 +02:00
|
|
|
return ep, fmt.Errorf("empty or root path is not supported in URL endpoint")
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
|
|
|
|
2019-08-06 21:08:58 +02:00
|
|
|
// On windows having a preceding SlashSeparator will cause problems, if the
|
2017-11-12 03:39:53 +01:00
|
|
|
// command line already has C:/<export-folder/ in it. Final resulting
|
|
|
|
// path on windows might become C:/C:/ this will cause problems
|
|
|
|
// of starting minio server properly in distributed mode on windows.
|
|
|
|
// As a special case make sure to trim the separator.
|
|
|
|
|
|
|
|
// NOTE: It is also perfectly fine for windows users to have a path
|
|
|
|
// without C:/ since at that point we treat it as relative path
|
|
|
|
// and obtain the full filesystem path as well. Providing C:/
|
|
|
|
// style is necessary to provide paths other than C:/,
|
|
|
|
// such as F:/, D:/ etc.
|
|
|
|
//
|
|
|
|
// Another additional benefit here is that this style also
|
|
|
|
// supports providing \\host\share support as well.
|
|
|
|
if runtime.GOOS == globalWindowsOSName {
|
|
|
|
if filepath.VolumeName(u.Path[1:]) != "" {
|
|
|
|
u.Path = u.Path[1:]
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-04-12 00:44:27 +02:00
|
|
|
} else {
|
2017-08-11 01:54:19 +02:00
|
|
|
// Only check if the arg is an ip address and ask for scheme since its absent.
|
|
|
|
// localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as
|
|
|
|
// /mnt/export1. So we go ahead and start the minio server in FS modes in these cases.
|
2018-12-14 08:37:46 +01:00
|
|
|
if isHostIP(arg) {
|
2017-08-11 01:54:19 +02:00
|
|
|
return ep, fmt.Errorf("invalid URL endpoint format: missing scheme http or https")
|
|
|
|
}
|
2017-04-12 00:44:27 +02:00
|
|
|
u = &url.URL{Path: path.Clean(arg)}
|
|
|
|
isLocal = true
|
|
|
|
}
|
|
|
|
|
|
|
|
return Endpoint{
|
2019-11-20 02:42:27 +01:00
|
|
|
URL: u,
|
|
|
|
IsLocal: isLocal,
|
2017-04-12 00:44:27 +02:00
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
2019-11-20 02:42:27 +01:00
|
|
|
// ZoneEndpoints represent endpoints in a given zone
|
|
|
|
// along with its setCount and drivesPerSet.
|
|
|
|
type ZoneEndpoints struct {
|
|
|
|
SetCount int
|
|
|
|
DrivesPerSet int
|
|
|
|
Endpoints Endpoints
|
|
|
|
}
|
2017-04-12 00:44:27 +02:00
|
|
|
|
2019-11-20 02:42:27 +01:00
|
|
|
// EndpointZones - list of list of endpoints
|
|
|
|
type EndpointZones []ZoneEndpoints
|
|
|
|
|
2020-01-07 18:13:44 +01:00
|
|
|
// Add add zone endpoints
|
|
|
|
func (l *EndpointZones) Add(zeps ZoneEndpoints) error {
|
|
|
|
existSet := set.NewStringSet()
|
|
|
|
for _, zep := range *l {
|
|
|
|
for _, ep := range zep.Endpoints {
|
|
|
|
existSet.Add(ep.String())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Validate if there are duplicate endpoints across zones
|
|
|
|
for _, ep := range zeps.Endpoints {
|
|
|
|
if existSet.Contains(ep.String()) {
|
|
|
|
return fmt.Errorf("duplicate endpoints found")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
*l = append(*l, zeps)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2019-11-22 21:45:13 +01:00
|
|
|
// FirstLocal returns true if the first endpoint is local.
|
|
|
|
func (l EndpointZones) FirstLocal() bool {
|
2019-11-21 13:24:51 +01:00
|
|
|
return l[0].Endpoints[0].IsLocal
|
|
|
|
}
|
|
|
|
|
2019-11-20 02:42:27 +01:00
|
|
|
// HTTPS - returns true if secure for URLEndpointType.
|
|
|
|
func (l EndpointZones) HTTPS() bool {
|
|
|
|
return l[0].Endpoints.HTTPS()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Nodes - returns all nodes count
|
|
|
|
func (l EndpointZones) Nodes() (count int) {
|
|
|
|
for _, ep := range l {
|
|
|
|
count += len(ep.Endpoints)
|
2018-09-18 03:28:34 +02:00
|
|
|
}
|
2019-11-20 02:42:27 +01:00
|
|
|
return count
|
2018-09-18 03:28:34 +02:00
|
|
|
}
|
|
|
|
|
2019-11-20 02:42:27 +01:00
|
|
|
// Endpoints - list of same type of endpoint.
|
|
|
|
type Endpoints []Endpoint
|
|
|
|
|
|
|
|
// HTTPS - returns true if secure for URLEndpointType.
|
|
|
|
func (endpoints Endpoints) HTTPS() bool {
|
|
|
|
return endpoints[0].HTTPS()
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
|
|
|
|
2018-01-22 23:54:55 +01:00
|
|
|
// GetString - returns endpoint string of i-th endpoint (0-based),
|
|
|
|
// and empty string for invalid indexes.
|
2019-11-20 02:42:27 +01:00
|
|
|
func (endpoints Endpoints) GetString(i int) string {
|
2018-01-22 23:54:55 +01:00
|
|
|
if i < 0 || i >= len(endpoints) {
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
return endpoints[i].String()
|
|
|
|
}
|
|
|
|
|
2019-12-24 20:31:32 +01:00
|
|
|
func hostResolveToLocalhost(endpoint Endpoint) bool {
|
|
|
|
hostIPs, err := getHostIP(endpoint.Hostname())
|
|
|
|
if err != nil {
|
|
|
|
// Log the message to console about the host resolving
|
|
|
|
reqInfo := (&logger.ReqInfo{}).AppendTags(
|
|
|
|
"host",
|
|
|
|
endpoint.Hostname(),
|
|
|
|
)
|
|
|
|
ctx := logger.SetReqInfo(context.Background(), reqInfo)
|
|
|
|
logger.LogIf(ctx, err, logger.Application)
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
var loopback int
|
|
|
|
for _, hostIP := range hostIPs.ToSlice() {
|
|
|
|
if net.ParseIP(hostIP).IsLoopback() {
|
|
|
|
loopback++
|
2019-12-19 02:05:24 +01:00
|
|
|
}
|
2019-12-11 05:28:22 +01:00
|
|
|
}
|
2019-12-24 20:31:32 +01:00
|
|
|
return loopback == len(hostIPs)
|
2019-12-11 05:28:22 +01:00
|
|
|
}
|
|
|
|
|
2019-12-19 22:45:56 +01:00
|
|
|
func (endpoints Endpoints) atleastOneEndpointLocal() bool {
|
|
|
|
for _, endpoint := range endpoints {
|
|
|
|
if endpoint.IsLocal {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2019-04-19 19:26:44 +02:00
|
|
|
// UpdateIsLocal - resolves the host and discovers the local host.
|
2019-12-19 22:45:56 +01:00
|
|
|
func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
|
2019-12-11 05:28:22 +01:00
|
|
|
orchestrated := IsDocker() || IsKubernetes()
|
2019-12-19 22:45:56 +01:00
|
|
|
k8sReplicaSet := IsKubernetesReplicaSet()
|
2019-12-11 05:28:22 +01:00
|
|
|
|
2019-04-19 19:26:44 +02:00
|
|
|
var epsResolved int
|
|
|
|
var foundLocal bool
|
|
|
|
resolvedList := make([]bool, len(endpoints))
|
|
|
|
// Mark the starting time
|
|
|
|
startTime := time.Now()
|
|
|
|
keepAliveTicker := time.NewTicker(retryInterval * time.Second)
|
|
|
|
defer keepAliveTicker.Stop()
|
|
|
|
for {
|
2019-10-10 06:44:17 +02:00
|
|
|
// Break if the local endpoint is found already Or all the endpoints are resolved.
|
2019-04-19 19:26:44 +02:00
|
|
|
if foundLocal || (epsResolved == len(endpoints)) {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
// Retry infinitely on Kubernetes and Docker swarm.
|
|
|
|
// This is needed as the remote hosts are sometime
|
|
|
|
// not available immediately.
|
|
|
|
select {
|
|
|
|
case <-globalOSSignalCh:
|
|
|
|
return fmt.Errorf("The endpoint resolution got interrupted")
|
|
|
|
default:
|
|
|
|
for i, resolved := range resolvedList {
|
|
|
|
if resolved {
|
2019-10-10 06:44:17 +02:00
|
|
|
// Continue if host is already resolved.
|
2019-04-19 19:26:44 +02:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2019-12-11 05:28:22 +01:00
|
|
|
// Log the message to console about the host resolving
|
|
|
|
reqInfo := (&logger.ReqInfo{}).AppendTags(
|
|
|
|
"host",
|
|
|
|
endpoints[i].Hostname(),
|
|
|
|
)
|
|
|
|
|
2019-12-24 20:31:32 +01:00
|
|
|
if k8sReplicaSet && hostResolveToLocalhost(endpoints[i]) {
|
|
|
|
err := fmt.Errorf("host %s resolves to 127.*, DNS incorrectly configured retrying",
|
|
|
|
endpoints[i])
|
2019-12-11 05:28:22 +01:00
|
|
|
// time elapsed
|
|
|
|
timeElapsed := time.Since(startTime)
|
|
|
|
// log error only if more than 1s elapsed
|
|
|
|
if timeElapsed > time.Second {
|
|
|
|
reqInfo.AppendTags("elapsedTime",
|
|
|
|
humanize.RelTime(startTime,
|
|
|
|
startTime.Add(timeElapsed),
|
|
|
|
"elapsed",
|
|
|
|
""))
|
|
|
|
ctx := logger.SetReqInfo(context.Background(), reqInfo)
|
|
|
|
logger.LogIf(ctx, err, logger.Application)
|
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2019-04-19 19:26:44 +02:00
|
|
|
// return err if not Docker or Kubernetes
|
2019-10-10 06:44:17 +02:00
|
|
|
// We use IsDocker() to check for Docker environment
|
|
|
|
// We use IsKubernetes() to check for Kubernetes environment
|
2019-12-11 05:28:22 +01:00
|
|
|
isLocal, err := isLocalHost(endpoints[i].Hostname(),
|
|
|
|
endpoints[i].Port(),
|
|
|
|
globalMinioPort,
|
|
|
|
)
|
|
|
|
if err != nil && !orchestrated {
|
|
|
|
return err
|
|
|
|
}
|
2019-04-19 19:26:44 +02:00
|
|
|
if err != nil {
|
|
|
|
// time elapsed
|
|
|
|
timeElapsed := time.Since(startTime)
|
|
|
|
// log error only if more than 1s elapsed
|
|
|
|
if timeElapsed > time.Second {
|
2019-11-20 02:42:27 +01:00
|
|
|
reqInfo.AppendTags("elapsedTime",
|
2019-12-11 05:28:22 +01:00
|
|
|
humanize.RelTime(startTime,
|
|
|
|
startTime.Add(timeElapsed),
|
|
|
|
"elapsed",
|
|
|
|
"",
|
|
|
|
))
|
|
|
|
ctx := logger.SetReqInfo(context.Background(),
|
|
|
|
reqInfo)
|
2019-10-12 03:50:54 +02:00
|
|
|
logger.LogIf(ctx, err, logger.Application)
|
2019-04-19 19:26:44 +02:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
resolvedList[i] = true
|
|
|
|
endpoints[i].IsLocal = isLocal
|
2019-12-19 22:45:56 +01:00
|
|
|
if k8sReplicaSet && !endpoints.atleastOneEndpointLocal() && !foundPrevLocal {
|
|
|
|
// In replicated set in k8s deployment, IPs might
|
|
|
|
// get resolved for older IPs, add this code
|
|
|
|
// to ensure that we wait for this server to
|
|
|
|
// participate atleast one disk and be local.
|
|
|
|
//
|
|
|
|
// In special cases for replica set with expanded
|
|
|
|
// zone setups we need to make sure to provide
|
|
|
|
// value of foundPrevLocal from zone1 if we already
|
|
|
|
// found a local setup. Only if we haven't found
|
|
|
|
// previous local we continue to wait to look for
|
|
|
|
// atleast one local.
|
|
|
|
resolvedList[i] = false
|
2019-12-24 20:31:32 +01:00
|
|
|
// time elapsed
|
|
|
|
err := fmt.Errorf("no endpoint is local to this host: %s", endpoints[i])
|
|
|
|
timeElapsed := time.Since(startTime)
|
|
|
|
// log error only if more than 1s elapsed
|
|
|
|
if timeElapsed > time.Second {
|
|
|
|
reqInfo.AppendTags("elapsedTime",
|
|
|
|
humanize.RelTime(startTime,
|
|
|
|
startTime.Add(timeElapsed),
|
|
|
|
"elapsed",
|
|
|
|
"",
|
|
|
|
))
|
|
|
|
ctx := logger.SetReqInfo(context.Background(),
|
|
|
|
reqInfo)
|
|
|
|
logger.LogIf(ctx, err, logger.Application)
|
|
|
|
}
|
2019-12-19 22:45:56 +01:00
|
|
|
continue
|
|
|
|
}
|
2019-04-19 19:26:44 +02:00
|
|
|
epsResolved++
|
|
|
|
if !foundLocal {
|
|
|
|
foundLocal = isLocal
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Wait for the tick, if the there exist a local endpoint in discovery.
|
2019-10-10 06:44:17 +02:00
|
|
|
// Non docker/kubernetes environment we do not need to wait.
|
2019-12-11 05:28:22 +01:00
|
|
|
if !foundLocal && orchestrated {
|
2019-04-19 19:26:44 +02:00
|
|
|
<-keepAliveTicker.C
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-10-10 06:44:17 +02:00
|
|
|
// On Kubernetes/Docker setups DNS resolves inappropriately sometimes
|
|
|
|
// where there are situations same endpoints with multiple disks
|
|
|
|
// come online indicating either one of them is local and some
|
|
|
|
// of them are not local. This situation can never happen and
|
|
|
|
// its only a possibility in orchestrated deployments with dynamic
|
|
|
|
// DNS. Following code ensures that we treat if one of the endpoint
|
|
|
|
// says its local for a given host - it is true for all endpoints
|
|
|
|
// for the same host. Following code ensures that this assumption
|
|
|
|
// is true and it works in all scenarios and it is safe to assume
|
|
|
|
// for a given host.
|
|
|
|
endpointLocalMap := make(map[string]bool)
|
|
|
|
for _, ep := range endpoints {
|
|
|
|
if ep.IsLocal {
|
|
|
|
endpointLocalMap[ep.Host] = ep.IsLocal
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for i := range endpoints {
|
|
|
|
endpoints[i].IsLocal = endpointLocalMap[endpoints[i].Host]
|
|
|
|
}
|
2019-04-19 19:26:44 +02:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2019-11-20 02:42:27 +01:00
|
|
|
// NewEndpoints - returns new endpoint list based on input args.
|
|
|
|
func NewEndpoints(args ...string) (endpoints Endpoints, err error) {
|
2017-04-12 00:44:27 +02:00
|
|
|
var endpointType EndpointType
|
|
|
|
var scheme string
|
|
|
|
|
|
|
|
uniqueArgs := set.NewStringSet()
|
|
|
|
// Loop through args and adds to endpoint list.
|
|
|
|
for i, arg := range args {
|
|
|
|
endpoint, err := NewEndpoint(arg)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("'%s': %s", arg, err.Error())
|
|
|
|
}
|
|
|
|
|
|
|
|
// All endpoints have to be same type and scheme if applicable.
|
|
|
|
if i == 0 {
|
|
|
|
endpointType = endpoint.Type()
|
|
|
|
scheme = endpoint.Scheme
|
|
|
|
} else if endpoint.Type() != endpointType {
|
|
|
|
return nil, fmt.Errorf("mixed style endpoints are not supported")
|
|
|
|
} else if endpoint.Scheme != scheme {
|
|
|
|
return nil, fmt.Errorf("mixed scheme is not supported")
|
|
|
|
}
|
|
|
|
|
|
|
|
arg = endpoint.String()
|
|
|
|
if uniqueArgs.Contains(arg) {
|
|
|
|
return nil, fmt.Errorf("duplicate endpoints found")
|
|
|
|
}
|
|
|
|
uniqueArgs.Add(arg)
|
|
|
|
endpoints = append(endpoints, endpoint)
|
|
|
|
}
|
2019-04-19 19:26:44 +02:00
|
|
|
|
2017-04-12 00:44:27 +02:00
|
|
|
return endpoints, nil
|
|
|
|
}
|
|
|
|
|
2017-08-16 00:10:50 +02:00
|
|
|
// Checks if there are any cross device mounts.
|
2019-11-20 02:42:27 +01:00
|
|
|
func checkCrossDeviceMounts(endpoints Endpoints) (err error) {
|
2017-08-16 00:10:50 +02:00
|
|
|
var absPaths []string
|
|
|
|
for _, endpoint := range endpoints {
|
|
|
|
if endpoint.IsLocal {
|
|
|
|
var absPath string
|
|
|
|
absPath, err = filepath.Abs(endpoint.Path)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
absPaths = append(absPaths, absPath)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return mountinfo.CheckCrossDevice(absPaths)
|
|
|
|
}
|
|
|
|
|
2017-04-12 00:44:27 +02:00
|
|
|
// CreateEndpoints - validates and creates new endpoints for given args.
|
2019-12-19 22:45:56 +01:00
|
|
|
func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endpoints, SetupType, error) {
|
2019-11-20 02:42:27 +01:00
|
|
|
var endpoints Endpoints
|
2017-04-12 00:44:27 +02:00
|
|
|
var setupType SetupType
|
|
|
|
var err error
|
|
|
|
|
|
|
|
// Check whether serverAddr is valid for this host.
|
|
|
|
if err = CheckLocalServerAddr(serverAddr); err != nil {
|
2019-11-20 02:42:27 +01:00
|
|
|
return endpoints, setupType, err
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
_, serverAddrPort := mustSplitHostPort(serverAddr)
|
|
|
|
|
|
|
|
// For single arg, return FS setup.
|
2018-02-16 02:45:57 +01:00
|
|
|
if len(args) == 1 && len(args[0]) == 1 {
|
2017-04-12 00:44:27 +02:00
|
|
|
var endpoint Endpoint
|
2018-02-16 02:45:57 +01:00
|
|
|
endpoint, err = NewEndpoint(args[0][0])
|
2017-04-12 00:44:27 +02:00
|
|
|
if err != nil {
|
2019-11-20 02:42:27 +01:00
|
|
|
return endpoints, setupType, err
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
2019-04-19 19:26:44 +02:00
|
|
|
if err := endpoint.UpdateIsLocal(); err != nil {
|
2019-11-20 02:42:27 +01:00
|
|
|
return endpoints, setupType, err
|
2019-04-19 19:26:44 +02:00
|
|
|
}
|
2017-04-12 00:44:27 +02:00
|
|
|
if endpoint.Type() != PathEndpointType {
|
2019-11-20 02:42:27 +01:00
|
|
|
return endpoints, setupType, config.ErrInvalidFSEndpoint(nil).Msg("use path style endpoint for FS setup")
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
|
|
|
endpoints = append(endpoints, endpoint)
|
|
|
|
setupType = FSSetupType
|
2017-08-16 00:10:50 +02:00
|
|
|
|
|
|
|
// Check for cross device mounts if any.
|
|
|
|
if err = checkCrossDeviceMounts(endpoints); err != nil {
|
2019-11-20 02:42:27 +01:00
|
|
|
return endpoints, setupType, config.ErrInvalidFSEndpoint(nil).Msg(err.Error())
|
2017-08-16 00:10:50 +02:00
|
|
|
}
|
2019-11-20 02:42:27 +01:00
|
|
|
|
|
|
|
return endpoints, setupType, nil
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
|
|
|
|
2020-01-10 11:35:06 +01:00
|
|
|
for _, iargs := range args {
|
2018-02-16 02:45:57 +01:00
|
|
|
// Convert args to endpoints
|
2019-11-20 02:42:27 +01:00
|
|
|
eps, err := NewEndpoints(iargs...)
|
2018-02-16 02:45:57 +01:00
|
|
|
if err != nil {
|
2019-11-20 02:42:27 +01:00
|
|
|
return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
|
2018-02-16 02:45:57 +01:00
|
|
|
}
|
2017-04-12 00:44:27 +02:00
|
|
|
|
2018-02-16 02:45:57 +01:00
|
|
|
// Check for cross device mounts if any.
|
|
|
|
if err = checkCrossDeviceMounts(eps); err != nil {
|
2019-11-20 02:42:27 +01:00
|
|
|
return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
|
2018-02-16 02:45:57 +01:00
|
|
|
}
|
|
|
|
|
2020-01-10 11:35:06 +01:00
|
|
|
endpoints = append(endpoints, eps...)
|
2017-08-16 00:10:50 +02:00
|
|
|
}
|
|
|
|
|
2019-11-26 20:42:10 +01:00
|
|
|
if len(endpoints) == 0 {
|
|
|
|
return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg("invalid number of endpoints")
|
|
|
|
}
|
|
|
|
|
2017-04-12 00:44:27 +02:00
|
|
|
// Return XL setup when all endpoints are path style.
|
|
|
|
if endpoints[0].Type() == PathEndpointType {
|
|
|
|
setupType = XLSetupType
|
2019-11-20 02:42:27 +01:00
|
|
|
return endpoints, setupType, nil
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
|
|
|
|
2019-12-19 22:45:56 +01:00
|
|
|
if err = endpoints.UpdateIsLocal(foundLocal); err != nil {
|
2019-11-20 02:42:27 +01:00
|
|
|
return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
|
2019-04-19 19:26:44 +02:00
|
|
|
}
|
|
|
|
|
2017-04-12 00:44:27 +02:00
|
|
|
// Here all endpoints are URL style.
|
|
|
|
endpointPathSet := set.NewStringSet()
|
|
|
|
localEndpointCount := 0
|
2019-11-20 02:42:27 +01:00
|
|
|
localServerHostSet := set.NewStringSet()
|
2017-04-12 00:44:27 +02:00
|
|
|
localPortSet := set.NewStringSet()
|
2017-08-11 01:54:19 +02:00
|
|
|
|
2017-04-12 00:44:27 +02:00
|
|
|
for _, endpoint := range endpoints {
|
|
|
|
endpointPathSet.Add(endpoint.Path)
|
|
|
|
if endpoint.IsLocal {
|
2019-11-20 02:42:27 +01:00
|
|
|
localServerHostSet.Add(endpoint.Hostname())
|
2017-04-12 00:44:27 +02:00
|
|
|
|
|
|
|
var port string
|
|
|
|
_, port, err = net.SplitHostPort(endpoint.Host)
|
|
|
|
if err != nil {
|
|
|
|
port = serverAddrPort
|
|
|
|
}
|
|
|
|
localPortSet.Add(port)
|
|
|
|
|
|
|
|
localEndpointCount++
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-02-15 10:08:17 +01:00
|
|
|
// Check whether same path is not used in endpoints of a host on different port.
|
2017-04-12 00:44:27 +02:00
|
|
|
{
|
|
|
|
pathIPMap := make(map[string]set.StringSet)
|
|
|
|
for _, endpoint := range endpoints {
|
2019-11-20 02:42:27 +01:00
|
|
|
host := endpoint.Hostname()
|
2018-12-14 08:37:46 +01:00
|
|
|
hostIPSet, _ := getHostIP(host)
|
2017-04-12 00:44:27 +02:00
|
|
|
if IPSet, ok := pathIPMap[endpoint.Path]; ok {
|
|
|
|
if !IPSet.Intersection(hostIPSet).IsEmpty() {
|
2019-11-20 02:42:27 +01:00
|
|
|
return endpoints, setupType,
|
2019-10-04 19:35:33 +02:00
|
|
|
config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' can not be served by different port on same address", endpoint.Path))
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
2017-04-13 05:27:24 +02:00
|
|
|
pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
|
2017-04-12 00:44:27 +02:00
|
|
|
} else {
|
|
|
|
pathIPMap[endpoint.Path] = hostIPSet
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-02-15 10:08:17 +01:00
|
|
|
// Check whether same path is used for more than 1 local endpoints.
|
|
|
|
{
|
|
|
|
localPathSet := set.CreateStringSet()
|
|
|
|
for _, endpoint := range endpoints {
|
|
|
|
if !endpoint.IsLocal {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if localPathSet.Contains(endpoint.Path) {
|
2019-11-20 02:42:27 +01:00
|
|
|
return endpoints, setupType,
|
2019-10-04 19:35:33 +02:00
|
|
|
config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' cannot be served by different address on same server", endpoint.Path))
|
2018-02-15 10:08:17 +01:00
|
|
|
}
|
|
|
|
localPathSet.Add(endpoint.Path)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-04-13 05:27:24 +02:00
|
|
|
// All endpoints are pointing to local host
|
|
|
|
if len(endpoints) == localEndpointCount {
|
2020-01-07 18:13:44 +01:00
|
|
|
// If all endpoints have same port number, Just treat it as distXL setup
|
|
|
|
// using URL style endpoints.
|
2017-04-13 05:27:24 +02:00
|
|
|
if len(localPortSet) == 1 {
|
2019-11-20 02:42:27 +01:00
|
|
|
if len(localServerHostSet) > 1 {
|
|
|
|
return endpoints, setupType,
|
|
|
|
config.ErrInvalidErasureEndpoints(nil).Msg("all local endpoints should not have different hostnames/ips")
|
2017-04-13 05:27:24 +02:00
|
|
|
}
|
2020-01-07 18:13:44 +01:00
|
|
|
return endpoints, DistXLSetupType, nil
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
|
|
|
|
2018-05-09 04:04:36 +02:00
|
|
|
// Even though all endpoints are local, but those endpoints use different ports.
|
2017-04-13 05:27:24 +02:00
|
|
|
// This means it is DistXL setup.
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Add missing port in all endpoints.
|
|
|
|
for i := range endpoints {
|
|
|
|
_, port, err := net.SplitHostPort(endpoints[i].Host)
|
|
|
|
if err != nil {
|
|
|
|
endpoints[i].Host = net.JoinHostPort(endpoints[i].Host, serverAddrPort)
|
|
|
|
} else if endpoints[i].IsLocal && serverAddrPort != port {
|
|
|
|
// If endpoint is local, but port is different than serverAddrPort, then make it as remote.
|
|
|
|
endpoints[i].IsLocal = false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-02-16 02:45:57 +01:00
|
|
|
uniqueArgs := set.NewStringSet()
|
|
|
|
for _, endpoint := range endpoints {
|
|
|
|
uniqueArgs.Add(endpoint.Host)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Error out if we have less than 2 unique servers.
|
|
|
|
if len(uniqueArgs.ToSlice()) < 2 && setupType == DistXLSetupType {
|
|
|
|
err := fmt.Errorf("Unsupported number of endpoints (%s), minimum number of servers cannot be less than 2 in distributed setup", endpoints)
|
2019-11-20 02:42:27 +01:00
|
|
|
return endpoints, setupType, err
|
2018-02-16 02:45:57 +01:00
|
|
|
}
|
|
|
|
|
2019-10-31 07:39:09 +01:00
|
|
|
publicIPs := env.Get(config.EnvPublicIPs, "")
|
|
|
|
if len(publicIPs) == 0 {
|
2018-12-05 02:35:22 +01:00
|
|
|
updateDomainIPs(uniqueArgs)
|
|
|
|
}
|
2018-05-16 03:20:22 +02:00
|
|
|
|
2017-04-12 00:44:27 +02:00
|
|
|
setupType = DistXLSetupType
|
2019-11-20 02:42:27 +01:00
|
|
|
return endpoints, setupType, nil
|
2017-04-12 00:44:27 +02:00
|
|
|
}
|
|
|
|
|
2017-10-09 05:23:42 +02:00
|
|
|
// GetLocalPeer - returns local peer value, returns globalMinioAddr
|
|
|
|
// for FS and Erasure mode. In case of distributed server return
|
|
|
|
// the first element from the set of peers which indicate that
|
|
|
|
// they are local. There is always one entry that is local
|
|
|
|
// even with repeated server endpoints.
|
2019-11-20 02:42:27 +01:00
|
|
|
func GetLocalPeer(endpointZones EndpointZones) (localPeer string) {
|
2017-10-09 05:23:42 +02:00
|
|
|
peerSet := set.NewStringSet()
|
2019-11-20 02:42:27 +01:00
|
|
|
for _, ep := range endpointZones {
|
|
|
|
for _, endpoint := range ep.Endpoints {
|
|
|
|
if endpoint.Type() != URLEndpointType {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if endpoint.IsLocal && endpoint.Host != "" {
|
|
|
|
peerSet.Add(endpoint.Host)
|
|
|
|
}
|
2017-10-09 05:23:42 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if peerSet.IsEmpty() {
|
2018-03-15 21:03:41 +01:00
|
|
|
// Local peer can be empty in FS or Erasure coded mode.
|
|
|
|
// If so, return globalMinioHost + globalMinioPort value.
|
|
|
|
if globalMinioHost != "" {
|
2018-12-14 08:37:46 +01:00
|
|
|
return net.JoinHostPort(globalMinioHost, globalMinioPort)
|
2018-03-15 21:03:41 +01:00
|
|
|
}
|
|
|
|
|
2018-12-14 08:37:46 +01:00
|
|
|
return net.JoinHostPort("127.0.0.1", globalMinioPort)
|
2017-10-09 05:23:42 +02:00
|
|
|
}
|
|
|
|
return peerSet.ToSlice()[0]
|
|
|
|
}
|
|
|
|
|
2017-04-12 00:44:27 +02:00
|
|
|
// GetRemotePeers - get hosts information other than this minio service.
|
2019-11-20 02:42:27 +01:00
|
|
|
func GetRemotePeers(endpointZones EndpointZones) []string {
|
2017-04-12 00:44:27 +02:00
|
|
|
peerSet := set.NewStringSet()
|
2019-11-20 02:42:27 +01:00
|
|
|
for _, ep := range endpointZones {
|
|
|
|
for _, endpoint := range ep.Endpoints {
|
|
|
|
if endpoint.Type() != URLEndpointType {
|
2017-04-12 00:44:27 +02:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2019-11-20 02:42:27 +01:00
|
|
|
peer := endpoint.Host
|
|
|
|
if endpoint.IsLocal {
|
|
|
|
if _, port := mustSplitHostPort(peer); port == globalMinioPort {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
2017-04-12 00:44:27 +02:00
|
|
|
|
2019-11-20 02:42:27 +01:00
|
|
|
peerSet.Add(peer)
|
|
|
|
}
|
|
|
|
}
|
2017-04-12 00:44:27 +02:00
|
|
|
return peerSet.ToSlice()
|
|
|
|
}
|
2018-05-16 03:20:22 +02:00
|
|
|
|
|
|
|
func updateDomainIPs(endPoints set.StringSet) {
|
2018-12-05 02:35:22 +01:00
|
|
|
ipList := set.NewStringSet()
|
|
|
|
for e := range endPoints {
|
2019-08-02 21:40:51 +02:00
|
|
|
host, port, err := net.SplitHostPort(e)
|
2018-12-05 02:35:22 +01:00
|
|
|
if err != nil {
|
|
|
|
if strings.Contains(err.Error(), "missing port in address") {
|
|
|
|
host = e
|
2019-08-02 21:40:51 +02:00
|
|
|
port = globalMinioPort
|
2018-12-05 02:35:22 +01:00
|
|
|
} else {
|
|
|
|
continue
|
|
|
|
}
|
2018-05-16 03:20:22 +02:00
|
|
|
}
|
2019-08-02 21:40:51 +02:00
|
|
|
|
2019-12-18 06:32:45 +01:00
|
|
|
if net.ParseIP(host) == nil {
|
|
|
|
IPs, err := getHostIP(host)
|
|
|
|
if err != nil {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
IPsWithPort := IPs.ApplyFunc(func(ip string) string {
|
|
|
|
return net.JoinHostPort(ip, port)
|
|
|
|
})
|
2019-08-02 21:40:51 +02:00
|
|
|
|
2019-12-18 06:32:45 +01:00
|
|
|
ipList = ipList.Union(IPsWithPort)
|
|
|
|
}
|
|
|
|
|
|
|
|
ipList.Add(net.JoinHostPort(host, port))
|
2018-05-16 03:20:22 +02:00
|
|
|
}
|
2019-12-18 06:32:45 +01:00
|
|
|
|
2018-12-05 02:35:22 +01:00
|
|
|
globalDomainIPs = ipList.FuncMatch(func(ip string, matchString string) bool {
|
2019-11-20 02:42:27 +01:00
|
|
|
host, _, err := net.SplitHostPort(ip)
|
|
|
|
if err != nil {
|
|
|
|
host = ip
|
|
|
|
}
|
2019-12-19 22:45:56 +01:00
|
|
|
return !net.ParseIP(host).IsLoopback() && host != "localhost"
|
2018-12-05 02:35:22 +01:00
|
|
|
}, "")
|
2018-05-16 03:20:22 +02:00
|
|
|
}
|