maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add etcd part of config support, add noColor/json support (#8439)

Browse source 
- Add color/json mode support for get/help commands
- Support ENV help for all sub-systems
- Add support for etcd as part of config
This commit is contained in:
Harshavardhana 2019-10-30 00:04:39 -07:00 committed by kannappanr
parent 51456e6adc
commit 47b13cdb80
37 changed files with 704 additions and 348 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

67
cmd/admin-handlers-config-kv.go
View file

@ -22,12 +22,10 @@ import (
"encoding/json"
"io"
"net/http"
"strings"
"github.com/gorilla/mux"
"github.com/minio/minio/cmd/config"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/color"
"github.com/minio/minio/pkg/madmin"
)
@ -68,6 +66,9 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
oldCfg := cfg.Clone()
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
for scanner.Scan() {
if scanner.Text() == "" {
continue
}
if err = cfg.DelKVS(scanner.Text()); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
@ -121,20 +122,14 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
defaultKVS := configDefaultKVS()
oldCfg := cfg.Clone()
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
var comment string
for scanner.Scan() {
if strings.HasPrefix(scanner.Text(), config.KvComment) {
// Join multiple comments for each newline, separated by ","
comments := []string{comment, strings.TrimPrefix(scanner.Text(), config.KvComment)}
comment = strings.Join(comments, config.KvNewline)
if scanner.Text() == "" {
continue
}
if err = cfg.SetKVS(scanner.Text(), comment, defaultKVS); err != nil {
if err = cfg.SetKVS(scanner.Text(), defaultKVS); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
// Empty the comment for the next sub-system
comment = ""
}
if err = scanner.Err(); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@ -169,39 +164,28 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
}
vars := mux.Vars(r)
var body strings.Builder
if vars["key"] != "" {
kvs, err := globalServerConfig.GetKVS(vars["key"])
var buf = &bytes.Buffer{}
key := vars["key"]
if key != "" {
kvs, err := globalServerConfig.GetKVS(key)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
for k, kv := range kvs {
c, ok := kv[config.Comment]
if ok {
// For multiple comments split it correctly.
for _, c1 := range strings.Split(c, config.KvNewline) {
if c1 == "" {
continue
}
body.WriteString(color.YellowBold(config.KvComment))
body.WriteString(config.KvSpaceSeparator)
body.WriteString(color.BlueBold(strings.TrimSpace(c1)))
body.WriteString(config.KvNewline)
}
}
body.WriteString(color.CyanBold(k))
body.WriteString(config.KvSpaceSeparator)
body.WriteString(kv.String())
buf.WriteString(k)
buf.WriteString(config.KvSpaceSeparator)
buf.WriteString(kv.String())
if len(kvs) > 1 {
body.WriteString(config.KvNewline)
buf.WriteString(config.KvNewline)
}
}
} else {
body.WriteString(globalServerConfig.String())
buf.WriteString(globalServerConfig.String())
}
password := globalActiveCred.SecretKey
econfigData, err := madmin.EncryptData(password, []byte(body.String()))
econfigData, err := madmin.EncryptData(password, buf.Bytes())
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
@ -275,18 +259,14 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
defaultKVS := configDefaultKVS()
oldCfg := cfg.Clone()
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
var comment string
for scanner.Scan() {
if strings.HasPrefix(scanner.Text(), config.KvComment) {
// Join multiple comments for each newline, separated by "\n"
comment = strings.Join([]string{comment, scanner.Text()}, config.KvNewline)
if scanner.Text() == "" {
continue
}
if err = cfg.SetKVS(scanner.Text(), comment, defaultKVS); err != nil {
if err = cfg.SetKVS(scanner.Text(), defaultKVS); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
comment = ""
}
if err = scanner.Err(); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@ -340,20 +320,23 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
}
vars := mux.Vars(r)
subSys := vars["subSys"]
key := vars["key"]
rd, err := GetHelp(subSys, key)
_, envOnly := r.URL.Query()["env"]
rd, err := GetHelp(subSys, key, envOnly)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
io.Copy(w, rd)
json.NewEncoder(w).Encode(rd)
w.(http.Flusher).Flush()
}
// SetConfigHandler - PUT /minio/admin/v1/config
// SetConfigHandler - PUT /minio/admin/v2/config
func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "SetConfigHandler")
@ -403,7 +386,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
writeSuccessResponseHeadersOnly(w)
}
// GetConfigHandler - GET /minio/admin/v1/config
// GetConfigHandler - GET /minio/admin/v2/config
// Get config.json of this minio setup.
func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "GetConfigHandler")

31
cmd/admin-handlers.go
View file

@ -982,22 +982,23 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
if err == nil {
return noError
}
apiErr := errorCodes.ToAPIErr(toAdminAPIErrCode(ctx, err))
if apiErr.Code == "InternalError" {
switch e := err.(type) {
case config.Error:
apiErr = APIError{
Code: "XMinioConfigError",
Description: e.Error(),
HTTPStatusCode: http.StatusBadRequest,
}
case AdminError:
apiErr = APIError{
Code: e.Code,
Description: e.Message,
HTTPStatusCode: e.StatusCode,
}
var apiErr APIError
switch e := err.(type) {
case config.Error:
apiErr = APIError{
Code: "XMinioConfigError",
Description: e.Error(),
HTTPStatusCode: http.StatusBadRequest,
}
case AdminError:
apiErr = APIError{
Code: e.Code,
Description: e.Message,
HTTPStatusCode: e.StatusCode,
}
default:
apiErr = errorCodes.ToAPIErr(toAdminAPIErrCode(ctx, err))
}
return apiErr
}

2
cmd/bucket-handlers.go
View file

@ -106,7 +106,7 @@ func initFederatorBackend(objLayer ObjectLayer) {
}
// This is not for our server, so we can continue
hostPort := net.JoinHostPort(dnsBuckets[index].Host, fmt.Sprintf("%d", dnsBuckets[index].Port))
hostPort := net.JoinHostPort(dnsBuckets[index].Host, dnsBuckets[index].Port)
if globalDomainIPs.Intersection(set.CreateStringSet(hostPort)).IsEmpty() {
return nil
}

19
cmd/common-main.go
View file

@ -28,10 +28,8 @@ import (
"github.com/minio/cli"
"github.com/minio/minio-go/v6/pkg/set"
"github.com/minio/minio/cmd/config"
"github.com/minio/minio/cmd/config/etcd"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/certs"
"github.com/minio/minio/pkg/dns"
"github.com/minio/minio/pkg/env"
)
@ -159,17 +157,12 @@ func handleCommonCmdArgs(ctx *cli.Context) {
func handleCommonEnvVars() {
var err error
globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, "on"))
globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, config.StateOn))
if err != nil {
logger.Fatal(config.ErrInvalidBrowserValue(err), "Invalid MINIO_BROWSER value in environment variable")
}
globalEtcdClient, err = etcd.New(globalRootCAs)
if err != nil {
logger.FatalIf(err, "Unable to initialize etcd config")
}
for _, domainName := range strings.Split(env.Get(config.EnvDomain, ""), ",") {
for _, domainName := range strings.Split(env.Get(config.EnvDomain, ""), config.ValueSeparator) {
if domainName != "" {
if _, ok := dns2.IsDomainName(domainName); !ok {
logger.Fatal(config.ErrInvalidDomainValue(nil).Msg("Unknown value `%s`", domainName),
@ -181,7 +174,7 @@ func handleCommonEnvVars() {
minioEndpointsEnv, ok := env.Lookup(config.EnvPublicIPs)
if ok {
minioEndpoints := strings.Split(minioEndpointsEnv, ",")
minioEndpoints := strings.Split(minioEndpointsEnv, config.ValueSeparator)
var domainIPs = set.NewStringSet()
for _, endpoint := range minioEndpoints {
if net.ParseIP(endpoint) == nil {
@ -204,12 +197,6 @@ func handleCommonEnvVars() {
updateDomainIPs(localIP4)
}
if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
var err error
globalDNSConfig, err = dns.NewCoreDNS(globalDomainNames, globalDomainIPs, globalMinioPort, globalEtcdClient)
logger.FatalIf(err, "Unable to initialize DNS config for %s.", globalDomainNames)
}
// In place update is true by default if the MINIO_UPDATE is not set
// or is not set to 'off', if MINIO_UPDATE is set to 'off' then
// in-place update is off.

85
cmd/config-current.go
View file

@ -19,14 +19,13 @@ package cmd
import (
"context"
"fmt"
"io"
"strings"
"sync"
"text/tabwriter"
"github.com/minio/minio/cmd/config"
"github.com/minio/minio/cmd/config/cache"
"github.com/minio/minio/cmd/config/compress"
"github.com/minio/minio/cmd/config/etcd"
xldap "github.com/minio/minio/cmd/config/identity/ldap"
"github.com/minio/minio/cmd/config/identity/openid"
"github.com/minio/minio/cmd/config/notify"
@ -36,6 +35,7 @@ import (
xhttp "github.com/minio/minio/cmd/http"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/cmd/logger/target/http"
"github.com/minio/minio/pkg/dns"
"github.com/minio/minio/pkg/env"
)
@ -61,6 +61,9 @@ func validateConfig(s config.Config) error {
return err
}
}
if _, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs); err != nil {
return err
}
if _, err := cache.LookupConfig(s[config.CacheSubSys][config.Default]); err != nil {
return err
}
@ -92,14 +95,35 @@ func lookupConfigs(s config.Config) {
var err error
if !globalActiveCred.IsValid() {
// Env doesn't seem to be set, we fallback to lookup
// creds from the config.
// Env doesn't seem to be set, we fallback to lookup creds from the config.
globalActiveCred, err = config.LookupCreds(s[config.CredentialsSubSys][config.Default])
if err != nil {
logger.Fatal(err, "Invalid credentials configuration")
}
}
etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
if err != nil {
logger.Fatal(err, "Unable to initialize etcd config")
}
globalEtcdClient, err = etcd.New(etcdCfg)
if err != nil {
logger.Fatal(err, "Unable to initialize etcd config")
}
if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
globalDNSConfig, err = dns.NewCoreDNS(globalEtcdClient,
dns.DomainNames(globalDomainNames),
dns.DomainIPs(globalDomainIPs),
dns.DomainPort(globalMinioPort),
dns.CoreDNSPath(etcdCfg.CoreDNSPath),
)
if err != nil {
logger.Fatal(err, "Unable to initialize DNS config for %s.", globalDomainNames)
}
}
globalServerRegion, err = config.LookupRegion(s[config.RegionSubSys][config.Default])
if err != nil {
logger.Fatal(err, "Invalid region configuration")
@ -202,6 +226,7 @@ func lookupConfigs(s config.Config) {
var helpMap = map[string]config.HelpKV{
config.RegionSubSys: config.RegionHelp,
config.WormSubSys: config.WormHelp,
config.EtcdSubSys: etcd.Help,
config.CacheSubSys: cache.Help,
config.CompressionSubSys: compress.Help,
config.StorageClassSubSys: storageclass.Help,
@ -224,29 +249,42 @@ var helpMap = map[string]config.HelpKV{
}
// GetHelp - returns help for sub-sys, a key for a sub-system or all the help.
func GetHelp(subSys, key string) (io.Reader, error) {
func GetHelp(subSys, key string, envOnly bool) (config.HelpKV, error) {
if len(subSys) == 0 {
return nil, config.Error("no help available for empty sub-system inputs")
}
help, ok := helpMap[subSys]
if !ok {
subSystemValue := strings.SplitN(subSys, config.SubSystemSeparator, 2)
if len(subSystemValue) == 0 {
return nil, config.Error(fmt.Sprintf("invalid number of arguments %s", subSys))
}
if !config.SubSystems.Contains(subSystemValue[0]) {
return nil, config.Error(fmt.Sprintf("unknown sub-system %s", subSys))
}
help := helpMap[subSystemValue[0]]
if key != "" {
value, ok := help[key]
if !ok {
return nil, config.Error(fmt.Sprintf("unknown key %s for sub-system %s", key, subSys))
}
return strings.NewReader(value), nil
help = config.HelpKV{
key: value,
}
}
var s strings.Builder
w := tabwriter.NewWriter(&s, 1, 8, 2, ' ', 0)
if err := config.HelpTemplate.Execute(w, help); err != nil {
return nil, config.Error(err.Error())
envHelp := config.HelpKV{}
if envOnly {
for k, v := range help {
envK := config.EnvPrefix + strings.Join([]string{
strings.ToTitle(subSys), strings.ToTitle(k),
}, config.EnvWordDelimiter)
envHelp[envK] = v
}
help = envHelp
}
w.Flush()
return strings.NewReader(s.String()), nil
return help, nil
}
func configDefaultKVS() map[string]config.KVS {
@ -262,6 +300,8 @@ func newServerConfig() config.Config {
for k := range srvCfg {
// Initialize with default KVS
switch k {
case config.EtcdSubSys:
srvCfg[k][config.Default] = etcd.DefaultKVS
case config.CacheSubSys:
srvCfg[k][config.Default] = cache.DefaultKVS
case config.CompressionSubSys:
@ -312,22 +352,29 @@ func newSrvConfig(objAPI ObjectLayer) error {
return saveServerConfig(context.Background(), objAPI, globalServerConfig, nil)
}
// getValidConfig - returns valid server configuration
func getValidConfig(objAPI ObjectLayer) (config.Config, error) {
srvCfg, err := readServerConfig(context.Background(), objAPI)
if err != nil {
return nil, err
}
defaultKVS := configDefaultKVS()
for _, k := range config.SubSystems.ToSlice() {
_, ok := srvCfg[k][config.Default]
if !ok {
// Populate default configs for any new
// sub-systems added automatically.
srvCfg[k][config.Default] = defaultKVS[k]
}
}
return srvCfg, nil
}
// loadConfig - loads a new config from disk, overrides params from env
// if found and valid
// loadConfig - loads a new config from disk, overrides params
// from env if found and valid
func loadConfig(objAPI ObjectLayer) error {
srvCfg, err := getValidConfig(objAPI)
if err != nil {
return config.ErrInvalidConfig(err)
return err
}
// Override any values from ENVs.

42
cmd/config-migrate.go
View file

@ -2563,12 +2563,7 @@ func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
cfg.Version = "28"
cfg.KMS = crypto.KMSConfig{}
data, err = json.Marshal(cfg)
if err != nil {
return err
}
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
if err = saveServerConfig(context.Background(), objAPI, cfg, nil); err != nil {
return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %v", err)
}
@ -2595,12 +2590,7 @@ func migrateV28ToV29MinioSys(objAPI ObjectLayer) error {
}
cfg.Version = "29"
data, err = json.Marshal(cfg)
if err != nil {
return err
}
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
if err = saveServerConfig(context.Background(), objAPI, cfg, nil); err != nil {
return fmt.Errorf("Failed to migrate config from ‘28’ to ‘29’. %v", err)
}
@ -2632,12 +2622,7 @@ func migrateV29ToV30MinioSys(objAPI ObjectLayer) error {
cfg.Compression.Extensions = strings.Split(compress.DefaultExtensions, config.ValueSeparator)
cfg.Compression.MimeTypes = strings.Split(compress.DefaultMimeTypes, config.ValueSeparator)
data, err = json.Marshal(cfg)
if err != nil {
return err
}
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
if err = saveServerConfig(context.Background(), objAPI, cfg, nil); err != nil {
return fmt.Errorf("Failed to migrate config from ‘29’ to ‘30’. %v", err)
}
@ -2672,12 +2657,7 @@ func migrateV30ToV31MinioSys(objAPI ObjectLayer) error {
AuthToken: "",
}
data, err = json.Marshal(cfg)
if err != nil {
return err
}
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
if err = saveServerConfig(context.Background(), objAPI, cfg, nil); err != nil {
return fmt.Errorf("Failed to migrate config from ‘30’ to ‘31’. %v", err)
}
@ -2707,12 +2687,7 @@ func migrateV31ToV32MinioSys(objAPI ObjectLayer) error {
cfg.Notify.NSQ = make(map[string]target.NSQArgs)
cfg.Notify.NSQ["1"] = target.NSQArgs{}
data, err = json.Marshal(cfg)
if err != nil {
return err
}
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
if err = saveServerConfig(context.Background(), objAPI, cfg, nil); err != nil {
return fmt.Errorf("Failed to migrate config from ‘31’ to ‘32’. %v", err)
}
@ -2740,12 +2715,7 @@ func migrateV32ToV33MinioSys(objAPI ObjectLayer) error {
cfg.Version = "33"
data, err = json.Marshal(cfg)
if err != nil {
return err
}
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
if err = saveServerConfig(context.Background(), objAPI, cfg, nil); err != nil {
return fmt.Errorf("Failed to migrate config from 32 to 33 . %v", err)
}

20
cmd/config.go
View file

@ -17,12 +17,10 @@
package cmd
import (
"bytes"
"context"
"encoding/json"
"fmt"
"path"
"runtime"
"sort"
"strings"
"time"
@ -80,7 +78,11 @@ func delServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV stri
func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) ([]byte, error) {
historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV)
return readConfig(ctx, objAPI, historyFile)
data, err := readConfig(ctx, objAPI, historyFile)
if err != nil {
return nil, err
}
return data, err
}
func saveServerConfigHistory(ctx context.Context, objAPI ObjectLayer, kv []byte) error {
@ -108,8 +110,12 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config interface{
if err != nil && err != errConfigNotFound {
return err
}
// Current config not found, so nothing to backup.
freshConfig = true
if err == errConfigNotFound {
// Current config not found, so nothing to backup.
freshConfig = true
}
// Do not need to decrypt oldData since we are going to
// save it anyway if freshConfig is false.
} else {
oldData, err = json.Marshal(oldConfig)
if err != nil {
@ -135,10 +141,6 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
return nil, err
}
if runtime.GOOS == "windows" {
configData = bytes.Replace(configData, []byte("\r\n"), []byte("\n"), -1)
}
var config = config.New()
if err = json.Unmarshal(configData, &config); err != nil {
return nil, err

8
cmd/config/cache/lookup.go vendored
View file

@ -75,11 +75,13 @@ func LookupConfig(kvs config.KVS) (Config, error) {
if err != nil {
return cfg, err
}
if !stateBool {
return cfg, nil
}
drives := env.Get(EnvCacheDrives, kvs.Get(Drives))
if stateBool {
if len(drives) == 0 {
return cfg, config.Error("'drives' key cannot be empty if you wish to enable caching")
}
}
if len(drives) == 0 {
return cfg, nil
}

4
cmd/config/compress/legacy.go
View file

@ -38,7 +38,7 @@ func SetCompressionConfig(s config.Config, cfg Config) {
return config.StateOff
}(),
config.Comment: "Settings for Compression, after migrating config",
Extensions: strings.Join(cfg.Extensions, ","),
MimeTypes: strings.Join(cfg.MimeTypes, ","),
Extensions: strings.Join(cfg.Extensions, config.ValueSeparator),
MimeTypes: strings.Join(cfg.MimeTypes, config.ValueSeparator),
}
}

68
cmd/config/config.go
View file

@ -23,7 +23,6 @@ import (
"github.com/minio/minio-go/pkg/set"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/color"
"github.com/minio/minio/pkg/env"
)
@ -36,7 +35,7 @@ func (e Error) Error() string {
// Default keys
const (
Default = "_"
Default = `_`
State = "state"
Comment = "comment"
@ -58,6 +57,7 @@ const (
WormSubSys = "worm"
CacheSubSys = "cache"
RegionSubSys = "region"
EtcdSubSys = "etcd"
StorageClassSubSys = "storageclass"
CompressionSubSys = "compression"
KmsVaultSubSys = "kms_vault"
@ -88,6 +88,7 @@ var SubSystems = set.CreateStringSet([]string{
CredentialsSubSys,
WormSubSys,
RegionSubSys,
EtcdSubSys,
CacheSubSys,
StorageClassSubSys,
CompressionSubSys,
@ -114,6 +115,7 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{
CredentialsSubSys,
WormSubSys,
RegionSubSys,
EtcdSubSys,
CacheSubSys,
StorageClassSubSys,
CompressionSubSys,
@ -132,6 +134,10 @@ const (
KvNewline = "\n"
KvDoubleQuote = `"`
KvSingleQuote = `'`
// Env prefix used for all envs in MinIO
EnvPrefix = "MINIO_"
EnvWordDelimiter = `_`
)
// KVS - is a shorthand for some wrapper functions
@ -141,10 +147,6 @@ type KVS map[string]string
func (kvs KVS) String() string {
var s strings.Builder
for k, v := range kvs {
if k == Comment {
// Skip the comment, comment will be printed elsewhere.
continue
}
s.WriteString(k)
s.WriteString(KvSeparator)
s.WriteString(KvDoubleQuote)
@ -167,20 +169,7 @@ func (c Config) String() string {
var s strings.Builder
for k, v := range c {
for target, kv := range v {
c, ok := kv[Comment]
if ok {
// For multiple comments split it correctly.
for _, c1 := range strings.Split(c, KvNewline) {
if c1 == "" {
continue
}
s.WriteString(color.YellowBold(KvComment))
s.WriteString(KvSpaceSeparator)
s.WriteString(color.BlueBold(strings.TrimSpace(c1)))
s.WriteString(KvNewline)
}
}
s.WriteString(color.CyanBold(k))
s.WriteString(k)
if target != Default {
s.WriteString(SubSystemSeparator)
s.WriteString(target)
@ -307,7 +296,7 @@ func (c Config) GetKVS(s string) (map[string]KVS, error) {
}
kvs[inputs[0]], ok = c[subSystemValue[0]][subSystemValue[1]]
if !ok {
err := fmt.Sprintf("sub-system target '%s' doesn't exist, proceed to create a new one", s)
err := fmt.Sprintf("sub-system target '%s' doesn't exist", s)
return nil, Error(err)
}
return kvs, nil
@ -377,7 +366,7 @@ func (c Config) Clone() Config {
}
// SetKVS - set specific key values per sub-system.
func (c Config) SetKVS(s string, comment string, defaultKVS map[string]KVS) error {
func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
if len(s) == 0 {
return Error("input arguments cannot be empty")
}
@ -422,27 +411,18 @@ func (c Config) SetKVS(s string, comment string, defaultKVS map[string]KVS) erro
kvs[kv[0]] = sanitizeValue(kv[1])
}
tgt := Default
if len(subSystemValue) == 2 {
_, ok := c[subSystemValue[0]][subSystemValue[1]]
if !ok {
c[subSystemValue[0]][subSystemValue[1]] = defaultKVS[subSystemValue[0]]
// Add a comment since its a new target, this comment may be
// overridden if client supplied it.
if comment == "" {
comment = fmt.Sprintf("Settings for sub-system target %s:%s",
subSystemValue[0], subSystemValue[1])
}
c[subSystemValue[0]][subSystemValue[1]][Comment] = comment
}
tgt = subSystemValue[1]
}
_, ok := c[subSystemValue[0]][tgt]
if !ok {
c[subSystemValue[0]][tgt] = defaultKVS[subSystemValue[0]]
comment := fmt.Sprintf("Settings for sub-system target %s:%s", subSystemValue[0], tgt)
c[subSystemValue[0]][tgt][Comment] = comment
}
var commentKv bool
for k, v := range kvs {
if k == Comment {
// Set this to true to indicate comment was
// supplied by client and is going to be preserved.
commentKv = true
}
if len(subSystemValue) == 2 {
c[subSystemValue[0]][subSystemValue[1]][k] = v
} else {
@ -450,15 +430,5 @@ func (c Config) SetKVS(s string, comment string, defaultKVS map[string]KVS) erro
}
}
// if client didn't supply the comment try to preserve
// the comment if any we found while parsing the incoming
// stream, if not preserve the default.
if !commentKv && comment != "" {
if len(subSystemValue) == 2 {
c[subSystemValue[0]][subSystemValue[1]][Comment] = comment
} else {
c[subSystemValue[0]][Default][Comment] = comment
}
}
return nil
}

6
cmd/config/errors.go
View file

@ -18,12 +18,6 @@ package config
// UI errors
var (
ErrInvalidConfig = newErrFn(
"Invalid value found in the configuration file",
"Please ensure a valid value in the configuration file",
"For more details, refer to https://docs.min.io/docs/minio-server-configuration-guide",
)
ErrInvalidBrowserValue = newErrFn(
"Invalid browser value",
"Please check the passed value",

105
cmd/config/etcd/etcd.go
View file

@ -36,61 +36,100 @@ const (
// etcd environment values
const (
Endpoints = "endpoints"
CoreDNSPath = "coredns_path"
ClientCert = "client_cert"
ClientCertKey = "client_cert_key"
EnvEtcdState = "MINIO_ETCD_STATE"
EnvEtcdEndpoints = "MINIO_ETCD_ENDPOINTS"
EnvEtcdCoreDNSPath = "MINIO_ETCD_COREDNS_PATH"
EnvEtcdClientCert = "MINIO_ETCD_CLIENT_CERT"
EnvEtcdClientCertKey = "MINIO_ETCD_CLIENT_CERT_KEY"
)
// New - Initialize new etcd client
func New(rootCAs *x509.CertPool) (*clientv3.Client, error) {
envEndpoints := env.Get(EnvEtcdEndpoints, "")
if envEndpoints == "" {
// etcd is not configured, nothing to do.
// DefaultKVS - default KV settings for etcd.
var (
DefaultKVS = config.KVS{
config.State: config.StateOff,
config.Comment: "This is a default etcd configuration",
Endpoints: "",
CoreDNSPath: "/skydns",
ClientCert: "",
ClientCertKey: "",
}
)
// Config - server etcd config.
type Config struct {
Enabled bool `json:"enabled"`
CoreDNSPath string `json:"coreDNSPath"`
clientv3.Config
}
// New - initialize new etcd client.
func New(cfg Config) (*clientv3.Client, error) {
if !cfg.Enabled {
return nil, nil
}
return clientv3.New(cfg.Config)
}
etcdEndpoints := strings.Split(envEndpoints, config.ValueSeparator)
// LookupConfig - Initialize new etcd config.
func LookupConfig(kv config.KVS, rootCAs *x509.CertPool) (Config, error) {
cfg := Config{}
if err := config.CheckValidKeys(config.EtcdSubSys, kv, DefaultKVS); err != nil {
return cfg, err
}
stateBool, err := config.ParseBool(env.Get(EnvEtcdState, kv.Get(config.State)))
if err != nil {
return cfg, err
}
endpoints := env.Get(EnvEtcdEndpoints, kv.Get(Endpoints))
if stateBool && len(endpoints) == 0 {
return cfg, config.Error("'endpoints' key cannot be empty if you wish to enable etcd")
}
if len(endpoints) == 0 {
return cfg, nil
}
cfg.Enabled = true
etcdEndpoints := strings.Split(endpoints, config.ValueSeparator)
var etcdSecure bool
for _, endpoint := range etcdEndpoints {
if endpoint == "" {
continue
}
u, err := xnet.ParseURL(endpoint)
if err != nil {
return nil, err
return cfg, err
}
// If one of the endpoint is https, we will use https directly.
etcdSecure = etcdSecure || u.Scheme == "https"
}
var err error
var etcdClnt *clientv3.Client
cfg.DialTimeout = defaultDialTimeout
cfg.DialKeepAliveTime = defaultDialKeepAlive
cfg.Endpoints = etcdEndpoints
cfg.CoreDNSPath = env.Get(EnvEtcdCoreDNSPath, kv.Get(CoreDNSPath))
if etcdSecure {
cfg.TLS = &tls.Config{
RootCAs: rootCAs,
}
// This is only to support client side certificate authentication
// https://coreos.com/etcd/docs/latest/op-guide/security.html
etcdClientCertFile, ok1 := env.Lookup(EnvEtcdClientCert)
etcdClientCertKey, ok2 := env.Lookup(EnvEtcdClientCertKey)
var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
if ok1 && ok2 {
getClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
cert, terr := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
return &cert, terr
etcdClientCertFile := env.Get(EnvEtcdClientCert, kv.Get(ClientCert))
etcdClientCertKey := env.Get(EnvEtcdClientCertKey, kv.Get(ClientCertKey))
if etcdClientCertFile != "" && etcdClientCertKey != "" {
cfg.TLS.GetClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
cert, err := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
return &cert, err
}
}
etcdClnt, err = clientv3.New(clientv3.Config{
Endpoints: etcdEndpoints,
DialTimeout: defaultDialTimeout,
DialKeepAliveTime: defaultDialKeepAlive,
TLS: &tls.Config{
RootCAs: rootCAs,
GetClientCertificate: getClientCertificate,
},
})
} else {
etcdClnt, err = clientv3.New(clientv3.Config{
Endpoints: etcdEndpoints,
DialTimeout: defaultDialTimeout,
DialKeepAliveTime: defaultDialKeepAlive,
})
}
return etcdClnt, err
return cfg, nil
}

31
cmd/config/etcd/help.go Normal file
View file

@ -0,0 +1,31 @@
/*
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package etcd
import "github.com/minio/minio/cmd/config"
// etcd config documented in default config
var (
Help = config.HelpKV{
Endpoints: `(required) Comma separated list of etcd endpoints eg: "http://localhost:2379"`,
CoreDNSPath: `(optional) CoreDNS etcd path location to populate DNS srv records eg: "/skydns"`,
ClientCert: `(optional) Etcd client cert for mTLS authentication`,
ClientCertKey: `(optional) Etcd client cert key for mTLS authentication`,
config.State: "Indicates if etcd config is on or off",
config.Comment: "A comment to describe the etcd settings",
}
)

22
cmd/config/help.go
View file

@ -16,32 +16,10 @@
package config
import (
"text/template"
"github.com/minio/minio/pkg/color"
)
// HelpKV - implements help messages for keys
// with value as description of the keys.
type HelpKV map[string]string
// Help template used by all sub-systems
const Help = `{{colorBlueBold "Key"}}{{"\t"}}{{colorBlueBold "Description"}}
{{colorYellowBold "----"}}{{"\t"}}{{colorYellowBold "----"}}
{{range $key, $value := .}}{{colorCyanBold $key}}{{ "\t" }}{{$value}}
{{end}}`
var funcMap = template.FuncMap{
"colorBlueBold": color.BlueBold,
"colorYellowBold": color.YellowBold,
"colorCyanBold": color.CyanBold,
"colorGreenBold": color.GreenBold,
}
// HelpTemplate - captures config help template
var HelpTemplate = template.Must(template.New("config-help").Funcs(funcMap).Parse(Help))
// Region and Worm help is documented in default config
var (
RegionHelp = HelpKV{

8
cmd/config/identity/ldap/config.go
View file

@ -117,10 +117,12 @@ func Lookup(kvs config.KVS, rootCAs *x509.CertPool) (l Config, err error) {
if err != nil {
return l, err
}
if !stateBool {
return l, nil
}
ldapServer := env.Get(EnvServerAddr, kvs.Get(ServerAddr))
if stateBool {
if ldapServer == "" {
return l, config.Error("'serveraddr' cannot be empty if you wish to enable AD/LDAP support")
}
}
if ldapServer == "" {
return l, nil
}

43
cmd/config/identity/openid/jwt.go
View file

@ -207,6 +207,7 @@ const (
ConfigURL = "config_url"
ClaimPrefix = "claim_prefix"
EnvIdentityOpenIDState = "MINIO_IDENTITY_OPENID_STATE"
EnvIdentityOpenIDJWKSURL = "MINIO_IDENTITY_OPENID_JWKS_URL"
EnvIdentityOpenIDURL = "MINIO_IDENTITY_OPENID_CONFIG_URL"
EnvIdentityOpenIDClaimPrefix = "MINIO_IDENTITY_OPENID_CLAIM_PREFIX"
@ -271,20 +272,10 @@ func LookupConfig(kv config.KVS, transport *http.Transport, closeRespFn func(io.
return c, err
}
stateBool, err := config.ParseBool(kv.Get(config.State))
stateBool, err := config.ParseBool(env.Get(EnvIdentityOpenIDState, kv.Get(config.State)))
if err != nil {
return c, err
}
if !stateBool {
return c, nil
}
c = Config{
ClaimPrefix: env.Get(EnvIdentityOpenIDClaimPrefix, kv.Get(ClaimPrefix)),
publicKeys: make(map[string]crypto.PublicKey),
transport: transport,
closeRespFn: closeRespFn,
}
jwksURL := env.Get(EnvIamJwksURL, "") // Legacy
if jwksURL == "" {
@ -306,15 +297,31 @@ func LookupConfig(kv config.KVS, transport *http.Transport, closeRespFn func(io.
// Fallback to discovery document jwksURL
jwksURL = c.DiscoveryDoc.JwksURI
}
if jwksURL != "" {
c.JWKS.URL, err = xnet.ParseURL(jwksURL)
if err != nil {
return c, err
}
if err = c.PopulatePublicKey(); err != nil {
return c, err
if stateBool {
// This check is needed to ensure that empty Jwks urls are not allowed.
if jwksURL == "" {
return c, config.Error("'config_url' must be set to a proper OpenID discovery document URL")
}
}
if jwksURL == "" {
return c, nil
}
c = Config{
ClaimPrefix: env.Get(EnvIdentityOpenIDClaimPrefix, kv.Get(ClaimPrefix)),
publicKeys: make(map[string]crypto.PublicKey),
transport: transport,
closeRespFn: closeRespFn,
}
c.JWKS.URL, err = xnet.ParseURL(jwksURL)
if err != nil {
return c, err
}
if err = c.PopulatePublicKey(); err != nil {
return c, err
}
return c, nil
}

2
cmd/config/notify/legacy.go
View file

@ -27,7 +27,7 @@ func SetNotifyKafka(s config.Config, kName string, cfg target.KafkaArgs) error {
for _, broker := range cfg.Brokers {
brokers = append(brokers, broker.String())
}
return strings.Join(brokers, ",")
return strings.Join(brokers, config.ValueSeparator)
}(),
config.Comment: "Settings for Kafka notification, after migrating config",
target.KafkaTopic: cfg.Topic,

9
cmd/config/storageclass/storage-class.go
View file

@ -224,8 +224,13 @@ func LookupConfig(kvs config.KVS, drivesPerSet int) (cfg Config, err error) {
if err != nil {
return cfg, err
}
if !stateBool {
return cfg, nil
if stateBool {
if ssc := env.Get(StandardEnv, kvs.Get(ClassStandard)); ssc == "" {
return cfg, config.Error("'standard' key cannot be empty if you wish to enable storage class")
}
if rrsc := env.Get(RRSEnv, kvs.Get(ClassRRS)); rrsc == "" {
return cfg, config.Error("'rrs' key cannot be empty if you wish to enable storage class")
}
}
// Check for environment variables and parse into storageClass struct

5
cmd/crypto/legacy.go
View file

@ -122,7 +122,10 @@ func lookupConfigLegacy(kvs config.KVS) (KMSConfig, error) {
cfg := KMSConfig{
AutoEncryption: autoBool,
}
stateBool, err := config.ParseBool(kvs.Get(config.State))
// Assume default as "on" for legacy config since we didn't have a _STATE
// flag to turn it off, but we should honor it nonetheless to turn it off
// if the vault endpoint is down and there is no way to start the server.
stateBool, err := config.ParseBool(env.Get(EnvKMSVaultState, config.StateOn))
if err != nil {
return cfg, err
}

6
cmd/crypto/vault.go
View file

@ -69,7 +69,11 @@ type vaultService struct {
var _ KMS = (*vaultService)(nil) // compiler check that *vaultService implements KMS
// empty/default vault configuration used to check whether a particular is empty.
var emptyVaultConfig = VaultConfig{}
var emptyVaultConfig = VaultConfig{
Auth: VaultAuth{
Type: "approle",
},
}
// IsEmpty returns true if the vault config struct is an
// empty configuration.

2
cmd/crypto/vault_test.go
View file

@ -25,7 +25,7 @@ var verifyVaultConfigTests = []struct {
}{
{
ShouldFail: false, // 0
Config: VaultConfig{},
Config: emptyVaultConfig,
},
{
ShouldFail: true,

3
cmd/globals.go
View file

@ -184,7 +184,8 @@ var (
// Time when object layer was initialized on start up.
globalBootTime time.Time
globalActiveCred auth.Credentials
globalActiveCred auth.Credentials
globalPublicCerts []*x509.Certificate
globalDomainNames []string // Root domains for virtual host style requests

1
cmd/iam-etcd-store.go
View file

@ -117,6 +117,7 @@ func (ies *IAMEtcdStore) loadIAMConfig(item interface{}, path string) error {
if err != nil {
return err
}
return json.Unmarshal(pdata, item)
}

4
cmd/object-api-utils.go
View file

@ -328,7 +328,7 @@ func isMinioReservedBucket(bucketName string) bool {
func getHostsSlice(records []dns.SrvRecord) []string {
var hosts []string
for _, r := range records {
hosts = append(hosts, net.JoinHostPort(r.Host, fmt.Sprintf("%d", r.Port)))
hosts = append(hosts, net.JoinHostPort(r.Host, r.Port))
}
return hosts
}
@ -337,7 +337,7 @@ func getHostsSlice(records []dns.SrvRecord) []string {
func getHostFromSrv(records []dns.SrvRecord) string {
rand.Seed(time.Now().Unix())
srvRecord := records[rand.Intn(len(records))]
return net.JoinHostPort(srvRecord.Host, fmt.Sprintf("%d", srvRecord.Port))
return net.JoinHostPort(srvRecord.Host, srvRecord.Port)
}
// IsCompressed returns true if the object is marked as compressed.

4
cmd/prepare-storage.go
View file

@ -20,6 +20,7 @@ import (
"context"
"fmt"
"os"
"sync"
"time"
"github.com/minio/minio/cmd/logger"
@ -27,11 +28,14 @@ import (
)
var printEndpointError = func() func(Endpoint, error) {
var mutex sync.Mutex
printOnce := make(map[Endpoint]map[string]bool)
return func(endpoint Endpoint, err error) {
reqInfo := (&logger.ReqInfo{}).AppendTags("endpoint", endpoint.String())
ctx := logger.SetReqInfo(context.Background(), reqInfo)
mutex.Lock()
defer mutex.Unlock()
m, ok := printOnce[endpoint]
if !ok {
m = make(map[string]bool)

1
cmd/server-main.go
View file

@ -192,6 +192,7 @@ func serverHandleEnvVars() {
}
globalActiveCred = cred
}
}
// serverMain handler called for 'minio server' command.

2
cmd/storage-rest-common.go
View file

@ -1,5 +1,5 @@
/*
* MinIO Cloud Storage, (C) 2018 MinIO, Inc.
* MinIO Cloud Storage, (C) 2018-2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

11
docs/config/README.md
View file

@ -12,6 +12,8 @@ Additionally `--config-dir` is now a legacy option which will is scheduled for r
minio server /data
```
MinIO also encrypts all the config, IAM and policies content with admin credentials.
### Certificate Directory
TLS certificates by default are stored under ``${HOME}/.minio/certs`` directory. You need to place certificates here to enable `HTTPS` based access. Read more about [How to secure access to MinIO server with TLS](https://docs.min.io/docs/how-to-secure-access-to-minio-server-with-tls).
@ -29,6 +31,15 @@ $ mc tree --files ~/.minio
You can provide a custom certs directory using `--certs-dir` command line option.
#### Credentials
On MinIO admin credentials or root credentials are only allowed to be changed using ENVs `MINIO_ACCESS_KEY` and `MINIO_SECRET_KEY`.
```
export MINIO_ACCESS_KEY=minio
export MINIO_SECRET_KEY=minio13
minio server /data
```
#### Region
| Field | Type | Description |
|:--------------------------|:---------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|

23
docs/gateway/s3.md
View file

@ -62,6 +62,29 @@ Minimum permissions required if you wish to provide restricted access with your
## Run MinIO Gateway for AWS S3 compatible services
As a prerequisite to run MinIO S3 gateway on an AWS S3 compatible service, you need valid access key, secret key and service endpoint.
## Run MinIO Gateway with double-encryption
MinIO gateway to S3 supports encryption of data at rest. Three types of encryption modes are supported
- encryption can be set to ``pass-through`` to backend
- ``single encryption`` (at the gateway)
- ``double encryption`` (single encryption at gateway and pass through to backend).
This can be specified by setting MINIO_GATEWAY_SSE environment variable. If MINIO_GATEWAY_SSE and KMS are not setup, all encryption headers are passed through to the backend. If KMS environment variables are set up, ``single encryption`` is automatically performed at the gateway and encrypted object is saved at the backend.
To specify ``double encryption``, MINIO_GATEWAY_SSE environment variable needs to be set to "s3" for sse-s3
and "c" for sse-c encryption. More than one encryption option can be set, delimited by ";". Objects are encrypted at the gateway and the gateway also does a pass-through to backend. Note that in the case of SSE-C encryption, gateway derives a unique SSE-C key for pass through from the SSE-C client key using a key derivation function (KDF).
```sh
export MINIO_GATEWAY_SSE="s3;c"
export MINIO_KMS_VAULT_STATE=on
export MINIO_KMS_VAULT_APPROLE_ID=9b56cc08-8258-45d5-24a3-679876769126
export MINIO_KMS_VAULT_APPROLE_SECRET=4e30c52f-13e4-a6f5-0763-d50e8cb4321f
export MINIO_KMS_VAULT_ENDPOINT=https://vault-endpoint-ip:8200
export MINIO_KMS_VAULT_KEY_NAME=my-minio-key
export MINIO_KMS_VAULT_AUTH_TYPE=approle
minio gateway s3
```
### Using Docker
```
docker run -p 9000:9000 --name minio-s3 \

23
docs/kms/README.md
View file

@ -20,10 +20,7 @@ MinIO supports two different KMS concepts:
Further if the MinIO server machine is ever compromised, then the master key must also be treated as compromised.
**Important:**
If multiple MinIO servers are configured as [gateways](https://github.com/minio/minio/blob/master/docs/gateway/README.md)
pointing to the *same* backend - for example the same NAS storage - then the KMS configuration **must** be the same for
all gateways. Otherwise one gateway may not be able to decrypt objects created by another gateway. It is the operators'
responsibility to ensure consistency.
If multiple MinIO servers are configured as [gateways](https://github.com/minio/minio/blob/master/docs/gateway/README.md) pointing to the *same* backend - for example the same NAS storage - then the KMS configuration **must** be the same for all gateways. Otherwise one gateway may not be able to decrypt objects created by another gateway. It is the operator responsibility to ensure consistency.
## Get started
@ -197,24 +194,6 @@ export MINIO_KMS_VAULT_NAMESPACE=ns1
Note: If [Vault Namespaces](https://learn.hashicorp.com/vault/operations/namespaces) are in use, MINIO_KMS_VAULT_VAULT_NAMESPACE variable needs to be set before setting approle and transit secrets engine.
MinIO gateway to S3 supports encryption. Three encryption modes are possible - encryption can be set to ``pass-through`` to backend, ``single encryption`` (at the gateway) or ``double encryption`` (single encryption at gateway and pass through to backend). This can be specified by setting MINIO_GATEWAY_SSE and KMS environment variables set in Step 2.1.2.
If MINIO_GATEWAY_SSE and KMS are not setup, all encryption headers are passed through to the backend. If KMS environment variables are set up, ``single encryption`` is automatically performed at the gateway and encrypted object is saved at the backend.
To specify ``double encryption``, MINIO_GATEWAY_SSE environment variable needs to be set to "s3" for sse-s3
and "c" for sse-c encryption. More than one encryption option can be set, delimited by ";". Objects are encrypted at the gateway and the gateway also does a pass-through to backend. Note that in the case of SSE-C encryption, gateway derives a unique SSE-C key for pass through from the SSE-C client key using a KDF.
```sh
export MINIO_GATEWAY_SSE="s3;c"
export MINIO_KMS_VAULT_STATE=on
export MINIO_KMS_VAULT_APPROLE_ID=9b56cc08-8258-45d5-24a3-679876769126
export MINIO_KMS_VAULT_APPROLE_SECRET=4e30c52f-13e4-a6f5-0763-d50e8cb4321f
export MINIO_KMS_VAULT_ENDPOINT=https://vault-endpoint-ip:8200
export MINIO_KMS_VAULT_KEY_NAME=my-minio-key
export MINIO_KMS_VAULT_AUTH_TYPE=approle
minio gateway s3
```
#### 2.2 Specify a master key
**2.2.1 KMS master key from environment variables**

2
docs/sts/README.md
View file

@ -34,6 +34,7 @@ Make sure we have followed the previous step and configured each software indepe
```
export MINIO_ACCESS_KEY=minio
export MINIO_SECRET_KEY=minio123
export MINIO_IDENTITY_OPENID_STATE="on"
export MINIO_IDENTITY_OPENID_CONFIG_URL=https://localhost:9443/oauth2/oidcdiscovery/.well-known/openid-configuration
minio server /mnt/data
```
@ -46,6 +47,7 @@ Make sure we have followed the previous step and configured each software indepe
```
export MINIO_ACCESS_KEY=aws_access_key
export MINIO_SECRET_KEY=aws_secret_key
export MINIO_IDENTITY_OPENID_STATE="on"
export MINIO_IDENTITY_OPENID_CONFIG_URL=https://localhost:9443/oauth2/oidcdiscovery/.well-known/openid-configuration
export MINIO_ETCD_ENDPOINTS=http://localhost:2379
minio gateway s3

33
pkg/color/color.go
View file

@ -1,11 +1,25 @@
/*
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package color
import (
"fmt"
"os"
"github.com/fatih/color"
"github.com/mattn/go-isatty"
)
// global colors.
@ -13,7 +27,7 @@ var (
// Check if we stderr, stdout are dumb terminals, we do not apply
// ansi coloring on dumb terminals.
IsTerminal = func() bool {
return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd())
return !color.NoColor
}
Bold = func() func(a ...interface{}) string {
@ -22,78 +36,91 @@ var (
}
return fmt.Sprint
}()
Red = func() func(format string, a ...interface{}) string {
if IsTerminal() {
return color.New(color.FgRed).SprintfFunc()
}
return fmt.Sprintf
}()
Blue = func() func(format string, a ...interface{}) string {
if IsTerminal() {
return color.New(color.FgBlue).SprintfFunc()
}
return fmt.Sprintf
}()
Yellow = func() func(format string, a ...interface{}) string {
if IsTerminal() {
return color.New(color.FgYellow).SprintfFunc()
}
return fmt.Sprintf
}()
Green = func() func(a ...interface{}) string {
if IsTerminal() {
return color.New(color.FgGreen).SprintFunc()
}
return fmt.Sprint
}()
GreenBold = func() func(a ...interface{}) string {
if IsTerminal() {
return color.New(color.FgGreen, color.Bold).SprintFunc()
}
return fmt.Sprint
}()
CyanBold = func() func(a ...interface{}) string {
if IsTerminal() {
return color.New(color.FgCyan, color.Bold).SprintFunc()
}
return fmt.Sprint
}()
YellowBold = func() func(format string, a ...interface{}) string {
if IsTerminal() {
return color.New(color.FgYellow, color.Bold).SprintfFunc()
}
return fmt.Sprintf
}()
BlueBold = func() func(format string, a ...interface{}) string {
if IsTerminal() {
return color.New(color.FgBlue, color.Bold).SprintfFunc()
}
return fmt.Sprintf
}()
BgYellow = func() func(format string, a ...interface{}) string {
if IsTerminal() {
return color.New(color.BgYellow).SprintfFunc()
}
return fmt.Sprintf
}()
Black = func() func(format string, a ...interface{}) string {
if IsTerminal() {
return color.New(color.FgBlack).SprintfFunc()
}
return fmt.Sprintf
}()
FgRed = func() func(a ...interface{}) string {
if IsTerminal() {
return color.New(color.FgRed).SprintFunc()
}
return fmt.Sprint
}()
BgRed = func() func(format string, a ...interface{}) string {
if IsTerminal() {
return color.New(color.BgRed).SprintfFunc()
}
return fmt.Sprintf
}()
FgWhite = func() func(format string, a ...interface{}) string {
if IsTerminal() {
return color.New(color.FgWhite).SprintfFunc()

104
pkg/dns/etcd_dns.go
View file

@ -23,7 +23,6 @@ import (
"fmt"
"net"
"sort"
"strconv"
"strings"
"time"
@ -39,7 +38,7 @@ var ErrNoEntriesFound = errors.New("No entries found for this key")
const etcdPathSeparator = "/"
// create a new coredns service record for the bucket.
func newCoreDNSMsg(ip string, port int, ttl uint32) ([]byte, error) {
func newCoreDNSMsg(ip string, port string, ttl uint32) ([]byte, error) {
return json.Marshal(&SrvRecord{
Host: ip,
Port: port,
@ -48,11 +47,11 @@ func newCoreDNSMsg(ip string, port int, ttl uint32) ([]byte, error) {
})
}
// Retrieves list of DNS entries for the domain.
func (c *coreDNS) List() ([]SrvRecord, error) {
// List - Retrieves list of DNS entries for the domain.
func (c *CoreDNS) List() ([]SrvRecord, error) {
var srvRecords []SrvRecord
for _, domainName := range c.domainNames {
key := msg.Path(fmt.Sprintf("%s.", domainName), defaultPrefixPath)
key := msg.Path(fmt.Sprintf("%s.", domainName), c.prefixPath)
records, err := c.list(key)
if err != nil {
return nil, err
@ -67,11 +66,11 @@ func (c *coreDNS) List() ([]SrvRecord, error) {
return srvRecords, nil
}
// Retrieves DNS records for a bucket.
func (c *coreDNS) Get(bucket string) ([]SrvRecord, error) {
// Get - Retrieves DNS records for a bucket.
func (c *CoreDNS) Get(bucket string) ([]SrvRecord, error) {
var srvRecords []SrvRecord
for _, domainName := range c.domainNames {
key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), defaultPrefixPath)
key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), c.prefixPath)
records, err := c.list(key)
if err != nil {
return nil, err
@ -105,7 +104,7 @@ func msgUnPath(s string) string {
// Retrieves list of entries under the key passed.
// Note that this method fetches entries upto only two levels deep.
func (c *coreDNS) list(key string) ([]SrvRecord, error) {
func (c *CoreDNS) list(key string) ([]SrvRecord, error) {
ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
r, err := c.etcdClient.Get(ctx, key, etcd.WithPrefix())
defer cancel()
@ -153,15 +152,15 @@ func (c *coreDNS) list(key string) ([]SrvRecord, error) {
return srvRecords, nil
}
// Adds DNS entries into etcd endpoint in CoreDNS etcd message format.
func (c *coreDNS) Put(bucket string) error {
// Put - Adds DNS entries into etcd endpoint in CoreDNS etcd message format.
func (c *CoreDNS) Put(bucket string) error {
for ip := range c.domainIPs {
bucketMsg, err := newCoreDNSMsg(ip, c.domainPort, defaultTTL)
if err != nil {
return err
}
for _, domainName := range c.domainNames {
key := msg.Path(fmt.Sprintf("%s.%s", bucket, domainName), defaultPrefixPath)
key := msg.Path(fmt.Sprintf("%s.%s", bucket, domainName), c.prefixPath)
key = key + etcdPathSeparator + ip
ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
_, err = c.etcdClient.Put(ctx, key, string(bucketMsg))
@ -177,10 +176,10 @@ func (c *coreDNS) Put(bucket string) error {
return nil
}
// Removes DNS entries added in Put().
func (c *coreDNS) Delete(bucket string) error {
// Delete - Removes DNS entries added in Put().
func (c *CoreDNS) Delete(bucket string) error {
for _, domainName := range c.domainNames {
key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), defaultPrefixPath)
key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), c.prefixPath)
srvRecords, err := c.list(key)
if err != nil {
return err
@ -197,10 +196,10 @@ func (c *coreDNS) Delete(bucket string) error {
return nil
}
// Removes a specific DNS entry
func (c *coreDNS) DeleteRecord(record SrvRecord) error {
// DeleteRecord - Removes a specific DNS entry
func (c *CoreDNS) DeleteRecord(record SrvRecord) error {
for _, domainName := range c.domainNames {
key := msg.Path(fmt.Sprintf("%s.%s.", record.Key, domainName), defaultPrefixPath)
key := msg.Path(fmt.Sprintf("%s.%s.", record.Key, domainName), c.prefixPath)
dctx, dcancel := context.WithTimeout(context.Background(), defaultContextTimeout)
if _, err := c.etcdClient.Delete(dctx, key+etcdPathSeparator+record.Host); err != nil {
@ -213,26 +212,71 @@ func (c *coreDNS) DeleteRecord(record SrvRecord) error {
}
// CoreDNS - represents dns config for coredns server.
type coreDNS struct {
type CoreDNS struct {
domainNames []string
domainIPs set.StringSet
domainPort int
domainPort string
prefixPath string
etcdClient *etcd.Client
}
// Option - functional options pattern style
type Option func(*CoreDNS)
// DomainNames set a list of domain names used by this CoreDNS
// client setting, note this will fail if set to empty when
// constructor initializes.
func DomainNames(domainNames []string) Option {
return func(args *CoreDNS) {
args.domainNames = domainNames
}
}
// DomainIPs set a list of custom domain IPs, note this will
// fail if set to empty when constructor initializes.
func DomainIPs(domainIPs set.StringSet) Option {
return func(args *CoreDNS) {
args.domainIPs = domainIPs
}
}
// DomainPort - is a string version of server port
func DomainPort(domainPort string) Option {
return func(args *CoreDNS) {
args.domainPort = domainPort
}
}
// CoreDNSPath - custom prefix on etcd to populate DNS
// service records, optional and can be empty.
// if empty then c.prefixPath is used i.e "/skydns"
func CoreDNSPath(prefix string) Option {
return func(args *CoreDNS) {
args.prefixPath = prefix
}
}
// NewCoreDNS - initialize a new coreDNS set/unset values.
func NewCoreDNS(domainNames []string, domainIPs set.StringSet, domainPort string, etcdClient *etcd.Client) (Config, error) {
if len(domainNames) == 0 || domainIPs.IsEmpty() {
func NewCoreDNS(etcdClient *etcd.Client, setters ...Option) (Config, error) {
if etcdClient == nil {
return nil, errors.New("invalid argument")
}
port, err := strconv.Atoi(domainPort)
if err != nil {
return nil, err
args := &CoreDNS{
etcdClient: etcdClient,
prefixPath: defaultPrefixPath,
}
for _, setter := range setters {
setter(args)
}
if len(args.domainNames) == 0 || args.domainIPs.IsEmpty() {
return nil, errors.New("invalid argument")
}
// strip ports off of domainIPs
domainIPsWithoutPorts := domainIPs.ApplyFunc(func(ip string) string {
domainIPsWithoutPorts := args.domainIPs.ApplyFunc(func(ip string) string {
host, _, err := net.SplitHostPort(ip)
if err != nil {
if strings.Contains(err.Error(), "missing port in address") {
@ -241,11 +285,7 @@ func NewCoreDNS(domainNames []string, domainIPs set.StringSet, domainPort string
}
return host
})
args.domainIPs = domainIPsWithoutPorts
return &coreDNS{
domainNames: domainNames,
domainIPs: domainIPsWithoutPorts,
domainPort: port,
etcdClient: etcdClient,
}, nil
return args, nil
}

2
pkg/dns/types.go
View file

@ -29,7 +29,7 @@ const (
// SrvRecord - represents a DNS service record
type SrvRecord struct {
Host string `json:"host,omitempty"`
Port int `json:"port,omitempty"`
Port string `json:"port,omitempty"`
Priority int `json:"priority,omitempty"`
Weight int `json:"weight,omitempty"`
Text string `json:"text,omitempty"`

57
pkg/madmin/config-help-commands.go
View file

@ -18,16 +18,50 @@
package madmin
import (
"encoding/json"
"io"
"net/http"
"net/url"
"strings"
"text/tabwriter"
"text/template"
"github.com/minio/minio/pkg/color"
)
// Help template used by all sub-systems
const Help = `{{colorBlueBold "Key"}}{{"\t"}}{{colorBlueBold "Description"}}
{{colorYellowBold "----"}}{{"\t"}}{{colorYellowBold "----"}}
{{range $key, $value := .}}{{colorCyanBold $key}}{{ "\t" }}{{$value}}
{{end}}`
// HelpEnv template used by all sub-systems
const HelpEnv = `{{colorBlueBold "KeyEnv"}}{{"\t"}}{{colorBlueBold "Description"}}
{{colorYellowBold "----"}}{{"\t"}}{{colorYellowBold "----"}}
{{range $key, $value := .}}{{colorCyanBold $key}}{{ "\t" }}{{$value}}
{{end}}`
var funcMap = template.FuncMap{
"colorBlueBold": color.BlueBold,
"colorYellowBold": color.YellowBold,
"colorCyanBold": color.CyanBold,
}
// HelpTemplate - captures config help template
var HelpTemplate = template.Must(template.New("config-help").Funcs(funcMap).Parse(Help))
// HelpEnvTemplate - captures config help template
var HelpEnvTemplate = template.Must(template.New("config-help-env").Funcs(funcMap).Parse(HelpEnv))
// HelpConfigKV - return help for a given sub-system.
func (adm *AdminClient) HelpConfigKV(subSys, key string) (io.ReadCloser, error) {
func (adm *AdminClient) HelpConfigKV(subSys, key string, envOnly bool) (io.Reader, error) {
v := url.Values{}
v.Set("subSys", subSys)
v.Set("key", key)
if envOnly {
v.Set("env", "")
}
reqData := requestData{
relPath: adminAPIPrefix + "/help-config-kv",
queryValues: v,
@ -38,12 +72,29 @@ func (adm *AdminClient) HelpConfigKV(subSys, key string) (io.ReadCloser, error)
if err != nil {
return nil, err
}
defer closeResponse(resp)
if resp.StatusCode != http.StatusOK {
defer closeResponse(resp)
return nil, httpRespToErrorResponse(resp)
}
return resp.Body, nil
var help = make(map[string]string)
d := json.NewDecoder(resp.Body)
if err = d.Decode(&help); err != nil {
return nil, err
}
var s strings.Builder
w := tabwriter.NewWriter(&s, 1, 8, 2, ' ', 0)
if !envOnly {
err = HelpTemplate.Execute(w, help)
} else {
err = HelpEnvTemplate.Execute(w, help)
}
if err != nil {
return nil, err
}
w.Flush()
return strings.NewReader(s.String()), nil
}

41
pkg/madmin/config-kv-commands.go
View file

@ -18,8 +18,11 @@
package madmin
import (
"bufio"
"encoding/base64"
"net/http"
"net/url"
"strings"
)
// DelConfigKV - delete key from server config.
@ -51,7 +54,32 @@ func (adm *AdminClient) DelConfigKV(k string) (err error) {
// SetConfigKV - set key value config to server.
func (adm *AdminClient) SetConfigKV(kv string) (err error) {
econfigBytes, err := EncryptData(adm.secretAccessKey, []byte(kv))
bio := bufio.NewScanner(strings.NewReader(kv))
var s strings.Builder
var comment string
for bio.Scan() {
if bio.Text() == "" {
continue
}
if strings.HasPrefix(bio.Text(), KvComment) {
// Join multiple comments for each newline, separated by "\n"
comments := []string{comment, strings.TrimPrefix(bio.Text(), KvComment)}
comment = strings.Join(comments, KvNewline)
continue
}
s.WriteString(bio.Text())
if comment != "" {
s.WriteString(KvSpaceSeparator)
s.WriteString(commentKey)
s.WriteString(KvSeparator)
s.WriteString(KvDoubleQuote)
s.WriteString(base64.RawStdEncoding.EncodeToString([]byte(comment)))
s.WriteString(KvDoubleQuote)
}
comment = ""
}
econfigBytes, err := EncryptData(adm.secretAccessKey, []byte(s.String()))
if err != nil {
return err
}
@ -77,7 +105,7 @@ func (adm *AdminClient) SetConfigKV(kv string) (err error) {
}
// GetConfigKV - returns the key, value of the requested key, incoming data is encrypted.
func (adm *AdminClient) GetConfigKV(key string) ([]byte, error) {
func (adm *AdminClient) GetConfigKV(key string) (Targets, error) {
v := url.Values{}
v.Set("key", key)
@ -92,9 +120,16 @@ func (adm *AdminClient) GetConfigKV(key string) ([]byte, error) {
return nil, err
}
defer closeResponse(resp)
if resp.StatusCode != http.StatusOK {
return nil, httpRespToErrorResponse(resp)
}
return DecryptData(adm.secretAccessKey, resp.Body)
data, err := DecryptData(adm.secretAccessKey, resp.Body)
if err != nil {
return nil, err
}
return parseSubSysTarget(data)
}

156
pkg/madmin/parse-kv.go Normal file
View file

@ -0,0 +1,156 @@
/*
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package madmin
import (
"bufio"
"bytes"
"encoding/base64"
"fmt"
"strings"
"github.com/minio/minio/pkg/color"
)
// KVS each sub-system key, value
type KVS map[string]string
// Targets sub-system targets
type Targets map[string]map[string]KVS
const (
commentKey = "comment"
)
func (t Targets) String() string {
var s strings.Builder
for subSys, targetKV := range t {
for target, kv := range targetKV {
c := kv[commentKey]
data, err := base64.RawStdEncoding.DecodeString(c)
if err == nil {
c = string(data)
}
for _, c1 := range strings.Split(c, KvNewline) {
if c1 == "" {
continue
}
s.WriteString(color.YellowBold(KvComment))
s.WriteString(KvSpaceSeparator)
s.WriteString(color.BlueBold(strings.TrimSpace(c1)))
s.WriteString(KvNewline)
}
s.WriteString(subSys)
if target != Default {
s.WriteString(SubSystemSeparator)
s.WriteString(target)
}
s.WriteString(KvSpaceSeparator)
for k, v := range kv {
// Comment is already printed, do not print it here.
if k == commentKey {
continue
}
s.WriteString(k)
s.WriteString(KvSeparator)
s.WriteString(KvDoubleQuote)
s.WriteString(v)
s.WriteString(KvDoubleQuote)
s.WriteString(KvSpaceSeparator)
}
if len(t) > 1 {
s.WriteString(KvNewline)
s.WriteString(KvNewline)
}
}
}
return s.String()
}
// Constant separators
const (
SubSystemSeparator = `:`
KvSeparator = `=`
KvSpaceSeparator = ` `
KvComment = `#`
KvDoubleQuote = `"`
KvSingleQuote = `'`
KvNewline = "\n"
Default = `_`
)
// This function is needed, to trim off single or double quotes, creeping into the values.
func sanitizeValue(v string) string {
v = strings.TrimSuffix(strings.TrimPrefix(strings.TrimSpace(v), KvDoubleQuote), KvDoubleQuote)
return strings.TrimSuffix(strings.TrimPrefix(v, KvSingleQuote), KvSingleQuote)
}
func convertTargets(s string, targets Targets) error {
inputs := strings.SplitN(s, KvSpaceSeparator, 2)
if len(inputs) <= 1 {
return fmt.Errorf("invalid number of arguments '%s'", s)
}
subSystemValue := strings.SplitN(inputs[0], SubSystemSeparator, 2)
if len(subSystemValue) == 0 {
return fmt.Errorf("invalid number of arguments %s", s)
}
var kvs = KVS{}
var prevK string
for _, v := range strings.Fields(inputs[1]) {
kv := strings.SplitN(v, KvSeparator, 2)
if len(kv) == 0 {
continue
}
if len(kv) == 1 && prevK != "" {
kvs[prevK] = strings.Join([]string{kvs[prevK], sanitizeValue(kv[0])}, KvSpaceSeparator)
continue
}
if len(kv[1]) == 0 {
return fmt.Errorf("value for key '%s' cannot be empty", kv[0])
}
prevK = kv[0]
kvs[kv[0]] = sanitizeValue(kv[1])
}
_, ok := targets[subSystemValue[0]]
if !ok {
targets[subSystemValue[0]] = map[string]KVS{}
}
if len(subSystemValue) == 2 {
targets[subSystemValue[0]][subSystemValue[1]] = kvs
} else {
targets[subSystemValue[0]][Default] = kvs
}
return nil
}
// parseSubSysTarget - parse sub-system target
func parseSubSysTarget(buf []byte) (Targets, error) {
targets := make(map[string]map[string]KVS)
bio := bufio.NewScanner(bytes.NewReader(buf))
for bio.Scan() {
if err := convertTargets(bio.Text(), targets); err != nil {
return nil, err
}
}
if err := bio.Err(); err != nil {
return nil, err
}
return targets, nil
}