maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add ReadFrom,WriteTo helpers for server config (#8580)

Browse source
This commit is contained in:
Harshavardhana 2019-11-27 09:36:08 -08:00 committed by GitHub
parent 87443af49e
commit 30e80d0a86
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 285 additions and 247 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

108
cmd/admin-handlers-config-kv.go
View file

@ -17,14 +17,12 @@
package cmd
import (
"bufio"
"bytes"
"context"
"encoding/json"
"io"
"net/http"
"strconv"
"strings"
"github.com/gorilla/mux"
"github.com/minio/minio/cmd/config"
@ -79,24 +77,14 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
return
}
cfg, err := readServerConfig(ctx, objectAPI)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
for scanner.Scan() {
// Skip any empty lines
if scanner.Text() == "" {
continue
}
if err = cfg.DelKVS(scanner.Text()); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
if err = scanner.Err(); err != nil {
if err = cfg.DelFrom(bytes.NewReader(kvBytes)); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
@ -135,31 +123,9 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
return
}
cfg, err := readServerConfig(ctx, objectAPI)
if err != nil {
// Config not found for some reason, allow things to continue
// by initializing a new fresh config in safe mode.
if err == errConfigNotFound && globalSafeMode {
cfg = newServerConfig()
err = nil
} else {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
for scanner.Scan() {
// Skip any empty lines, or comment like characters
if scanner.Text() == "" || strings.HasPrefix(scanner.Text(), config.KvComment) {
continue
}
if err = cfg.SetKVS(scanner.Text(), defaultKVS()); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
if err = scanner.Err(); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
@ -210,23 +176,10 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
vars := mux.Vars(r)
var buf = &bytes.Buffer{}
key := vars["key"]
if key != "" {
kvs, err := cfg.GetKVS(key, defaultKVS())
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
for k, kv := range kvs {
buf.WriteString(k)
buf.WriteString(config.KvSpaceSeparator)
buf.WriteString(kv.String())
if len(kvs) > 1 {
buf.WriteString(config.KvNewline)
}
}
} else {
buf.WriteString(cfg.String())
cw := config.NewConfigWriteTo(cfg, vars["key"])
if _, err := cw.WriteTo(buf); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
password := globalActiveCred.SecretKey
@ -297,30 +250,11 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
cfg, err := readServerConfig(ctx, objectAPI)
if err != nil {
// Config not found for some reason, allow things to continue
// by initializing a new fresh config in safe mode.
if err == errConfigNotFound && globalSafeMode {
cfg = newServerConfig()
err = nil
} else {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
for scanner.Scan() {
// Skip any empty lines, or comment like characters
if scanner.Text() == "" || strings.HasPrefix(scanner.Text(), config.KvComment) {
continue
}
if err = cfg.SetKVS(scanner.Text(), defaultKVS()); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
if err = scanner.Err(); err != nil {
if _, err = cfg.ReadFrom(bytes.NewReader(kvBytes)); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
@ -432,19 +366,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
}
cfg := newServerConfig()
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
for scanner.Scan() {
// Skip any empty lines, or comment like characters
if scanner.Text() == "" || strings.HasPrefix(scanner.Text(), config.KvComment) {
continue
}
if err = cfg.SetKVS(scanner.Text(), defaultKVS()); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
if err = scanner.Err(); err != nil {
if _, err = cfg.ReadFrom(bytes.NewReader(kvBytes)); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
@ -484,14 +406,18 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
return
}
config, err := readServerConfig(ctx, objectAPI)
cfg, err := readServerConfig(ctx, objectAPI)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
var buf = &bytes.Buffer{}
buf.WriteString(config.String())
cw := config.NewConfigWriteTo(cfg, "")
if _, err = cw.WriteTo(buf); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
password := globalActiveCred.SecretKey
econfigData, err := madmin.EncryptData(password, buf.Bytes())

313
cmd/config-current.go
View file

@ -39,6 +39,154 @@ import (
"github.com/minio/minio/pkg/env"
)
func init() {
var kvs = map[string]config.KVS{
config.EtcdSubSys: etcd.DefaultKVS,
config.CacheSubSys: cache.DefaultKVS,
config.CompressionSubSys: compress.DefaultKVS,
config.StorageClassSubSys: storageclass.DefaultKVS,
config.IdentityLDAPSubSys: xldap.DefaultKVS,
config.IdentityOpenIDSubSys: openid.DefaultKVS,
config.PolicyOPASubSys: opa.DefaultKVS,
config.RegionSubSys: config.DefaultRegionKVS,
config.CredentialsSubSys: config.DefaultCredentialKVS,
config.KmsVaultSubSys: crypto.DefaultKVS,
config.LoggerWebhookSubSys: logger.DefaultKVS,
config.AuditWebhookSubSys: logger.DefaultAuditKVS,
}
for k, v := range notify.DefaultNotificationKVS {
kvs[k] = v
}
config.RegisterDefaultKVS(kvs)
// Captures help for each sub-system
var helpSubSys = config.HelpKVS{
config.HelpKV{
Key: config.RegionSubSys,
Description: "Configure to describe the physical location of the server",
},
config.HelpKV{
Key: config.StorageClassSubSys,
Description: "Configure to control data and parity per object",
},
config.HelpKV{
Key: config.CacheSubSys,
Description: "Configure to enable edge caching",
},
config.HelpKV{
Key: config.CompressionSubSys,
Description: "Configure to enable streaming on disk compression",
},
config.HelpKV{
Key: config.EtcdSubSys,
Description: "Configure to enable 'etcd' configuration",
},
config.HelpKV{
Key: config.IdentityOpenIDSubSys,
Description: "Configure to enable OpenID SSO support",
},
config.HelpKV{
Key: config.IdentityLDAPSubSys,
Description: "Configure to enable LDAP SSO support",
},
config.HelpKV{
Key: config.PolicyOPASubSys,
Description: "Configure to enable external OPA policy support",
},
config.HelpKV{
Key: config.KmsVaultSubSys,
Description: "Configure to enable Vault based external KMS",
},
config.HelpKV{
Key: config.LoggerWebhookSubSys,
Description: "Configure to enable Webhook based logger",
MultipleTargets: true,
},
config.HelpKV{
Key: config.AuditWebhookSubSys,
Description: "Configure to enable Webhook based audit logger",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyWebhookSubSys,
Description: "Configure to publish events to Webhook target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyAMQPSubSys,
Description: "Configure to publish events to AMQP target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyKafkaSubSys,
Description: "Configure to publish events to Kafka target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyMQTTSubSys,
Description: "Configure to publish events to MQTT target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyNATSSubSys,
Description: "Configure to publish events to NATS target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyNSQSubSys,
Description: "Configure to publish events to NSQ target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyMySQLSubSys,
Description: "Configure to publish events to MySQL target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyPostgresSubSys,
Description: "Configure to publish events to Postgres target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyRedisSubSys,
Description: "Configure to publish events to Redis target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyESSubSys,
Description: "Configure to publish events to Elasticsearch target",
MultipleTargets: true,
},
}
var helpMap = map[string]config.HelpKVS{
"": helpSubSys, // Help for all sub-systems.
config.RegionSubSys: config.RegionHelp,
config.EtcdSubSys: etcd.Help,
config.CacheSubSys: cache.Help,
config.CompressionSubSys: compress.Help,
config.StorageClassSubSys: storageclass.Help,
config.IdentityOpenIDSubSys: openid.Help,
config.IdentityLDAPSubSys: xldap.Help,
config.PolicyOPASubSys: opa.Help,
config.KmsVaultSubSys: crypto.Help,
config.LoggerWebhookSubSys: logger.Help,
config.AuditWebhookSubSys: logger.HelpAudit,
config.NotifyAMQPSubSys: notify.HelpAMQP,
config.NotifyKafkaSubSys: notify.HelpKafka,
config.NotifyMQTTSubSys: notify.HelpMQTT,
config.NotifyNATSSubSys: notify.HelpNATS,
config.NotifyNSQSubSys: notify.HelpNSQ,
config.NotifyMySQLSubSys: notify.HelpMySQL,
config.NotifyPostgresSubSys: notify.HelpPostgres,
config.NotifyRedisSubSys: notify.HelpRedis,
config.NotifyWebhookSubSys: notify.HelpWebhook,
config.NotifyESSubSys: notify.HelpES,
}
config.RegisterHelpSubSys(helpMap)
}
var (
// globalServerConfig server config.
globalServerConfig config.Config
@ -257,152 +405,6 @@ func lookupConfigs(s config.Config) (err error) {
return nil
}
func defaultKVS() map[string]config.KVS {
var kvs = map[string]config.KVS{
config.EtcdSubSys: etcd.DefaultKVS,
config.CacheSubSys: cache.DefaultKVS,
config.CompressionSubSys: compress.DefaultKVS,
config.StorageClassSubSys: storageclass.DefaultKVS,
config.IdentityLDAPSubSys: xldap.DefaultKVS,
config.IdentityOpenIDSubSys: openid.DefaultKVS,
config.PolicyOPASubSys: opa.DefaultKVS,
config.RegionSubSys: config.DefaultRegionKVS,
config.CredentialsSubSys: config.DefaultCredentialKVS,
config.KmsVaultSubSys: crypto.DefaultKVS,
config.LoggerWebhookSubSys: logger.DefaultKVS,
config.AuditWebhookSubSys: logger.DefaultAuditKVS,
}
for k, v := range notify.DefaultNotificationKVS {
kvs[k] = v
}
return kvs
}
// Captures help for each sub-system
var helpSubSys = config.HelpKVS{
config.HelpKV{
Key: config.RegionSubSys,
Description: "Configure to describe the physical location of the server",
},
config.HelpKV{
Key: config.StorageClassSubSys,
Description: "Configure to control data and parity per object",
},
config.HelpKV{
Key: config.CacheSubSys,
Description: "Configure to enable edge caching",
},
config.HelpKV{
Key: config.CompressionSubSys,
Description: "Configure to enable streaming on disk compression",
},
config.HelpKV{
Key: config.EtcdSubSys,
Description: "Configure to enable 'etcd' configuration",
},
config.HelpKV{
Key: config.IdentityOpenIDSubSys,
Description: "Configure to enable OpenID SSO support",
},
config.HelpKV{
Key: config.IdentityLDAPSubSys,
Description: "Configure to enable LDAP SSO support",
},
config.HelpKV{
Key: config.PolicyOPASubSys,
Description: "Configure to enable external OPA policy support",
},
config.HelpKV{
Key: config.KmsVaultSubSys,
Description: "Configure to enable Vault based external KMS",
},
config.HelpKV{
Key: config.LoggerWebhookSubSys,
Description: "Configure to enable Webhook based logger",
MultipleTargets: true,
},
config.HelpKV{
Key: config.AuditWebhookSubSys,
Description: "Configure to enable Webhook based audit logger",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyWebhookSubSys,
Description: "Configure to publish events to Webhook target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyAMQPSubSys,
Description: "Configure to publish events to AMQP target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyKafkaSubSys,
Description: "Configure to publish events to Kafka target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyMQTTSubSys,
Description: "Configure to publish events to MQTT target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyNATSSubSys,
Description: "Configure to publish events to NATS target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyNSQSubSys,
Description: "Configure to publish events to NSQ target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyMySQLSubSys,
Description: "Configure to publish events to MySQL target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyPostgresSubSys,
Description: "Configure to publish events to Postgres target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyRedisSubSys,
Description: "Configure to publish events to Redis target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyESSubSys,
Description: "Configure to publish events to Elasticsearch target",
MultipleTargets: true,
},
}
var helpMap = map[string]config.HelpKVS{
config.RegionSubSys: config.RegionHelp,
config.EtcdSubSys: etcd.Help,
config.CacheSubSys: cache.Help,
config.CompressionSubSys: compress.Help,
config.StorageClassSubSys: storageclass.Help,
config.IdentityOpenIDSubSys: openid.Help,
config.IdentityLDAPSubSys: xldap.Help,
config.PolicyOPASubSys: opa.Help,
config.KmsVaultSubSys: crypto.Help,
config.LoggerWebhookSubSys: logger.Help,
config.AuditWebhookSubSys: logger.HelpAudit,
config.NotifyAMQPSubSys: notify.HelpAMQP,
config.NotifyKafkaSubSys: notify.HelpKafka,
config.NotifyMQTTSubSys: notify.HelpMQTT,
config.NotifyNATSSubSys: notify.HelpNATS,
config.NotifyNSQSubSys: notify.HelpNSQ,
config.NotifyMySQLSubSys: notify.HelpMySQL,
config.NotifyPostgresSubSys: notify.HelpPostgres,
config.NotifyRedisSubSys: notify.HelpRedis,
config.NotifyWebhookSubSys: notify.HelpWebhook,
config.NotifyESSubSys: notify.HelpES,
}
// Help - return sub-system level help
type Help struct {
SubSys string `json:"subSys"`
@ -414,18 +416,24 @@ type Help struct {
// GetHelp - returns help for sub-sys, a key for a sub-system or all the help.
func GetHelp(subSys, key string, envOnly bool) (Help, error) {
if len(subSys) == 0 {
return Help{KeysHelp: helpSubSys}, nil
return Help{KeysHelp: config.HelpSubSysMap[subSys]}, nil
}
subSystemValue := strings.SplitN(subSys, config.SubSystemSeparator, 2)
if len(subSystemValue) == 0 {
return Help{}, config.Errorf("invalid number of arguments %s", subSys)
}
if !config.SubSystems.Contains(subSystemValue[0]) {
subSys = subSystemValue[0]
subSysHelp, ok := config.HelpSubSysMap[""].Lookup(subSys)
if !ok {
return Help{}, config.Errorf("unknown sub-system %s", subSys)
}
h := helpMap[subSystemValue[0]]
h, ok := config.HelpSubSysMap[subSys]
if !ok {
return Help{}, config.Errorf("unknown sub-system %s", subSys)
}
if key != "" {
value, ok := h.Lookup(key)
if !ok {
@ -434,8 +442,6 @@ func GetHelp(subSys, key string, envOnly bool) (Help, error) {
h = config.HelpKVS{value}
}
subSys = subSystemValue[0]
envHelp := config.HelpKVS{}
if envOnly {
for _, hkv := range h {
@ -452,11 +458,6 @@ func GetHelp(subSys, key string, envOnly bool) (Help, error) {
h = envHelp
}
subSysHelp, ok := helpSubSys.Lookup(subSys)
if !ok {
return Help{}, config.Errorf("unknown sub-system %s", subSys)
}
return Help{
SubSys: subSys,
Description: subSysHelp.Description,

5
cmd/config.go
View file

@ -151,6 +151,11 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
configFile := path.Join(minioConfigPrefix, minioConfigFile)
configData, err := readConfig(ctx, objAPI, configFile)
if err != nil {
// Config not found for some reason, allow things to continue
// by initializing a new fresh config in safe mode.
if err == errConfigNotFound && globalSafeMode {
return newServerConfig(), nil
}
return nil, err
}

106
cmd/config/config.go
View file

@ -18,7 +18,9 @@
package config
import (
"bufio"
"fmt"
"io"
"regexp"
"strings"
@ -145,6 +147,35 @@ const (
EnvWordDelimiter = `_`
)
// DefaultKVS - default kvs for all sub-systems
var DefaultKVS map[string]KVS
// RegisterDefaultKVS - this function saves input kvsMap
// globally, this should be called only once preferably
// during `init()`.
func RegisterDefaultKVS(kvsMap map[string]KVS) {
DefaultKVS = map[string]KVS{}
for subSys, kvs := range kvsMap {
DefaultKVS[subSys] = kvs
}
}
// HelpSubSysMap - help for all individual KVS for each sub-systems
// also carries a special empty sub-system which dumps
// help for each sub-system key.
var HelpSubSysMap map[string]HelpKVS
// RegisterHelpSubSys - this function saves
// input help KVS for each sub-system globally,
// this function should be called only once
// preferably in during `init()`.
func RegisterHelpSubSys(helpKVSMap map[string]HelpKVS) {
HelpSubSysMap = map[string]HelpKVS{}
for subSys, hkvs := range helpKVSMap {
HelpSubSysMap[subSys] = hkvs
}
}
// KV - is a shorthand of each key value.
type KV struct {
Key string `json:"key"`
@ -216,6 +247,81 @@ func (c Config) String() string {
return s.String()
}
// DelFrom - deletes all keys in the input reader.
func (c Config) DelFrom(r io.Reader) error {
scanner := bufio.NewScanner(r)
for scanner.Scan() {
// Skip any empty lines, or comment like characters
if scanner.Text() == "" || strings.HasPrefix(scanner.Text(), KvComment) {
continue
}
if err := c.DelKVS(scanner.Text()); err != nil {
return err
}
}
if err := scanner.Err(); err != nil {
return err
}
return nil
}
// ReadFrom - implements io.ReaderFrom interface
func (c Config) ReadFrom(r io.Reader) (int64, error) {
var n int
scanner := bufio.NewScanner(r)
for scanner.Scan() {
// Skip any empty lines, or comment like characters
if scanner.Text() == "" || strings.HasPrefix(scanner.Text(), KvComment) {
continue
}
if err := c.SetKVS(scanner.Text(), DefaultKVS); err != nil {
return 0, err
}
n += len(scanner.Text())
}
if err := scanner.Err(); err != nil {
return 0, err
}
return int64(n), nil
}
type configWriteTo struct {
Config
filterByKey string
}
// NewConfigWriteTo - returns a struct which
// allows for serializing the config/kv struct
// to a io.WriterTo
func NewConfigWriteTo(cfg Config, key string) io.WriterTo {
return &configWriteTo{Config: cfg, filterByKey: key}
}
// WriteTo - implements io.WriterTo interface implementation for config.
func (c *configWriteTo) WriteTo(w io.Writer) (int64, error) {
if c.filterByKey == "" {
n, err := w.Write([]byte(c.String()))
return int64(n), err
}
kvs, err := c.GetKVS(c.filterByKey, DefaultKVS)
if err != nil {
return 0, err
}
var n int
for k, kv := range kvs {
m1, _ := w.Write([]byte(k))
m2, _ := w.Write([]byte(KvSpaceSeparator))
m3, _ := w.Write([]byte(kv.String()))
if len(kvs) > 1 {
m4, _ := w.Write([]byte(KvNewline))
n += m1 + m2 + m3 + m4
} else {
n += m1 + m2 + m3
}
}
return int64(n), nil
}
// Default KV configs for worm and region
var (
DefaultCredentialKVS = KVS{