maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add deliveryMode parameter for AMQP notfication target (#4008)

Browse source 
Configuration migration was done.

Also adds documentation about AMQP configuration parameters.

Fixes #3982
This commit is contained in:
Aditya Manthramurthy 2017-03-31 16:04:26 +05:30 committed by Harshavardhana
parent 5cec6bd80d
commit 096427f973
10 changed files with 205 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

108
cmd/config-migrate.go
View file

@ -89,6 +89,10 @@ func migrateConfig() error {
if err := migrateV16ToV17(); err != nil {
return err
}
// Migration version '17' to '18'.
if err := migrateV17ToV18(); err != nil {
return err
}
return nil
}
@ -1207,3 +1211,107 @@ func migrateV16ToV17() error {
log.Printf("Migration from version ‘%s’ to ‘%s’ completed successfully.\n", cv16.Version, srvConfig.Version)
return nil
}
// Version '17' to '18' migration. Adds "deliveryMode" configuration
// parameter for AMQP notification target
func migrateV17ToV18() error {
configFile := getConfigFile()
cv17 := &serverConfigV17{}
_, err := quick.Load(configFile, cv17)
if os.IsNotExist(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘17’. %v", err)
}
if cv17.Version != "17" {
return nil
}
// Copy over fields from V17 into V18 config struct
srvConfig := &serverConfigV17{
Logger: &loggers{},
Notify: &notifier{},
}
srvConfig.Version = "18"
srvConfig.Credential = cv17.Credential
srvConfig.Region = cv17.Region
if srvConfig.Region == "" {
// Region needs to be set for AWS Signature Version 4.
srvConfig.Region = globalMinioDefaultRegion
}
srvConfig.Logger.Console = cv17.Logger.Console
srvConfig.Logger.File = cv17.Logger.File
// check and set notifiers config
if len(cv17.Notify.AMQP) == 0 {
srvConfig.Notify.AMQP = make(map[string]amqpNotify)
srvConfig.Notify.AMQP["1"] = amqpNotify{}
} else {
// New deliveryMode parameter is added for AMQP,
// default value is already 0, so nothing to
// explicitly migrate here.
srvConfig.Notify.AMQP = cv17.Notify.AMQP
}
if len(cv17.Notify.ElasticSearch) == 0 {
srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{
Format: formatNamespace,
}
} else {
srvConfig.Notify.ElasticSearch = cv17.Notify.ElasticSearch
}
if len(cv17.Notify.Redis) == 0 {
srvConfig.Notify.Redis = make(map[string]redisNotify)
srvConfig.Notify.Redis["1"] = redisNotify{
Format: formatNamespace,
}
} else {
srvConfig.Notify.Redis = cv17.Notify.Redis
}
if len(cv17.Notify.PostgreSQL) == 0 {
srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
srvConfig.Notify.PostgreSQL["1"] = postgreSQLNotify{
Format: formatNamespace,
}
} else {
srvConfig.Notify.PostgreSQL = cv17.Notify.PostgreSQL
}
if len(cv17.Notify.Kafka) == 0 {
srvConfig.Notify.Kafka = make(map[string]kafkaNotify)
srvConfig.Notify.Kafka["1"] = kafkaNotify{}
} else {
srvConfig.Notify.Kafka = cv17.Notify.Kafka
}
if len(cv17.Notify.NATS) == 0 {
srvConfig.Notify.NATS = make(map[string]natsNotify)
srvConfig.Notify.NATS["1"] = natsNotify{}
} else {
srvConfig.Notify.NATS = cv17.Notify.NATS
}
if len(cv17.Notify.Webhook) == 0 {
srvConfig.Notify.Webhook = make(map[string]webhookNotify)
srvConfig.Notify.Webhook["1"] = webhookNotify{}
} else {
srvConfig.Notify.Webhook = cv17.Notify.Webhook
}
if len(cv17.Notify.MySQL) == 0 {
srvConfig.Notify.MySQL = make(map[string]mySQLNotify)
srvConfig.Notify.MySQL["1"] = mySQLNotify{
Format: formatNamespace,
}
} else {
srvConfig.Notify.MySQL = cv17.Notify.MySQL
}
// Load browser config from existing config in the file.
srvConfig.Browser = cv17.Browser
if err = quick.Save(configFile, srvConfig); err != nil {
return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv17.Version, srvConfig.Version, err)
}
log.Printf("Migration from version ‘%s’ to ‘%s’ completed successfully.\n", cv17.Version, srvConfig.Version)
return nil
}

12
cmd/config-migrate_test.go
View file

@ -118,11 +118,14 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
if err := migrateV16ToV17(); err != nil {
t.Fatal("migrate v16 to v17 should succeed when no config file is found")
}
if err := migrateV17ToV18(); err != nil {
t.Fatal("migrate v17 to v18 should succeed when no config file is found")
}
}
// Test if a config migration from v2 to v17 is successfully done
func TestServerConfigMigrateV2toV16(t *testing.T) {
// Test if a config migration from v2 to v18 is successfully done
func TestServerConfigMigrateV2toV18(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatalf("Init Test config failed")
@ -161,7 +164,7 @@ func TestServerConfigMigrateV2toV16(t *testing.T) {
}
// Check the version number in the upgraded config file
expectedVersion := v17
expectedVersion := v18
if serverConfig.Version != expectedVersion {
t.Fatalf("Expect version "+expectedVersion+", found: %v", serverConfig.Version)
}
@ -238,4 +241,7 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
if err := migrateV16ToV17(); err == nil {
t.Fatal("migrateConfigV16ToV17() should fail with a corrupted json")
}
if err := migrateV17ToV18(); err == nil {
t.Fatal("migrateConfigV17ToV18() should fail with a corrupted json")
}
}

19
cmd/config-old.go
View file

@ -430,3 +430,22 @@ type serverConfigV16 struct {
// Notification queue configuration.
Notify *notifier `json:"notify"`
}
// serverConfigV17 server configuration version '17' which is like
// version '16' except it adds support for "format" parameter in
// database event notification targets: PostgreSQL, MySQL, Redis and
// Elasticsearch.
type serverConfigV17 struct {
Version string `json:"version"`
// S3 API configuration.
Credential credential `json:"credential"`
Region string `json:"region"`
Browser BrowserFlag `json:"browser"`
// Additional error logging configuration.
Logger *loggers `json:"logger"`
// Notification queue configuration.
Notify *notifier `json:"notify"`
}

46
cmd/config-v17.go → cmd/config-v18.go
View file

@ -27,19 +27,18 @@ import (
)
// Config version
const v17 = "17"
const v18 = "18"
var (
// serverConfig server config.
serverConfig *serverConfigV17
serverConfig *serverConfigV18
serverConfigMu sync.RWMutex
)
// serverConfigV17 server configuration version '17' which is like
// version '16' except it adds support for "format" parameter in
// database event notification targets: PostgreSQL, MySQL, Redis and
// Elasticsearch.
type serverConfigV17 struct {
// serverConfigV18 server configuration version '18' which is like
// version '17' except it adds support for "deliveryMode" parameter in
// the AMQP notification target.
type serverConfigV18 struct {
sync.RWMutex
Version string `json:"version"`
@ -56,7 +55,7 @@ type serverConfigV17 struct {
}
// GetVersion get current config version.
func (s *serverConfigV17) GetVersion() string {
func (s *serverConfigV18) GetVersion() string {
s.RLock()
defer s.RUnlock()
@ -64,7 +63,7 @@ func (s *serverConfigV17) GetVersion() string {
}
// SetRegion set new region.
func (s *serverConfigV17) SetRegion(region string) {
func (s *serverConfigV18) SetRegion(region string) {
s.Lock()
defer s.Unlock()
@ -72,7 +71,7 @@ func (s *serverConfigV17) SetRegion(region string) {
}
// GetRegion get current region.
func (s *serverConfigV17) GetRegion() string {
func (s *serverConfigV18) GetRegion() string {
s.RLock()
defer s.RUnlock()
@ -80,7 +79,7 @@ func (s *serverConfigV17) GetRegion() string {
}
// SetCredentials set new credentials.
func (s *serverConfigV17) SetCredential(creds credential) {
func (s *serverConfigV18) SetCredential(creds credential) {
s.Lock()
defer s.Unlock()
@ -89,7 +88,7 @@ func (s *serverConfigV17) SetCredential(creds credential) {
}
// GetCredentials get current credentials.
func (s *serverConfigV17) GetCredential() credential {
func (s *serverConfigV18) GetCredential() credential {
s.RLock()
defer s.RUnlock()
@ -97,7 +96,7 @@ func (s *serverConfigV17) GetCredential() credential {
}
// SetBrowser set if browser is enabled.
func (s *serverConfigV17) SetBrowser(b bool) {
func (s *serverConfigV18) SetBrowser(b bool) {
s.Lock()
defer s.Unlock()
@ -106,7 +105,7 @@ func (s *serverConfigV17) SetBrowser(b bool) {
}
// GetCredentials get current credentials.
func (s *serverConfigV17) GetBrowser() bool {
func (s *serverConfigV18) GetBrowser() bool {
s.RLock()
defer s.RUnlock()
@ -114,7 +113,7 @@ func (s *serverConfigV17) GetBrowser() bool {
}
// Save config.
func (s *serverConfigV17) Save() error {
func (s *serverConfigV18) Save() error {
s.RLock()
defer s.RUnlock()
@ -122,9 +121,9 @@ func (s *serverConfigV17) Save() error {
return quick.Save(getConfigFile(), s)
}
func newServerConfigV17() *serverConfigV17 {
srvCfg := &serverConfigV17{
Version: v17,
func newServerConfigV18() *serverConfigV18 {
srvCfg := &serverConfigV18{
Version: v18,
Credential: mustGetNewCredential(),
Region: globalMinioDefaultRegion,
Browser: true,
@ -160,12 +159,13 @@ func newServerConfigV17() *serverConfigV17 {
// found, otherwise use default parameters
func newConfig() error {
// Initialize server config.
srvCfg := newServerConfigV17()
srvCfg := newServerConfigV18()
// If env is set override the credentials from config file.
if globalIsEnvCreds {
srvCfg.SetCredential(globalActiveCred)
}
if globalIsEnvBrowser {
srvCfg.SetBrowser(globalIsBrowserEnabled)
}
@ -233,8 +233,8 @@ func checkDupJSONKeys(json string) error {
}
// getValidConfig - returns valid server configuration
func getValidConfig() (*serverConfigV17, error) {
srvCfg := &serverConfigV17{
func getValidConfig() (*serverConfigV18, error) {
srvCfg := &serverConfigV18{
Region: globalMinioDefaultRegion,
Browser: true,
}
@ -244,8 +244,8 @@ func getValidConfig() (*serverConfigV17, error) {
return nil, err
}
if srvCfg.Version != v17 {
return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", v17, srvCfg.Version)
if srvCfg.Version != v18 {
return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", v18, srvCfg.Version)
}
// Load config file json and check for duplication json keys

6
cmd/config-v17_test.go → cmd/config-v18_test.go
View file

@ -109,8 +109,8 @@ func TestServerConfig(t *testing.T) {
serverConfig.Logger.SetFile(fileLogger)
// Match version.
if serverConfig.GetVersion() != v17 {
t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v17)
if serverConfig.GetVersion() != v18 {
t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v18)
}
// Attempt to save.
@ -217,7 +217,7 @@ func TestValidateConfig(t *testing.T) {
configPath := filepath.Join(rootPath, minioConfigFile)
v := v17
v := v18
testCases := []struct {
configData string

2
cmd/gateway-main.go
View file

@ -96,7 +96,7 @@ func newGatewayLayer(backendType, accessKey, secretKey string) (GatewayLayer, er
// only used in memory.
func newGatewayConfig(accessKey, secretKey, region string) error {
// Initialize server config.
srvCfg := newServerConfigV17()
srvCfg := newServerConfigV18()
// If env is set for a fresh start, save them to config file.
srvCfg.SetCredential(credential{

6
cmd/notify-amqp.go
View file

@ -32,6 +32,7 @@ type amqpNotify struct {
Exchange string `json:"exchange"`
RoutingKey string `json:"routingKey"`
ExchangeType string `json:"exchangeType"`
DeliveryMode uint8 `json:"deliveryMode"`
Mandatory bool `json:"mandatory"`
Immediate bool `json:"immediate"`
Durable bool `json:"durable"`
@ -145,8 +146,9 @@ func (q amqpConn) Fire(entry *logrus.Entry) error {
q.params.Mandatory,
q.params.Immediate,
amqp.Publishing{
ContentType: "application/json",
Body: []byte(body),
ContentType: "application/json",
DeliveryMode: q.params.DeliveryMode,
Body: []byte(body),
})
if err != nil {
return err

49
docs/bucket/notifications/README.md
View file

@ -26,27 +26,50 @@ Install RabbitMQ from [here](https://www.rabbitmq.com/).
### Step 1: Add AMQP endpoint to Minio
The default location of Minio server configuration file is ``~/.minio/config.json``. Update the AMQP configuration block in ``config.json`` as follows:
The default location of Minio server configuration file is ``~/.minio/config.json``. The AMQP configuration is located in the `amqp` key under the `notify` top-level key. Create a configuration key-value pair here for your AMQP instance. The key is a name for your AMQP endpoint, and the value is a collection of key-value parameters described in the table below.
| Parameter | Type | Description |
|:---|:---|:---|
| `enable` | _bool_ | (Required) Is this server endpoint configuration active/enabled? |
| `url` | _string_ | (Required) AMQP server endpoint, e.g. `amqp://myuser:mypassword@localhost:5672` |
| `exchange` | _string_ | Name of the exchange. |
| `routingKey` | _string_ | Routing key for publishing. |
| `exchangeType` | _string_ | Kind of exchange. |
| `deliveryMode` | _uint8_ | Delivery mode for publishing. 0 or 1 - transient; 2 - persistent. |
| `mandatory` | _bool_ | Publishing related bool. |
| `immediate` | _bool_ | Publishing related bool. |
| `durable` | _bool_ | Exchange declaration related bool. |
| `internal` | _bool_ | Exchange declaration related bool. |
| `noWait` | _bool_ | Exchange declaration related bool. |
| `autoDeleted` | _bool_ | Exchange declaration related bool. |
An example configuration for RabbitMQ is shown below:
```json
"amqp": {
"1": {
"enable": true,
"url": "amqp://myuser:mypassword@localhost:5672",
"exchange": "bucketevents",
"routingKey": "bucketlogs",
"exchangeType": "fanout",
"mandatory": false,
"immediate": false,
"durable": false,
"internal": false,
"noWait": false,
"autoDeleted": false
"enable": true,
"url": "amqp://myuser:mypassword@localhost:5672",
"exchange": "bucketevents",
"routingKey": "bucketlogs",
"exchangeType": "fanout",
"deliveryMode": 0,
"mandatory": false,
"immediate": false,
"durable": false,
"internal": false,
"noWait": false,
"autoDeleted": false
}
}
```
Restart Minio server to reflect config changes. Minio supports all the exchanges available in [RabbitMQ](https://www.rabbitmq.com/). For this setup, we are using ``fanout`` exchange.
After updating the configuration file, restart the Minio server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs:us-east-1:1:amqp` at start-up if there were no errors.
Minio supports all the exchanges available in [RabbitMQ](https://www.rabbitmq.com/). For this setup, we are using ``fanout`` exchange.
Note that, you can add as many AMQP server endpoint configurations as needed by providing an identifier (like "1" in the example above) for the AMQP instance and an object of per-server configuration parameters.
### Step 2: Enable bucket notification using Minio client

2
docs/config/README.md
View file

@ -1,4 +1,4 @@
# Minio Server `config.json` (v17) Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
# Minio Server `config.json` (v18) Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
Minio server stores all its configuration data in `${HOME}/.minio/config.json` file by default. Following sections provide detailed explanation of each fields and how to customize them. A complete example of `config.json` is available [here](https://raw.githubusercontent.com/minio/minio/master/docs/config/config.sample.json)

1
docs/config/config.sample.json
View file

@ -23,6 +23,7 @@
"exchange": "bucketevents",
"routingKey": "bucketlogs",
"exchangeType": "fanout",
"deliveryMode": 0,
"mandatory": false,
"immediate": false,
"durable": false,