Add deliveryMode parameter for AMQP notfication target (#4008)

Configuration migration was done.

Also adds documentation about AMQP configuration parameters.

Fixes #3982

cmd/config-migrate.go

@@ -89,6 +89,10 @@ func migrateConfig() error {
 	if err := migrateV16ToV17(); err != nil {
 		return err
 	}
+	// Migration version '17' to '18'.
+	if err := migrateV17ToV18(); err != nil {
+		return err
+	}
 
 	return nil
 }
@@ -1207,3 +1211,107 @@ func migrateV16ToV17() error {
 	log.Printf("Migration from version ‘%s’ to ‘%s’ completed successfully.\n", cv16.Version, srvConfig.Version)
 	return nil
 }
+
+// Version '17' to '18' migration. Adds "deliveryMode" configuration
+// parameter for AMQP notification target
+func migrateV17ToV18() error {
+	configFile := getConfigFile()
+
+	cv17 := &serverConfigV17{}
+	_, err := quick.Load(configFile, cv17)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config version ‘17’. %v", err)
+	}
+	if cv17.Version != "17" {
+		return nil
+	}
+
+	// Copy over fields from V17 into V18 config struct
+	srvConfig := &serverConfigV17{
+		Logger: &loggers{},
+		Notify: &notifier{},
+	}
+	srvConfig.Version = "18"
+	srvConfig.Credential = cv17.Credential
+	srvConfig.Region = cv17.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = globalMinioDefaultRegion
+	}
+
+	srvConfig.Logger.Console = cv17.Logger.Console
+	srvConfig.Logger.File = cv17.Logger.File
+
+	// check and set notifiers config
+	if len(cv17.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP = make(map[string]amqpNotify)
+		srvConfig.Notify.AMQP["1"] = amqpNotify{}
+	} else {
+		// New deliveryMode parameter is added for AMQP,
+		// default value is already 0, so nothing to
+		// explicitly migrate here.
+		srvConfig.Notify.AMQP = cv17.Notify.AMQP
+	}
+	if len(cv17.Notify.ElasticSearch) == 0 {
+		srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
+		srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.ElasticSearch = cv17.Notify.ElasticSearch
+	}
+	if len(cv17.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis = make(map[string]redisNotify)
+		srvConfig.Notify.Redis["1"] = redisNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.Redis = cv17.Notify.Redis
+	}
+	if len(cv17.Notify.PostgreSQL) == 0 {
+		srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
+		srvConfig.Notify.PostgreSQL["1"] = postgreSQLNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.PostgreSQL = cv17.Notify.PostgreSQL
+	}
+	if len(cv17.Notify.Kafka) == 0 {
+		srvConfig.Notify.Kafka = make(map[string]kafkaNotify)
+		srvConfig.Notify.Kafka["1"] = kafkaNotify{}
+	} else {
+		srvConfig.Notify.Kafka = cv17.Notify.Kafka
+	}
+	if len(cv17.Notify.NATS) == 0 {
+		srvConfig.Notify.NATS = make(map[string]natsNotify)
+		srvConfig.Notify.NATS["1"] = natsNotify{}
+	} else {
+		srvConfig.Notify.NATS = cv17.Notify.NATS
+	}
+	if len(cv17.Notify.Webhook) == 0 {
+		srvConfig.Notify.Webhook = make(map[string]webhookNotify)
+		srvConfig.Notify.Webhook["1"] = webhookNotify{}
+	} else {
+		srvConfig.Notify.Webhook = cv17.Notify.Webhook
+	}
+	if len(cv17.Notify.MySQL) == 0 {
+		srvConfig.Notify.MySQL = make(map[string]mySQLNotify)
+		srvConfig.Notify.MySQL["1"] = mySQLNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.MySQL = cv17.Notify.MySQL
+	}
+
+	// Load browser config from existing config in the file.
+	srvConfig.Browser = cv17.Browser
+
+	if err = quick.Save(configFile, srvConfig); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv17.Version, srvConfig.Version, err)
+	}
+
+	log.Printf("Migration from version ‘%s’ to ‘%s’ completed successfully.\n", cv17.Version, srvConfig.Version)
+	return nil
+}

cmd/config-migrate_test.go

@@ -118,11 +118,14 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 	if err := migrateV16ToV17(); err != nil {
 		t.Fatal("migrate v16 to v17 should succeed when no config file is found")
 	}
+	if err := migrateV17ToV18(); err != nil {
+		t.Fatal("migrate v17 to v18 should succeed when no config file is found")
+	}
 
 }
 
-// Test if a config migration from v2 to v17 is successfully done
+// Test if a config migration from v2 to v18 is successfully done
-func TestServerConfigMigrateV2toV16(t *testing.T) {
+func TestServerConfigMigrateV2toV18(t *testing.T) {
 	rootPath, err := newTestConfig(globalMinioDefaultRegion)
 	if err != nil {
 		t.Fatalf("Init Test config failed")
@@ -161,7 +164,7 @@ func TestServerConfigMigrateV2toV16(t *testing.T) {
 	}
 
 	// Check the version number in the upgraded config file
-	expectedVersion := v17
+	expectedVersion := v18
 	if serverConfig.Version != expectedVersion {
 		t.Fatalf("Expect version "+expectedVersion+", found: %v", serverConfig.Version)
 	}
@@ -238,4 +241,7 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 	if err := migrateV16ToV17(); err == nil {
 		t.Fatal("migrateConfigV16ToV17() should fail with a corrupted json")
 	}
+	if err := migrateV17ToV18(); err == nil {
+		t.Fatal("migrateConfigV17ToV18() should fail with a corrupted json")
+	}
 }

cmd/config-old.go

@@ -430,3 +430,22 @@ type serverConfigV16 struct {
 	// Notification queue configuration.
 	Notify *notifier `json:"notify"`
 }
+
+// serverConfigV17 server configuration version '17' which is like
+// version '16' except it adds support for "format" parameter in
+// database event notification targets: PostgreSQL, MySQL, Redis and
+// Elasticsearch.
+type serverConfigV17 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential  `json:"credential"`
+	Region     string      `json:"region"`
+	Browser    BrowserFlag `json:"browser"`
+
+	// Additional error logging configuration.
+	Logger *loggers `json:"logger"`
+
+	// Notification queue configuration.
+	Notify *notifier `json:"notify"`
+}

cmd/config-v18.go

@@ -27,19 +27,18 @@ import (
 )
 
 // Config version
-const v17 = "17"
+const v18 = "18"
 
 var (
 	// serverConfig server config.
-	serverConfig   *serverConfigV17
+	serverConfig   *serverConfigV18
 	serverConfigMu sync.RWMutex
 )
 
-// serverConfigV17 server configuration version '17' which is like
+// serverConfigV18 server configuration version '18' which is like
-// version '16' except it adds support for "format" parameter in
+// version '17' except it adds support for "deliveryMode" parameter in
-// database event notification targets: PostgreSQL, MySQL, Redis and
+// the AMQP notification target.
-// Elasticsearch.
+type serverConfigV18 struct {
-type serverConfigV17 struct {
 	sync.RWMutex
 	Version string `json:"version"`
 
@@ -56,7 +55,7 @@ type serverConfigV17 struct {
 }
 
 // GetVersion get current config version.
-func (s *serverConfigV17) GetVersion() string {
+func (s *serverConfigV18) GetVersion() string {
 	s.RLock()
 	defer s.RUnlock()
 
@@ -64,7 +63,7 @@ func (s *serverConfigV17) GetVersion() string {
 }
 
 // SetRegion set new region.
-func (s *serverConfigV17) SetRegion(region string) {
+func (s *serverConfigV18) SetRegion(region string) {
 	s.Lock()
 	defer s.Unlock()
 
@@ -72,7 +71,7 @@ func (s *serverConfigV17) SetRegion(region string) {
 }
 
 // GetRegion get current region.
-func (s *serverConfigV17) GetRegion() string {
+func (s *serverConfigV18) GetRegion() string {
 	s.RLock()
 	defer s.RUnlock()
 
@@ -80,7 +79,7 @@ func (s *serverConfigV17) GetRegion() string {
 }
 
 // SetCredentials set new credentials.
-func (s *serverConfigV17) SetCredential(creds credential) {
+func (s *serverConfigV18) SetCredential(creds credential) {
 	s.Lock()
 	defer s.Unlock()
 
@@ -89,7 +88,7 @@ func (s *serverConfigV17) SetCredential(creds credential) {
 }
 
 // GetCredentials get current credentials.
-func (s *serverConfigV17) GetCredential() credential {
+func (s *serverConfigV18) GetCredential() credential {
 	s.RLock()
 	defer s.RUnlock()
 
@@ -97,7 +96,7 @@ func (s *serverConfigV17) GetCredential() credential {
 }
 
 // SetBrowser set if browser is enabled.
-func (s *serverConfigV17) SetBrowser(b bool) {
+func (s *serverConfigV18) SetBrowser(b bool) {
 	s.Lock()
 	defer s.Unlock()
 
@@ -106,7 +105,7 @@ func (s *serverConfigV17) SetBrowser(b bool) {
 }
 
 // GetCredentials get current credentials.
-func (s *serverConfigV17) GetBrowser() bool {
+func (s *serverConfigV18) GetBrowser() bool {
 	s.RLock()
 	defer s.RUnlock()
 
@@ -114,7 +113,7 @@ func (s *serverConfigV17) GetBrowser() bool {
 }
 
 // Save config.
-func (s *serverConfigV17) Save() error {
+func (s *serverConfigV18) Save() error {
 	s.RLock()
 	defer s.RUnlock()
 
@@ -122,9 +121,9 @@ func (s *serverConfigV17) Save() error {
 	return quick.Save(getConfigFile(), s)
 }
 
-func newServerConfigV17() *serverConfigV17 {
+func newServerConfigV18() *serverConfigV18 {
-	srvCfg := &serverConfigV17{
+	srvCfg := &serverConfigV18{
-		Version:    v17,
+		Version:    v18,
 		Credential: mustGetNewCredential(),
 		Region:     globalMinioDefaultRegion,
 		Browser:    true,
@@ -160,12 +159,13 @@ func newServerConfigV17() *serverConfigV17 {
 // found, otherwise use default parameters
 func newConfig() error {
 	// Initialize server config.
-	srvCfg := newServerConfigV17()
+	srvCfg := newServerConfigV18()
 
 	// If env is set override the credentials from config file.
 	if globalIsEnvCreds {
 		srvCfg.SetCredential(globalActiveCred)
 	}
+
 	if globalIsEnvBrowser {
 		srvCfg.SetBrowser(globalIsBrowserEnabled)
 	}
@@ -233,8 +233,8 @@ func checkDupJSONKeys(json string) error {
 }
 
 // getValidConfig - returns valid server configuration
-func getValidConfig() (*serverConfigV17, error) {
+func getValidConfig() (*serverConfigV18, error) {
-	srvCfg := &serverConfigV17{
+	srvCfg := &serverConfigV18{
 		Region:  globalMinioDefaultRegion,
 		Browser: true,
 	}
@@ -244,8 +244,8 @@ func getValidConfig() (*serverConfigV17, error) {
 		return nil, err
 	}
 
-	if srvCfg.Version != v17 {
+	if srvCfg.Version != v18 {
-		return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", v17, srvCfg.Version)
+		return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", v18, srvCfg.Version)
 	}
 
 	// Load config file json and check for duplication json keys

cmd/config-v18_test.go

@@ -109,8 +109,8 @@ func TestServerConfig(t *testing.T) {
 	serverConfig.Logger.SetFile(fileLogger)
 
 	// Match version.
-	if serverConfig.GetVersion() != v17 {
+	if serverConfig.GetVersion() != v18 {
-		t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v17)
+		t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v18)
 	}
 
 	// Attempt to save.
@@ -217,7 +217,7 @@ func TestValidateConfig(t *testing.T) {
 
 	configPath := filepath.Join(rootPath, minioConfigFile)
 
-	v := v17
+	v := v18
 
 	testCases := []struct {
 		configData string

cmd/gateway-main.go

@@ -96,7 +96,7 @@ func newGatewayLayer(backendType, accessKey, secretKey string) (GatewayLayer, er
 // only used in memory.
 func newGatewayConfig(accessKey, secretKey, region string) error {
 	// Initialize server config.
-	srvCfg := newServerConfigV17()
+	srvCfg := newServerConfigV18()
 
 	// If env is set for a fresh start, save them to config file.
 	srvCfg.SetCredential(credential{

cmd/notify-amqp.go

@@ -32,6 +32,7 @@ type amqpNotify struct {
 	Exchange     string `json:"exchange"`
 	RoutingKey   string `json:"routingKey"`
 	ExchangeType string `json:"exchangeType"`
+	DeliveryMode uint8  `json:"deliveryMode"`
 	Mandatory    bool   `json:"mandatory"`
 	Immediate    bool   `json:"immediate"`
 	Durable      bool   `json:"durable"`
@@ -146,6 +147,7 @@ func (q amqpConn) Fire(entry *logrus.Entry) error {
 		q.params.Immediate,
 		amqp.Publishing{
 			ContentType:  "application/json",
+			DeliveryMode: q.params.DeliveryMode,
 			Body:         []byte(body),
 		})
 	if err != nil {

docs/bucket/notifications/README.md

@@ -26,7 +26,24 @@ Install RabbitMQ from [here](https://www.rabbitmq.com/).
 
 ### Step 1: Add AMQP endpoint to Minio
 
-The default location of Minio server configuration file is ``~/.minio/config.json``. Update the AMQP configuration block in ``config.json`` as follows:
+The default location of Minio server configuration file is ``~/.minio/config.json``. The AMQP configuration is located in the `amqp` key under the `notify` top-level key. Create a configuration key-value pair here for your AMQP instance. The key is a name for your AMQP endpoint, and the value is a collection of key-value parameters described in the table below.
+
+| Parameter | Type | Description |
+|:---|:---|:---|
+| `enable` | _bool_ | (Required) Is this server endpoint configuration active/enabled? |
+| `url` | _string_ | (Required) AMQP server endpoint, e.g. `amqp://myuser:mypassword@localhost:5672` |
+| `exchange` | _string_ | Name of the exchange. |
+| `routingKey` | _string_ | Routing key for publishing. |
+| `exchangeType` | _string_ | Kind of exchange. |
+| `deliveryMode` | _uint8_ | Delivery mode for publishing. 0 or 1 - transient; 2 - persistent. |
+| `mandatory` | _bool_ | Publishing related bool. |
+| `immediate` | _bool_ | Publishing related bool. |
+| `durable` | _bool_ | Exchange declaration related bool. |
+| `internal` | _bool_ | Exchange declaration related bool. |
+| `noWait` | _bool_ | Exchange declaration related bool. |
+| `autoDeleted` | _bool_ | Exchange declaration related bool. |
+
+An example configuration for RabbitMQ is shown below:
 
 ```json
 "amqp": {
@@ -36,6 +53,7 @@ The default location of Minio server configuration file is ``~/.minio/config.jso
         "exchange": "bucketevents",
         "routingKey": "bucketlogs",
         "exchangeType": "fanout",
+        "deliveryMode": 0,
         "mandatory": false,
         "immediate": false,
         "durable": false,
@@ -46,7 +64,12 @@ The default location of Minio server configuration file is ``~/.minio/config.jso
 }
 ```
 
-Restart Minio server to reflect config changes. Minio supports all the exchanges available in [RabbitMQ](https://www.rabbitmq.com/). For this setup, we are using ``fanout`` exchange.
+After updating the configuration file, restart the Minio server to put the changes into effect. The server will print a line like `SQS ARNs:  arn:minio:sqs:us-east-1:1:amqp` at start-up if there were no errors.
+
+Minio supports all the exchanges available in [RabbitMQ](https://www.rabbitmq.com/). For this setup, we are using ``fanout`` exchange.
+
+Note that, you can add as many AMQP server endpoint configurations as needed by providing an identifier (like "1" in the example above) for the AMQP instance and an object of per-server configuration parameters.
+
 
 ### Step 2: Enable bucket notification using Minio client
 

docs/config/README.md

@@ -1,4 +1,4 @@
-# Minio Server `config.json` (v17) Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
+# Minio Server `config.json` (v18) Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
 
 Minio server stores all its configuration data in `${HOME}/.minio/config.json` file by default. Following sections provide detailed explanation of each fields and how to customize them. A complete example of `config.json` is available [here](https://raw.githubusercontent.com/minio/minio/master/docs/config/config.sample.json)
 

docs/config/config.sample.json

@@ -23,6 +23,7 @@
                 "exchange": "bucketevents",
                 "routingKey": "bucketlogs",
                 "exchangeType": "fanout",
+                "deliveryMode": 0,
                 "mandatory": false,
                 "immediate": false,
                 "durable": false,