maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Set the maximum open connections limit in PG and MySQL target configs (#10558)

Browse source 
As the bulk/recursive delete will require multiple connections to open at an instance,
The default open connections limit will be reached which results in the following error

```FATAL:  sorry, too many clients already```

By setting the open connections to a reasonable value - `2`, We ensure that the max open connections
will not be exhausted and lie under bounds.

The queries are simple inserts/updates/deletes which is operational and sufficient with the
the maximum open connection limit is 2.

Fixes #10553

Allow user configuration for MaxOpenConnections
This commit is contained in:
Praveen raj Mani 2020-09-25 10:50:30 +05:30 committed by GitHub
parent 37a5d5d7a0
commit b880796aef
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 198 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
cmd/config/notify/help.go
View file

@ -340,6 +340,12 @@ var (
Optional: true, Optional: true,
Type: "sentence", Type: "sentence",
}, },
config.HelpKV{
Key: target.PostgresMaxOpenConnections,
Description: "To set the maximum number of open connections to the database. The value is set to `2` by default.",
Optional: true,
Type: "number",
},
} }
HelpMySQL = config.HelpKVS{ HelpMySQL = config.HelpKVS{
@ -377,6 +383,12 @@ var (
Optional: true, Optional: true,
Type: "sentence", Type: "sentence",
}, },
config.HelpKV{
Key: target.MySQLMaxOpenConnections,
Description: "To set the maximum number of open connections to the database. The value is set to `2` by default.",
Optional: true,
Type: "number",
},
} }
HelpNATS = config.HelpKVS{ HelpNATS = config.HelpKVS{

8
cmd/config/notify/legacy.go
View file

@ -357,6 +357,10 @@ func SetNotifyPostgres(s config.Config, psqName string, cfg target.PostgreSQLArg
Key: target.PostgresQueueLimit, Key: target.PostgresQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)), Value: strconv.Itoa(int(cfg.QueueLimit)),
}, },
config.KV{
Key: target.PostgresMaxOpenConnections,
Value: strconv.Itoa(cfg.MaxOpenConnections),
},
} }
return nil return nil
@ -554,6 +558,10 @@ func SetNotifyMySQL(s config.Config, sqlName string, cfg target.MySQLArgs) error
Key: target.MySQLQueueLimit, Key: target.MySQLQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)), Value: strconv.Itoa(int(cfg.QueueLimit)),
}, },
config.KV{
Key: target.MySQLMaxOpenConnections,
Value: strconv.Itoa(cfg.MaxOpenConnections),
},
} }
return nil return nil

55
cmd/config/notify/parse.go
View file

@ -807,6 +807,10 @@ var (
Key: target.MySQLQueueLimit, Key: target.MySQLQueueLimit,
Value: "0", Value: "0",
}, },
config.KV{
Key: target.MySQLMaxOpenConnections,
Value: "2",
},
} }
) )
@ -855,13 +859,25 @@ func GetNotifyMySQL(mysqlKVS map[string]config.KVS) (map[string]target.MySQLArgs
if k != config.Default { if k != config.Default {
queueDirEnv = queueDirEnv + config.Default + k queueDirEnv = queueDirEnv + config.Default + k
} }
maxOpenConnectionsEnv := target.EnvMySQLMaxOpenConnections
if k != config.Default {
maxOpenConnectionsEnv = maxOpenConnectionsEnv + config.Default + k
}
maxOpenConnections, cErr := strconv.Atoi(env.Get(maxOpenConnectionsEnv, kv.Get(target.MySQLMaxOpenConnections)))
if cErr != nil {
return nil, cErr
}
mysqlArgs := target.MySQLArgs{ mysqlArgs := target.MySQLArgs{
Enable: enabled, Enable: enabled,
Format: env.Get(formatEnv, kv.Get(target.MySQLFormat)), Format: env.Get(formatEnv, kv.Get(target.MySQLFormat)),
DSN: env.Get(dsnStringEnv, kv.Get(target.MySQLDSNString)), DSN: env.Get(dsnStringEnv, kv.Get(target.MySQLDSNString)),
Table: env.Get(tableEnv, kv.Get(target.MySQLTable)), Table: env.Get(tableEnv, kv.Get(target.MySQLTable)),
QueueDir: env.Get(queueDirEnv, kv.Get(target.MySQLQueueDir)), QueueDir: env.Get(queueDirEnv, kv.Get(target.MySQLQueueDir)),
QueueLimit: queueLimit, QueueLimit: queueLimit,
MaxOpenConnections: maxOpenConnections,
} }
if err = mysqlArgs.Validate(); err != nil { if err = mysqlArgs.Validate(); err != nil {
return nil, err return nil, err
@ -1235,6 +1251,10 @@ var (
Key: target.PostgresQueueLimit, Key: target.PostgresQueueLimit,
Value: "0", Value: "0",
}, },
config.KV{
Key: target.PostgresMaxOpenConnections,
Value: "2",
},
} }
) )
@ -1285,13 +1305,24 @@ func GetNotifyPostgres(postgresKVS map[string]config.KVS) (map[string]target.Pos
queueDirEnv = queueDirEnv + config.Default + k queueDirEnv = queueDirEnv + config.Default + k
} }
maxOpenConnectionsEnv := target.EnvPostgresMaxOpenConnections
if k != config.Default {
maxOpenConnectionsEnv = maxOpenConnectionsEnv + config.Default + k
}
maxOpenConnections, cErr := strconv.Atoi(env.Get(maxOpenConnectionsEnv, kv.Get(target.PostgresMaxOpenConnections)))
if cErr != nil {
return nil, cErr
}
psqlArgs := target.PostgreSQLArgs{ psqlArgs := target.PostgreSQLArgs{
Enable: enabled, Enable: enabled,
Format: env.Get(formatEnv, kv.Get(target.PostgresFormat)), Format: env.Get(formatEnv, kv.Get(target.PostgresFormat)),
ConnectionString: env.Get(connectionStringEnv, kv.Get(target.PostgresConnectionString)), ConnectionString: env.Get(connectionStringEnv, kv.Get(target.PostgresConnectionString)),
Table: env.Get(tableEnv, kv.Get(target.PostgresTable)), Table: env.Get(tableEnv, kv.Get(target.PostgresTable)),
QueueDir: env.Get(queueDirEnv, kv.Get(target.PostgresQueueDir)), QueueDir: env.Get(queueDirEnv, kv.Get(target.PostgresQueueDir)),
QueueLimit: uint64(queueLimit), QueueLimit: uint64(queueLimit),
MaxOpenConnections: maxOpenConnections,
} }
if err = psqlArgs.Validate(); err != nil { if err = psqlArgs.Validate(); err != nil {
return nil, err return nil, err

62
docs/bucket/notifications/README.md
View file

@ -856,12 +856,13 @@ KEY:
notify_postgres[:name] publish bucket notifications to Postgres databases notify_postgres[:name] publish bucket notifications to Postgres databases
ARGS: ARGS:
connection_string* (string) Postgres server connection-string e.g. "host=localhost port=5432 dbname=minio_events user=postgres password=password sslmode=disable" connection_string* (string) Postgres server connection-string e.g. "host=localhost port=5432 dbname=minio_events user=postgres password=password sslmode=disable"
table* (string) DB table name to store/update events, table is auto-created table* (string) DB table name to store/update events, table is auto-created
format* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' format* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'
queue_dir (path) staging dir for undelivered messages e.g. '/home/events' queue_dir (path) staging dir for undelivered messages e.g. '/home/events'
queue_limit (number) maximum limit for undelivered messages, defaults to '100000' queue_limit (number) maximum limit for undelivered messages, defaults to '100000'
comment (sentence) optionally add a comment to this setting max_open_connections (number) maximum number of open connections to the database, defaults to '2'
comment (sentence) optionally add a comment to this setting
``` ```
or environment variables or environment variables
@ -870,15 +871,19 @@ KEY:
notify_postgres[:name] publish bucket notifications to Postgres databases notify_postgres[:name] publish bucket notifications to Postgres databases
ARGS: ARGS:
MINIO_NOTIFY_POSTGRES_ENABLE* (on|off) enable notify_postgres target, default is 'off' MINIO_NOTIFY_POSTGRES_ENABLE* (on|off) enable notify_postgres target, default is 'off'
MINIO_NOTIFY_POSTGRES_CONNECTION_STRING* (string) Postgres server connection-string e.g. "host=localhost port=5432 dbname=minio_events user=postgres password=password sslmode=disable" MINIO_NOTIFY_POSTGRES_CONNECTION_STRING* (string) Postgres server connection-string e.g. "host=localhost port=5432 dbname=minio_events user=postgres password=password sslmode=disable"
MINIO_NOTIFY_POSTGRES_TABLE* (string) DB table name to store/update events, table is auto-created MINIO_NOTIFY_POSTGRES_TABLE* (string) DB table name to store/update events, table is auto-created
MINIO_NOTIFY_POSTGRES_FORMAT* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' MINIO_NOTIFY_POSTGRES_FORMAT* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'
MINIO_NOTIFY_POSTGRES_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' MINIO_NOTIFY_POSTGRES_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events'
MINIO_NOTIFY_POSTGRES_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' MINIO_NOTIFY_POSTGRES_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000'
MINIO_NOTIFY_POSTGRES_COMMENT (sentence) optionally add a comment to this setting MINIO_NOTIFY_POSTGRES_COMMENT (sentence) optionally add a comment to this setting
MINIO_NOTIFY_POSTGRES_MAX_OPEN_CONNECTIONS (number) maximum number of open connections to the database, defaults to '2'
``` ```
> NOTE: If the `max_open_connections` key or the environment variable `MINIO_NOTIFY_POSTGRES_MAX_OPEN_CONNECTIONS` is set to `0`, There will be no limit set on the number of
> open connections to the database. This setting is generally NOT recommended as the behaviour may be inconsistent during recursive deletes in `namespace` format.
MinIO supports persistent event store. The persistent store will backup events when the PostgreSQL connection goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. MinIO supports persistent event store. The persistent store will backup events when the PostgreSQL connection goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000.
Note that for illustration here, we have disabled SSL. In the interest of security, for production this is not recommended. Note that for illustration here, we have disabled SSL. In the interest of security, for production this is not recommended.
@ -985,12 +990,13 @@ KEY:
notify_mysql[:name] publish bucket notifications to MySQL databases. When multiple MySQL server endpoints are needed, a user specified "name" can be added for each configuration, (e.g."notify_mysql:myinstance"). notify_mysql[:name] publish bucket notifications to MySQL databases. When multiple MySQL server endpoints are needed, a user specified "name" can be added for each configuration, (e.g."notify_mysql:myinstance").
ARGS: ARGS:
dsn_string* (string) MySQL data-source-name connection string e.g. "<user>:<password>@tcp(<host>:<port>)/<database>" dsn_string* (string) MySQL data-source-name connection string e.g. "<user>:<password>@tcp(<host>:<port>)/<database>"
table* (string) DB table name to store/update events, table is auto-created table* (string) DB table name to store/update events, table is auto-created
format* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' format* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'
queue_dir (path) staging dir for undelivered messages e.g. '/home/events' queue_dir (path) staging dir for undelivered messages e.g. '/home/events'
queue_limit (number) maximum limit for undelivered messages, defaults to '100000' queue_limit (number) maximum limit for undelivered messages, defaults to '100000'
comment (sentence) optionally add a comment to this setting max_open_connections (number) maximum number of open connections to the database, defaults to '2'
comment (sentence) optionally add a comment to this setting
``` ```
or environment variables or environment variables
@ -999,15 +1005,19 @@ KEY:
notify_mysql[:name] publish bucket notifications to MySQL databases notify_mysql[:name] publish bucket notifications to MySQL databases
ARGS: ARGS:
MINIO_NOTIFY_MYSQL_ENABLE* (on|off) enable notify_mysql target, default is 'off' MINIO_NOTIFY_MYSQL_ENABLE* (on|off) enable notify_mysql target, default is 'off'
MINIO_NOTIFY_MYSQL_DSN_STRING* (string) MySQL data-source-name connection string e.g. "<user>:<password>@tcp(<host>:<port>)/<database>" MINIO_NOTIFY_MYSQL_DSN_STRING* (string) MySQL data-source-name connection string e.g. "<user>:<password>@tcp(<host>:<port>)/<database>"
MINIO_NOTIFY_MYSQL_TABLE* (string) DB table name to store/update events, table is auto-created MINIO_NOTIFY_MYSQL_TABLE* (string) DB table name to store/update events, table is auto-created
MINIO_NOTIFY_MYSQL_FORMAT* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' MINIO_NOTIFY_MYSQL_FORMAT* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'
MINIO_NOTIFY_MYSQL_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' MINIO_NOTIFY_MYSQL_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events'
MINIO_NOTIFY_MYSQL_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' MINIO_NOTIFY_MYSQL_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000'
MINIO_NOTIFY_MYSQL_COMMENT (sentence) optionally add a comment to this setting MINIO_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS (number) maximum number of open connections to the database, defaults to '2'
MINIO_NOTIFY_MYSQL_COMMENT (sentence) optionally add a comment to this setting
``` ```
> NOTE: If the `max_open_connections` key or the environment variable `MINIO_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS` is set to `0`, There will be no limit set on the number of
> open connections to the database. This setting is generally NOT recommended as the behaviour may be inconsistent during recursive deletes in `namespace` format.
`dsn_string` is required and is of form `"<user>:<password>@tcp(<host>:<port>)/<database>"` `dsn_string` is required and is of form `"<user>:<password>@tcp(<host>:<port>)/<database>"`
MinIO supports persistent event store. The persistent store will backup events if MySQL connection goes offline and then replays the stored events when the broken connection comes back up. The event store can be configured by setting a directory path in `queue_dir` field, and the maximum number of events, which can be stored in a `queue_dir`, in `queue_limit` field. For example, `queue_dir` can be set to `/home/events` and `queue_limit` can be set to `1000`. By default, the `queue_limit` is set to `100000`. MinIO supports persistent event store. The persistent store will backup events if MySQL connection goes offline and then replays the stored events when the broken connection comes back up. The event store can be configured by setting a directory path in `queue_dir` field, and the maximum number of events, which can be stored in a `queue_dir`, in `queue_limit` field. For example, `queue_dir` can be set to `/home/events` and `queue_limit` can be set to `1000`. By default, the `queue_limit` is set to `100000`.

2
go.mod
View file

@ -40,7 +40,7 @@ require (
github.com/klauspost/pgzip v1.2.1 github.com/klauspost/pgzip v1.2.1
github.com/klauspost/readahead v1.3.1 github.com/klauspost/readahead v1.3.1
github.com/klauspost/reedsolomon v1.9.9 github.com/klauspost/reedsolomon v1.9.9
github.com/lib/pq v1.7.0 github.com/lib/pq v1.8.0
github.com/mattn/go-colorable v0.1.4 github.com/mattn/go-colorable v0.1.4
github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/mattn/go-ieproxy v0.0.1 // indirect
github.com/mattn/go-isatty v0.0.8 github.com/mattn/go-isatty v0.0.8

2
go.sum
View file

@ -280,6 +280,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY= github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY=
github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg=
github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8=
github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=

80
pkg/event/target/mysql.go
View file

@ -83,43 +83,46 @@ const (
// MySQL related constants // MySQL related constants
const ( const (
MySQLFormat = "format" MySQLFormat = "format"
MySQLDSNString = "dsn_string" MySQLDSNString = "dsn_string"
MySQLTable = "table" MySQLTable = "table"
MySQLHost = "host" MySQLHost = "host"
MySQLPort = "port" MySQLPort = "port"
MySQLUsername = "username" MySQLUsername = "username"
MySQLPassword = "password" MySQLPassword = "password"
MySQLDatabase = "database" MySQLDatabase = "database"
MySQLQueueLimit = "queue_limit" MySQLQueueLimit = "queue_limit"
MySQLQueueDir = "queue_dir" MySQLQueueDir = "queue_dir"
MySQLMaxOpenConnections = "max_open_connections"
EnvMySQLEnable = "MINIO_NOTIFY_MYSQL_ENABLE" EnvMySQLEnable = "MINIO_NOTIFY_MYSQL_ENABLE"
EnvMySQLFormat = "MINIO_NOTIFY_MYSQL_FORMAT" EnvMySQLFormat = "MINIO_NOTIFY_MYSQL_FORMAT"
EnvMySQLDSNString = "MINIO_NOTIFY_MYSQL_DSN_STRING" EnvMySQLDSNString = "MINIO_NOTIFY_MYSQL_DSN_STRING"
EnvMySQLTable = "MINIO_NOTIFY_MYSQL_TABLE" EnvMySQLTable = "MINIO_NOTIFY_MYSQL_TABLE"
EnvMySQLHost = "MINIO_NOTIFY_MYSQL_HOST" EnvMySQLHost = "MINIO_NOTIFY_MYSQL_HOST"
EnvMySQLPort = "MINIO_NOTIFY_MYSQL_PORT" EnvMySQLPort = "MINIO_NOTIFY_MYSQL_PORT"
EnvMySQLUsername = "MINIO_NOTIFY_MYSQL_USERNAME" EnvMySQLUsername = "MINIO_NOTIFY_MYSQL_USERNAME"
EnvMySQLPassword = "MINIO_NOTIFY_MYSQL_PASSWORD" EnvMySQLPassword = "MINIO_NOTIFY_MYSQL_PASSWORD"
EnvMySQLDatabase = "MINIO_NOTIFY_MYSQL_DATABASE" EnvMySQLDatabase = "MINIO_NOTIFY_MYSQL_DATABASE"
EnvMySQLQueueLimit = "MINIO_NOTIFY_MYSQL_QUEUE_LIMIT" EnvMySQLQueueLimit = "MINIO_NOTIFY_MYSQL_QUEUE_LIMIT"
EnvMySQLQueueDir = "MINIO_NOTIFY_MYSQL_QUEUE_DIR" EnvMySQLQueueDir = "MINIO_NOTIFY_MYSQL_QUEUE_DIR"
EnvMySQLMaxOpenConnections = "MINIO_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS"
) )
// MySQLArgs - MySQL target arguments. // MySQLArgs - MySQL target arguments.
type MySQLArgs struct { type MySQLArgs struct {
Enable bool `json:"enable"` Enable bool `json:"enable"`
Format string `json:"format"` Format string `json:"format"`
DSN string `json:"dsnString"` DSN string `json:"dsnString"`
Table string `json:"table"` Table string `json:"table"`
Host xnet.URL `json:"host"` Host xnet.URL `json:"host"`
Port string `json:"port"` Port string `json:"port"`
User string `json:"user"` User string `json:"user"`
Password string `json:"password"` Password string `json:"password"`
Database string `json:"database"` Database string `json:"database"`
QueueDir string `json:"queueDir"` QueueDir string `json:"queueDir"`
QueueLimit uint64 `json:"queueLimit"` QueueLimit uint64 `json:"queueLimit"`
MaxOpenConnections int `json:"maxOpenConnections"`
} }
// Validate MySQLArgs fields // Validate MySQLArgs fields
@ -162,6 +165,10 @@ func (m MySQLArgs) Validate() error {
} }
} }
if m.MaxOpenConnections < 0 {
return errors.New("maxOpenConnections cannot be less than zero")
}
return nil return nil
} }
@ -196,6 +203,10 @@ func (target *MySQLTarget) IsActive() (bool, error) {
return false, sErr return false, sErr
} }
target.db = db target.db = db
if target.args.MaxOpenConnections > 0 {
// Set the maximum connections limit
target.db.SetMaxOpenConns(target.args.MaxOpenConnections)
}
} }
if err := target.db.Ping(); err != nil { if err := target.db.Ping(); err != nil {
if IsConnErr(err) { if IsConnErr(err) {
@ -384,6 +395,11 @@ func NewMySQLTarget(id string, args MySQLArgs, doneCh <-chan struct{}, loggerOnc
} }
target.db = db target.db = db
if args.MaxOpenConnections > 0 {
// Set the maximum connections limit
target.db.SetMaxOpenConns(args.MaxOpenConnections)
}
var store Store var store Store
if args.QueueDir != "" { if args.QueueDir != "" {

80
pkg/event/target/postgresql.go
View file

@ -84,43 +84,46 @@ const (
// Postgres constants // Postgres constants
const ( const (
PostgresFormat = "format" PostgresFormat = "format"
PostgresConnectionString = "connection_string" PostgresConnectionString = "connection_string"
PostgresTable = "table" PostgresTable = "table"
PostgresHost = "host" PostgresHost = "host"
PostgresPort = "port" PostgresPort = "port"
PostgresUsername = "username" PostgresUsername = "username"
PostgresPassword = "password" PostgresPassword = "password"
PostgresDatabase = "database" PostgresDatabase = "database"
PostgresQueueDir = "queue_dir" PostgresQueueDir = "queue_dir"
PostgresQueueLimit = "queue_limit" PostgresQueueLimit = "queue_limit"
PostgresMaxOpenConnections = "max_open_connections"
EnvPostgresEnable = "MINIO_NOTIFY_POSTGRES_ENABLE" EnvPostgresEnable = "MINIO_NOTIFY_POSTGRES_ENABLE"
EnvPostgresFormat = "MINIO_NOTIFY_POSTGRES_FORMAT" EnvPostgresFormat = "MINIO_NOTIFY_POSTGRES_FORMAT"
EnvPostgresConnectionString = "MINIO_NOTIFY_POSTGRES_CONNECTION_STRING" EnvPostgresConnectionString = "MINIO_NOTIFY_POSTGRES_CONNECTION_STRING"
EnvPostgresTable = "MINIO_NOTIFY_POSTGRES_TABLE" EnvPostgresTable = "MINIO_NOTIFY_POSTGRES_TABLE"
EnvPostgresHost = "MINIO_NOTIFY_POSTGRES_HOST" EnvPostgresHost = "MINIO_NOTIFY_POSTGRES_HOST"
EnvPostgresPort = "MINIO_NOTIFY_POSTGRES_PORT" EnvPostgresPort = "MINIO_NOTIFY_POSTGRES_PORT"
EnvPostgresUsername = "MINIO_NOTIFY_POSTGRES_USERNAME" EnvPostgresUsername = "MINIO_NOTIFY_POSTGRES_USERNAME"
EnvPostgresPassword = "MINIO_NOTIFY_POSTGRES_PASSWORD" EnvPostgresPassword = "MINIO_NOTIFY_POSTGRES_PASSWORD"
EnvPostgresDatabase = "MINIO_NOTIFY_POSTGRES_DATABASE" EnvPostgresDatabase = "MINIO_NOTIFY_POSTGRES_DATABASE"
EnvPostgresQueueDir = "MINIO_NOTIFY_POSTGRES_QUEUE_DIR" EnvPostgresQueueDir = "MINIO_NOTIFY_POSTGRES_QUEUE_DIR"
EnvPostgresQueueLimit = "MINIO_NOTIFY_POSTGRES_QUEUE_LIMIT" EnvPostgresQueueLimit = "MINIO_NOTIFY_POSTGRES_QUEUE_LIMIT"
EnvPostgresMaxOpenConnections = "MINIO_NOTIFY_POSTGRES_MAX_OPEN_CONNECTIONS"
) )
// PostgreSQLArgs - PostgreSQL target arguments. // PostgreSQLArgs - PostgreSQL target arguments.
type PostgreSQLArgs struct { type PostgreSQLArgs struct {
Enable bool `json:"enable"` Enable bool `json:"enable"`
Format string `json:"format"` Format string `json:"format"`
ConnectionString string `json:"connectionString"` ConnectionString string `json:"connectionString"`
Table string `json:"table"` Table string `json:"table"`
Host xnet.Host `json:"host"` // default: localhost Host xnet.Host `json:"host"` // default: localhost
Port string `json:"port"` // default: 5432 Port string `json:"port"` // default: 5432
Username string `json:"username"` // default: user running minio Username string `json:"username"` // default: user running minio
Password string `json:"password"` // default: no password Password string `json:"password"` // default: no password
Database string `json:"database"` // default: same as user Database string `json:"database"` // default: same as user
QueueDir string `json:"queueDir"` QueueDir string `json:"queueDir"`
QueueLimit uint64 `json:"queueLimit"` QueueLimit uint64 `json:"queueLimit"`
MaxOpenConnections int `json:"maxOpenConnections"`
} }
// Validate PostgreSQLArgs fields // Validate PostgreSQLArgs fields
@ -160,6 +163,10 @@ func (p PostgreSQLArgs) Validate() error {
} }
} }
if p.MaxOpenConnections < 0 {
return errors.New("maxOpenConnections cannot be less than zero")
}
return nil return nil
} }
@ -195,6 +202,10 @@ func (target *PostgreSQLTarget) IsActive() (bool, error) {
return false, err return false, err
} }
target.db = db target.db = db
if target.args.MaxOpenConnections > 0 {
// Set the maximum connections limit
target.db.SetMaxOpenConns(target.args.MaxOpenConnections)
}
} }
if err := target.db.Ping(); err != nil { if err := target.db.Ping(); err != nil {
if IsConnErr(err) { if IsConnErr(err) {
@ -391,6 +402,11 @@ func NewPostgreSQLTarget(id string, args PostgreSQLArgs, doneCh <-chan struct{},
} }
target.db = db target.db = db
if args.MaxOpenConnections > 0 {
// Set the maximum connections limit
target.db.SetMaxOpenConns(args.MaxOpenConnections)
}
var store Store var store Store
if args.QueueDir != "" { if args.QueueDir != "" {

1
ruleguard.rules.go
View file

@ -438,7 +438,6 @@ func httpheaderadd(m fluent.Matcher) {
Suggest(`$H.Set($KEY, $VALUE)`) Suggest(`$H.Set($KEY, $VALUE)`)
} }
func hmacnew(m fluent.Matcher) { func hmacnew(m fluent.Matcher) {
m.Match("hmac.New(func() hash.Hash { return $x }, $_)", m.Match("hmac.New(func() hash.Hash { return $x }, $_)",
`$f := func() hash.Hash { return $x } `$f := func() hash.Hash { return $x }