From 3be9aa2319a4572703c8a764103585b8a185b66e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 25 Aug 2019 17:25:19 +0300 Subject: [PATCH] Improve errors in state store migration edge cases --- config/config.go | 2 +- .../upgrades/2018-09-01-initial-schema.go | 4 ++-- .../2019-05-16-message-delete-cascade.go | 4 ++-- .../2019-05-21-message-timestamp-column.go | 2 +- .../2019-05-22-user-last-connection-column.go | 2 +- database/upgrades/2019-05-23-protoupgrade.go | 4 ++-- .../2019-05-23-puppet-custom-mxid-columns.go | 2 +- .../upgrades/2019-05-28-user-portal-table.go | 2 +- .../upgrades/2019-06-01-avatar-url-fields.go | 2 +- .../2019-08-10-portal-in-community-field.go | 2 +- .../2019-08-25-move-state-store-to-db.go | 10 +++++++--- database/upgrades/upgrades.go | 20 ++++++++++++------- main.go | 9 +++++++++ 13 files changed, 42 insertions(+), 23 deletions(-) diff --git a/config/config.go b/config/config.go index 20eff6d..c5e3b5a 100644 --- a/config/config.go +++ b/config/config.go @@ -43,7 +43,7 @@ type Config struct { MaxIdleConns int `yaml:"max_idle_conns"` } `yaml:"database"` - StateStore string `yaml:"state_store_path"` + StateStore string `yaml:"state_store_path,omitempty"` ID string `yaml:"id"` Bot struct { diff --git a/database/upgrades/2018-09-01-initial-schema.go b/database/upgrades/2018-09-01-initial-schema.go index 3e0c7c6..b1b8709 100644 --- a/database/upgrades/2018-09-01-initial-schema.go +++ b/database/upgrades/2018-09-01-initial-schema.go @@ -6,9 +6,9 @@ import ( ) func init() { - upgrades[0] = upgrade{"Initial schema", func(dialect Dialect, tx *sql.Tx, db *sql.DB) error { + upgrades[0] = upgrade{"Initial schema", func(tx *sql.Tx, ctx context) error { var byteType string - if dialect == SQLite { + if ctx.dialect == SQLite { byteType = "BLOB" } else { byteType = "bytea" diff --git a/database/upgrades/2019-05-16-message-delete-cascade.go b/database/upgrades/2019-05-16-message-delete-cascade.go index 131e086..970bceb 100644 --- a/database/upgrades/2019-05-16-message-delete-cascade.go +++ b/database/upgrades/2019-05-16-message-delete-cascade.go @@ -5,8 +5,8 @@ import ( ) func init() { - upgrades[1] = upgrade{"Add ON DELETE CASCADE to message table", func(dialect Dialect, tx *sql.Tx, db *sql.DB) error { - if dialect == SQLite { + upgrades[1] = upgrade{"Add ON DELETE CASCADE to message table", func(tx *sql.Tx, ctx context) error { + if ctx.dialect == SQLite { // SQLite doesn't support constraint updates, but it isn't that careful with constraints anyway. return nil } diff --git a/database/upgrades/2019-05-21-message-timestamp-column.go b/database/upgrades/2019-05-21-message-timestamp-column.go index ebd7e38..cb93614 100644 --- a/database/upgrades/2019-05-21-message-timestamp-column.go +++ b/database/upgrades/2019-05-21-message-timestamp-column.go @@ -5,7 +5,7 @@ import ( ) func init() { - upgrades[2] = upgrade{"Add timestamp column to messages", func(dialect Dialect, tx *sql.Tx, db *sql.DB) error { + upgrades[2] = upgrade{"Add timestamp column to messages", func(tx *sql.Tx, ctx context) error { _, err := tx.Exec("ALTER TABLE message ADD COLUMN timestamp BIGINT NOT NULL DEFAULT 0") if err != nil { return err diff --git a/database/upgrades/2019-05-22-user-last-connection-column.go b/database/upgrades/2019-05-22-user-last-connection-column.go index db1a214..3e1a236 100644 --- a/database/upgrades/2019-05-22-user-last-connection-column.go +++ b/database/upgrades/2019-05-22-user-last-connection-column.go @@ -5,7 +5,7 @@ import ( ) func init() { - upgrades[3] = upgrade{"Add last_connection column to users", func(dialect Dialect, tx *sql.Tx, db *sql.DB) error { + upgrades[3] = upgrade{"Add last_connection column to users", func(tx *sql.Tx, ctx context) error { _, err := tx.Exec(`ALTER TABLE "user" ADD COLUMN last_connection BIGINT NOT NULL DEFAULT 0`) if err != nil { return err diff --git a/database/upgrades/2019-05-23-protoupgrade.go b/database/upgrades/2019-05-23-protoupgrade.go index 4bbc01a..ce250dd 100644 --- a/database/upgrades/2019-05-23-protoupgrade.go +++ b/database/upgrades/2019-05-23-protoupgrade.go @@ -8,8 +8,8 @@ import ( func init() { var keys = []string{"imageMessage", "contactMessage", "locationMessage", "extendedTextMessage", "documentMessage", "audioMessage", "videoMessage"} - upgrades[4] = upgrade{"Update message content to new protocol version. This may take a while.", func(dialect Dialect, tx *sql.Tx, db *sql.DB) error { - rows, err := db.Query("SELECT mxid, content FROM message") + upgrades[4] = upgrade{"Update message content to new protocol version. This may take a while.", func(tx *sql.Tx, ctx context) error { + rows, err := ctx.db.Query("SELECT mxid, content FROM message") if err != nil { return err } diff --git a/database/upgrades/2019-05-23-puppet-custom-mxid-columns.go b/database/upgrades/2019-05-23-puppet-custom-mxid-columns.go index c99d521..2f17154 100644 --- a/database/upgrades/2019-05-23-puppet-custom-mxid-columns.go +++ b/database/upgrades/2019-05-23-puppet-custom-mxid-columns.go @@ -5,7 +5,7 @@ import ( ) func init() { - upgrades[5] = upgrade{"Add columns to store custom puppet info", func(dialect Dialect, tx *sql.Tx, db *sql.DB) error { + upgrades[5] = upgrade{"Add columns to store custom puppet info", func(tx *sql.Tx, ctx context) error { _, err := tx.Exec(`ALTER TABLE puppet ADD COLUMN custom_mxid VARCHAR(255)`) if err != nil { return err diff --git a/database/upgrades/2019-05-28-user-portal-table.go b/database/upgrades/2019-05-28-user-portal-table.go index cbc44a7..18d8550 100644 --- a/database/upgrades/2019-05-28-user-portal-table.go +++ b/database/upgrades/2019-05-28-user-portal-table.go @@ -5,7 +5,7 @@ import ( ) func init() { - upgrades[6] = upgrade{"Add user-portal mapping table", func(dialect Dialect, tx *sql.Tx, db *sql.DB) error { + upgrades[6] = upgrade{"Add user-portal mapping table", func(tx *sql.Tx, ctx context) error { _, err := tx.Exec(`CREATE TABLE user_portal ( user_jid VARCHAR(255), portal_jid VARCHAR(255), diff --git a/database/upgrades/2019-06-01-avatar-url-fields.go b/database/upgrades/2019-06-01-avatar-url-fields.go index ae6cf8b..938b291 100644 --- a/database/upgrades/2019-06-01-avatar-url-fields.go +++ b/database/upgrades/2019-06-01-avatar-url-fields.go @@ -5,7 +5,7 @@ import ( ) func init() { - upgrades[7] = upgrade{"Add columns to store avatar MXC URIs", func(dialect Dialect, tx *sql.Tx, db *sql.DB) error { + upgrades[7] = upgrade{"Add columns to store avatar MXC URIs", func(tx *sql.Tx, ctx context) error { _, err := tx.Exec(`ALTER TABLE puppet ADD COLUMN avatar_url VARCHAR(255)`) if err != nil { return err diff --git a/database/upgrades/2019-08-10-portal-in-community-field.go b/database/upgrades/2019-08-10-portal-in-community-field.go index e5cb608..44893fd 100644 --- a/database/upgrades/2019-08-10-portal-in-community-field.go +++ b/database/upgrades/2019-08-10-portal-in-community-field.go @@ -5,7 +5,7 @@ import ( ) func init() { - upgrades[8] = upgrade{"Add columns to store portal in filtering community meta", func(dialect Dialect, tx *sql.Tx, db *sql.DB) error { + upgrades[8] = upgrade{"Add columns to store portal in filtering community meta", func(tx *sql.Tx, ctx context) error { _, err := tx.Exec(`ALTER TABLE user_portal ADD COLUMN in_community BOOLEAN NOT NULL DEFAULT FALSE`) return err }} diff --git a/database/upgrades/2019-08-25-move-state-store-to-db.go b/database/upgrades/2019-08-25-move-state-store-to-db.go index 22711c7..86e9104 100644 --- a/database/upgrades/2019-08-25-move-state-store-to-db.go +++ b/database/upgrades/2019-08-25-move-state-store-to-db.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "os" "strings" "maunium.net/go/mautrix" @@ -78,15 +79,16 @@ func init() { user_id VARCHAR(255) PRIMARY KEY )` - upgrades[9] = upgrade{"Move state store to main DB", func(dialect Dialect, tx *sql.Tx, db *sql.DB) error { + upgrades[9] = upgrade{"Move state store to main DB", func(tx *sql.Tx, ctx context) error { store := appservice.NewBasicStateStore().(*appservice.BasicStateStore) - if dialect == Postgres { + if ctx.dialect == Postgres { roomStateTable = strings.Replace(roomStateTable, "TEXT", "JSONB", 1) } if data, err := ioutil.ReadFile("mx-state.json"); err != nil { - return err + ctx.log.Debugln("mx-state.json not found, not migrating state store") + return nil } else if err = json.Unmarshal(data, &store); err != nil { return err } else if _, err := tx.Exec(userProfileTable); err != nil { @@ -101,6 +103,8 @@ func init() { return err } else if err = migratePowerLevels(tx, store.PowerLevels); err != nil { return err + } else if err = os.Rename("mx-state.json", "mx-state.json.bak"); err != nil { + return err } return nil }} diff --git a/database/upgrades/upgrades.go b/database/upgrades/upgrades.go index acc39ac..b332fa0 100644 --- a/database/upgrades/upgrades.go +++ b/database/upgrades/upgrades.go @@ -15,11 +15,17 @@ const ( SQLite ) -type upgradeFunc func(Dialect, *sql.Tx, *sql.DB) error +type upgradeFunc func(*sql.Tx, context) error + +type context struct { + dialect Dialect + db *sql.DB + log log.Logger +} type upgrade struct { message string - fn upgradeFunc + fn upgradeFunc } const NumberOfUpgrades = 10 @@ -28,7 +34,7 @@ var upgrades [NumberOfUpgrades]upgrade var UnsupportedDatabaseVersion = fmt.Errorf("unsupported database version") -func getVersion(dialect Dialect, db *sql.DB) (int, error) { +func GetVersion(db *sql.DB) (int, error) { _, err := db.Exec("CREATE TABLE IF NOT EXISTS version (version INTEGER)") if err != nil { return -1, err @@ -42,7 +48,7 @@ func getVersion(dialect Dialect, db *sql.DB) (int, error) { return version, nil } -func setVersion(dialect Dialect, tx *sql.Tx, version int) error { +func SetVersion(tx *sql.Tx, version int) error { _, err := tx.Exec("DELETE FROM version") if err != nil { return err @@ -62,7 +68,7 @@ func Run(log log.Logger, dialectName string, db *sql.DB) error { return fmt.Errorf("unknown dialect %s", dialectName) } - version, err := getVersion(dialect, db) + version, err := GetVersion(db) if err != nil { return err } @@ -78,11 +84,11 @@ func Run(log log.Logger, dialectName string, db *sql.DB) error { if err != nil { return err } - err = upgrade.fn(dialect, tx, db) + err = upgrade.fn(tx, context{dialect, db, log}) if err != nil { return err } - err = setVersion(dialect, tx, version+i+1) + err = SetVersion(tx, version+i+1) if err != nil { return err } diff --git a/main.go b/main.go index de67fb4..f1c1b29 100644 --- a/main.go +++ b/main.go @@ -164,6 +164,15 @@ func (bridge *Bridge) Init() { os.Exit(14) } + if len(bridge.Config.AppService.StateStore) > 0 && bridge.Config.AppService.StateStore != "./mx-state.json" { + version, err := upgrades.GetVersion(bridge.DB.DB) + if version < 0 && err == nil { + bridge.Log.Fatalln("Non-standard state store path. Please move the state store to ./mx-state.json " + + "and update the config. The state store will be migrated into the db on the next launch.") + os.Exit(18) + } + } + bridge.Log.Debugln("Initializing state store") bridge.StateStore = database.NewSQLStateStore(bridge.DB) bridge.AS.StateStore = bridge.StateStore