From 2463f8e60f85374818af7ba46b0b6facd626a73c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 26 May 2022 11:23:15 +0300 Subject: [PATCH] Port some more old db schema upgrades --- CHANGELOG.md | 3 -- database/upgrades/00-latest-revision.sql | 5 +-- database/upgrades/36-phone-last-seen-ts.sql | 3 ++ database/upgrades/37-message-error-string.sql | 11 +++++++ database/upgrades/38-phone-ping-ts.sql | 3 ++ database/upgrades/39-reactions.sql | 21 +++++++++++++ database/upgrades/40-prioritized-backfill.sql | 22 +++++++++++++ database/upgrades/41-historysync-store.sql | 31 +++++++++++++++++++ .../upgrades/42-backfillqueue-type-order.sql | 9 ++++++ .../upgrades/43-media-backfill-requests.sql | 14 +++++++++ database/upgrades/44-user-timezone.sql | 3 ++ database/upgrades/upgrades.go | 2 +- 12 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 database/upgrades/36-phone-last-seen-ts.sql create mode 100644 database/upgrades/37-message-error-string.sql create mode 100644 database/upgrades/38-phone-ping-ts.sql create mode 100644 database/upgrades/39-reactions.sql create mode 100644 database/upgrades/40-prioritized-backfill.sql create mode 100644 database/upgrades/41-historysync-store.sql create mode 100644 database/upgrades/42-backfillqueue-type-order.sql create mode 100644 database/upgrades/43-media-backfill-requests.sql create mode 100644 database/upgrades/44-user-timezone.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index f9f4a03..308527d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,5 @@ # unreleased -**N.B.** Due to some changes to the database upgrade system, you must upgrade -the bridge to v0.4.0 before updating to a newer version. - * Improved handling edge cases in backfill system. * Fixed chats not being marked as read when sending a message from another WhatsApp client after receiving a call. diff --git a/database/upgrades/00-latest-revision.sql b/database/upgrades/00-latest-revision.sql index 5920ffa..b4b3830 100644 --- a/database/upgrades/00-latest-revision.sql +++ b/database/upgrades/00-latest-revision.sql @@ -28,7 +28,7 @@ CREATE TABLE portal ( first_event_id TEXT, next_batch_id TEXT, relay_user_id TEXT, - expiration_time BIGINT NOT NULL DEFAULT 0, + expiration_time BIGINT NOT NULL DEFAULT 0 CHECK (expiration_time >= 0 AND expiration_time < 4294967296), PRIMARY KEY (jid, receiver) ); @@ -118,7 +118,8 @@ CREATE TABLE backfill_queue ( max_batch_events INTEGER NOT NULL, max_total_events INTEGER, - FOREIGN KEY (portal_jid, portal_receiver) REFERENCES portal(jid, receiver) ON DELETE CASCADE ON UPDATE CASCADE + FOREIGN KEY (user_mxid) REFERENCES "user" (mxid) ON DELETE CASCADE ON UPDATE CASCADE, + FOREIGN KEY (portal_jid, portal_receiver) REFERENCES portal(jid, receiver) ON DELETE CASCADE ); CREATE TABLE backfill_state ( diff --git a/database/upgrades/36-phone-last-seen-ts.sql b/database/upgrades/36-phone-last-seen-ts.sql new file mode 100644 index 0000000..21e5b70 --- /dev/null +++ b/database/upgrades/36-phone-last-seen-ts.sql @@ -0,0 +1,3 @@ +-- v36: Store approximate last seen timestamp of the main device + +ALTER TABLE "user" ADD COLUMN phone_last_seen BIGINT; diff --git a/database/upgrades/37-message-error-string.sql b/database/upgrades/37-message-error-string.sql new file mode 100644 index 0000000..88a647c --- /dev/null +++ b/database/upgrades/37-message-error-string.sql @@ -0,0 +1,11 @@ +-- v37: Store message error type as string + +-- only: postgres +CREATE TYPE error_type AS ENUM ('', 'decryption_failed', 'media_not_found'); + +ALTER TABLE message ADD COLUMN error error_type NOT NULL DEFAULT ''; +UPDATE message SET error='decryption_failed' WHERE decryption_error=true; + +-- TODO do this on sqlite at some point +-- only: postgres +ALTER TABLE message DROP COLUMN decryption_error; diff --git a/database/upgrades/38-phone-ping-ts.sql b/database/upgrades/38-phone-ping-ts.sql new file mode 100644 index 0000000..77e24ed --- /dev/null +++ b/database/upgrades/38-phone-ping-ts.sql @@ -0,0 +1,3 @@ +-- v38: Store timestamp for previous phone ping + +ALTER TABLE "user" ADD COLUMN phone_last_pinged BIGINT; diff --git a/database/upgrades/39-reactions.sql b/database/upgrades/39-reactions.sql new file mode 100644 index 0000000..d90c6b7 --- /dev/null +++ b/database/upgrades/39-reactions.sql @@ -0,0 +1,21 @@ +-- v39: Add support for reactions + +ALTER TABLE message ADD COLUMN type TEXT NOT NULL DEFAULT 'message'; +-- only: postgres +ALTER TABLE message ALTER COLUMN type DROP DEFAULT; + +UPDATE message SET type='' WHERE error='decryption_failed'; +UPDATE message SET type='fake' WHERE jid LIKE 'FAKE::%' OR mxid LIKE 'net.maunium.whatsapp.fake::%' OR jid=mxid; + +CREATE TABLE reaction ( + chat_jid TEXT, + chat_receiver TEXT, + target_jid TEXT, + sender TEXT, + mxid TEXT NOT NULL, + jid TEXT NOT NULL, + PRIMARY KEY (chat_jid, chat_receiver, target_jid, sender), + CONSTRAINT target_message_fkey FOREIGN KEY (chat_jid, chat_receiver, target_jid) + REFERENCES message (chat_jid, chat_receiver, jid) + ON DELETE CASCADE ON UPDATE CASCADE +); diff --git a/database/upgrades/40-prioritized-backfill.sql b/database/upgrades/40-prioritized-backfill.sql new file mode 100644 index 0000000..b435e61 --- /dev/null +++ b/database/upgrades/40-prioritized-backfill.sql @@ -0,0 +1,22 @@ +-- v40: Add backfill queue + +CREATE TABLE backfill_queue ( + queue_id INTEGER PRIMARY KEY + -- only: postgres + GENERATED ALWAYS AS IDENTITY + , + user_mxid TEXT, + type INTEGER NOT NULL, + priority INTEGER NOT NULL, + portal_jid TEXT, + portal_receiver TEXT, + time_start TIMESTAMP, + time_end TIMESTAMP, + completed_at TIMESTAMP, + batch_delay INTEGER, + max_batch_events INTEGER NOT NULL, + max_total_events INTEGER, + + FOREIGN KEY (user_mxid) REFERENCES "user" (mxid) ON DELETE CASCADE ON UPDATE CASCADE, + FOREIGN KEY (portal_jid, portal_receiver) REFERENCES portal(jid, receiver) ON DELETE CASCADE +); diff --git a/database/upgrades/41-historysync-store.sql b/database/upgrades/41-historysync-store.sql new file mode 100644 index 0000000..0a00cb7 --- /dev/null +++ b/database/upgrades/41-historysync-store.sql @@ -0,0 +1,31 @@ +-- v41: Store history syncs for later backfills + +CREATE TABLE history_sync_conversation ( + user_mxid TEXT, + conversation_id TEXT, + portal_jid TEXT, + portal_receiver TEXT, + last_message_timestamp TIMESTAMP, + archived BOOLEAN, + pinned INTEGER, + mute_end_time TIMESTAMP, + disappearing_mode INTEGER, + end_of_history_transfer_type INTEGER, + ephemeral_expiration INTEGER, + marked_as_unread BOOLEAN, + unread_count INTEGER, + PRIMARY KEY (user_mxid, conversation_id), + FOREIGN KEY (user_mxid) REFERENCES "user" (mxid) ON DELETE CASCADE ON UPDATE CASCADE, + FOREIGN KEY (portal_jid, portal_receiver) REFERENCES portal (jid, receiver) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE TABLE history_sync_message ( + user_mxid TEXT, + conversation_id TEXT, + message_id TEXT, + timestamp TIMESTAMP, + data BYTEA, + PRIMARY KEY (user_mxid, conversation_id, message_id), + FOREIGN KEY (user_mxid) REFERENCES "user" (mxid) ON DELETE CASCADE ON UPDATE CASCADE, + FOREIGN KEY (user_mxid, conversation_id) REFERENCES history_sync_conversation (user_mxid, conversation_id) ON DELETE CASCADE +); diff --git a/database/upgrades/42-backfillqueue-type-order.sql b/database/upgrades/42-backfillqueue-type-order.sql new file mode 100644 index 0000000..a45834d --- /dev/null +++ b/database/upgrades/42-backfillqueue-type-order.sql @@ -0,0 +1,9 @@ +-- v42: Update backfill queue tables to be sortable by priority + +UPDATE backfill_queue +SET type=CASE + WHEN type = 1 THEN 200 + WHEN type = 2 THEN 300 + ELSE type +END +WHERE type = 1 OR type = 2; diff --git a/database/upgrades/43-media-backfill-requests.sql b/database/upgrades/43-media-backfill-requests.sql new file mode 100644 index 0000000..7c13803 --- /dev/null +++ b/database/upgrades/43-media-backfill-requests.sql @@ -0,0 +1,14 @@ +-- v43: Add table for tracking which media needs to be requested from the user's phone + +CREATE TABLE media_backfill_requests ( + user_mxid TEXT, + portal_jid TEXT, + portal_receiver TEXT, + event_id TEXT, + media_key BYTEA, + status INTEGER, + error TEXT, + PRIMARY KEY (user_mxid, portal_jid, portal_receiver, event_id), + FOREIGN KEY (user_mxid) REFERENCES "user" (mxid) ON DELETE CASCADE ON UPDATE CASCADE, + FOREIGN KEY (portal_jid, portal_receiver) REFERENCES portal (jid, receiver) ON DELETE CASCADE +); diff --git a/database/upgrades/44-user-timezone.sql b/database/upgrades/44-user-timezone.sql new file mode 100644 index 0000000..1a35fa9 --- /dev/null +++ b/database/upgrades/44-user-timezone.sql @@ -0,0 +1,3 @@ +-- v44: Add timezone column for users + +ALTER TABLE "user" ADD COLUMN timezone TEXT; diff --git a/database/upgrades/upgrades.go b/database/upgrades/upgrades.go index 64642ec..3d3282c 100644 --- a/database/upgrades/upgrades.go +++ b/database/upgrades/upgrades.go @@ -30,7 +30,7 @@ var Table dbutil.UpgradeTable var rawUpgrades embed.FS func init() { - Table.Register(-1, 43, "Unsupported version", func(tx *sql.Tx, database *dbutil.Database) error { + Table.Register(-1, 35, "Unsupported version", func(tx *sql.Tx, database *dbutil.Database) error { return errors.New("please upgrade to mautrix-whatsapp v0.4.0 before upgrading to a newer version") }) Table.RegisterFS(rawUpgrades)