From 4fe179d0d73511816b0e372620768e165a1aba0e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 19 Aug 2021 19:17:19 +0300 Subject: [PATCH] Remove content from message table. Fixes #320 --- database/message.go | 43 ++++------------- .../2021-08-19-remove-message-content.go | 44 +++++++++++++++++ .../2021-08-19-varchar-to-text-crypto.go | 13 +++++ .../upgrades/2021-08-19-varchar-to-text.go | 48 +++++++++++++++++++ database/upgrades/upgrades.go | 21 ++++++-- go.mod | 6 +-- go.sum | 12 ++--- portal.go | 42 ++++++++++++++-- 8 files changed, 177 insertions(+), 52 deletions(-) create mode 100644 database/upgrades/2021-08-19-remove-message-content.go create mode 100644 database/upgrades/2021-08-19-varchar-to-text-crypto.go create mode 100644 database/upgrades/2021-08-19-varchar-to-text.go diff --git a/database/message.go b/database/message.go index 7464e35..83ca81a 100644 --- a/database/message.go +++ b/database/message.go @@ -17,14 +17,11 @@ package database import ( - "bytes" "database/sql" - "encoding/json" "strings" "time" "github.com/Rhymen/go-whatsapp" - waProto "github.com/Rhymen/go-whatsapp/binary/proto" log "maunium.net/go/maulogger/v2" @@ -44,7 +41,7 @@ func (mq *MessageQuery) New() *Message { } func (mq *MessageQuery) GetAll(chat PortalKey) (messages []*Message) { - rows, err := mq.db.Query("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent, content FROM message WHERE chat_jid=$1 AND chat_receiver=$2", chat.JID, chat.Receiver) + rows, err := mq.db.Query("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent FROM message WHERE chat_jid=$1 AND chat_receiver=$2", chat.JID, chat.Receiver) if err != nil || rows == nil { return nil } @@ -56,12 +53,12 @@ func (mq *MessageQuery) GetAll(chat PortalKey) (messages []*Message) { } func (mq *MessageQuery) GetByJID(chat PortalKey, jid whatsapp.MessageID) *Message { - return mq.get("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent, content "+ + return mq.get("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent "+ "FROM message WHERE chat_jid=$1 AND chat_receiver=$2 AND jid=$3", chat.JID, chat.Receiver, jid) } func (mq *MessageQuery) GetByMXID(mxid id.EventID) *Message { - return mq.get("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent, content "+ + return mq.get("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent "+ "FROM message WHERE mxid=$1", mxid) } @@ -70,7 +67,7 @@ func (mq *MessageQuery) GetLastInChat(chat PortalKey) *Message { } func (mq *MessageQuery) GetLastInChatBefore(chat PortalKey, maxTimestamp int64) *Message { - msg := mq.get("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent, content "+ + msg := mq.get("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent "+ "FROM message WHERE chat_jid=$1 AND chat_receiver=$2 AND timestamp<=$3 AND sent=true ORDER BY timestamp DESC LIMIT 1", chat.JID, chat.Receiver, maxTimestamp) if msg == nil || msg.Timestamp == 0 { @@ -98,7 +95,6 @@ type Message struct { Sender whatsapp.JID Timestamp int64 Sent bool - Content *waProto.Message } func (msg *Message) IsFakeMXID() bool { @@ -106,8 +102,7 @@ func (msg *Message) IsFakeMXID() bool { } func (msg *Message) Scan(row Scannable) *Message { - var content []byte - err := row.Scan(&msg.Chat.JID, &msg.Chat.Receiver, &msg.JID, &msg.MXID, &msg.Sender, &msg.Timestamp, &msg.Sent, &content) + err := row.Scan(&msg.Chat.JID, &msg.Chat.Receiver, &msg.JID, &msg.MXID, &msg.Sender, &msg.Timestamp, &msg.Sent) if err != nil { if err != sql.ErrNoRows { msg.log.Errorln("Database scan failed:", err) @@ -115,36 +110,14 @@ func (msg *Message) Scan(row Scannable) *Message { return nil } - msg.decodeBinaryContent(content) - return msg } -func (msg *Message) decodeBinaryContent(content []byte) { - msg.Content = &waProto.Message{} - reader := bytes.NewReader(content) - dec := json.NewDecoder(reader) - err := dec.Decode(msg.Content) - if err != nil { - msg.log.Warnln("Failed to decode message content:", err) - } -} - -func (msg *Message) encodeBinaryContent() []byte { - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - err := enc.Encode(msg.Content) - if err != nil { - msg.log.Warnln("Failed to encode message content:", err) - } - return buf.Bytes() -} - func (msg *Message) Insert() { _, err := msg.db.Exec(`INSERT INTO message - (chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent, content) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, - msg.Chat.JID, msg.Chat.Receiver, msg.JID, msg.MXID, msg.Sender, msg.Timestamp, msg.Sent, msg.encodeBinaryContent()) + (chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent) + VALUES ($1, $2, $3, $4, $5, $6, $7)`, + msg.Chat.JID, msg.Chat.Receiver, msg.JID, msg.MXID, msg.Sender, msg.Timestamp, msg.Sent) if err != nil { msg.log.Warnfln("Failed to insert %s@%s: %v", msg.Chat, msg.JID, err) } diff --git a/database/upgrades/2021-08-19-remove-message-content.go b/database/upgrades/2021-08-19-remove-message-content.go new file mode 100644 index 0000000..c3c7611 --- /dev/null +++ b/database/upgrades/2021-08-19-remove-message-content.go @@ -0,0 +1,44 @@ +package upgrades + +import ( + "database/sql" +) + +func init() { + upgrades[21] = upgrade{"Remove message content from local database", func(tx *sql.Tx, ctx context) error { + if ctx.dialect == SQLite { + _, err := tx.Exec("ALTER TABLE message RENAME TO old_message") + if err != nil { + return err + } + _, err = tx.Exec(`CREATE TABLE IF NOT EXISTS message ( + chat_jid TEXT, + chat_receiver TEXT, + jid TEXT, + mxid TEXT NOT NULL UNIQUE, + sender TEXT NOT NULL, + timestamp BIGINT NOT NULL, + sent BOOLEAN NOT NULL, + + PRIMARY KEY (chat_jid, chat_receiver, jid), + FOREIGN KEY (chat_jid, chat_receiver) REFERENCES portal(jid, receiver) ON DELETE CASCADE + )`) + if err != nil { + return err + } + _, err = tx.Exec("INSERT INTO message SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent FROM old_message") + return err + } else { + _, err := tx.Exec(`ALTER TABLE message DROP COLUMN content`) + if err != nil { + return err + } + _, err = tx.Exec(`ALTER TABLE message ALTER COLUMN timestamp DROP DEFAULT`) + if err != nil { + return err + } + _, err = tx.Exec(`ALTER TABLE message ALTER COLUMN sent DROP DEFAULT`) + return err + } + }} +} diff --git a/database/upgrades/2021-08-19-varchar-to-text-crypto.go b/database/upgrades/2021-08-19-varchar-to-text-crypto.go new file mode 100644 index 0000000..39f5a09 --- /dev/null +++ b/database/upgrades/2021-08-19-varchar-to-text-crypto.go @@ -0,0 +1,13 @@ +package upgrades + +import ( + "database/sql" + + "maunium.net/go/mautrix/crypto/sql_store_upgrade" +) + +func init() { + upgrades[23] = upgrade{"Replace VARCHAR(255) with TEXT in the crypto database", func(tx *sql.Tx, ctx context) error { + return sql_store_upgrade.Upgrades[4](tx, ctx.dialect.String()) + }} +} diff --git a/database/upgrades/2021-08-19-varchar-to-text.go b/database/upgrades/2021-08-19-varchar-to-text.go new file mode 100644 index 0000000..9fcd3ae --- /dev/null +++ b/database/upgrades/2021-08-19-varchar-to-text.go @@ -0,0 +1,48 @@ +package upgrades + +import ( + "database/sql" +) + +func init() { + upgrades[22] = upgrade{"Replace VARCHAR(255) with TEXT in the database", func(tx *sql.Tx, ctx context) error { + if ctx.dialect == SQLite { + // SQLite doesn't enforce varchar sizes anyway + return nil + } + return execMany(tx, + `ALTER TABLE message ALTER COLUMN chat_jid TYPE TEXT`, + `ALTER TABLE message ALTER COLUMN chat_receiver TYPE TEXT`, + `ALTER TABLE message ALTER COLUMN jid TYPE TEXT`, + `ALTER TABLE message ALTER COLUMN mxid TYPE TEXT`, + `ALTER TABLE message ALTER COLUMN sender TYPE TEXT`, + + `ALTER TABLE portal ALTER COLUMN jid TYPE TEXT`, + `ALTER TABLE portal ALTER COLUMN receiver TYPE TEXT`, + `ALTER TABLE portal ALTER COLUMN mxid TYPE TEXT`, + `ALTER TABLE portal ALTER COLUMN name TYPE TEXT`, + `ALTER TABLE portal ALTER COLUMN topic TYPE TEXT`, + `ALTER TABLE portal ALTER COLUMN avatar TYPE TEXT`, + `ALTER TABLE portal ALTER COLUMN avatar_url TYPE TEXT`, + + `ALTER TABLE puppet ALTER COLUMN jid TYPE TEXT`, + `ALTER TABLE puppet ALTER COLUMN avatar TYPE TEXT`, + `ALTER TABLE puppet ALTER COLUMN displayname TYPE TEXT`, + `ALTER TABLE puppet ALTER COLUMN custom_mxid TYPE TEXT`, + `ALTER TABLE puppet ALTER COLUMN access_token TYPE TEXT`, + `ALTER TABLE puppet ALTER COLUMN next_batch TYPE TEXT`, + `ALTER TABLE puppet ALTER COLUMN avatar_url TYPE TEXT`, + + `ALTER TABLE "user" ALTER COLUMN mxid TYPE TEXT`, + `ALTER TABLE "user" ALTER COLUMN jid TYPE TEXT`, + `ALTER TABLE "user" ALTER COLUMN management_room TYPE TEXT`, + `ALTER TABLE "user" ALTER COLUMN client_id TYPE TEXT`, + `ALTER TABLE "user" ALTER COLUMN client_token TYPE TEXT`, + `ALTER TABLE "user" ALTER COLUMN server_token TYPE TEXT`, + + `ALTER TABLE user_portal ALTER COLUMN user_jid TYPE TEXT`, + `ALTER TABLE user_portal ALTER COLUMN portal_jid TYPE TEXT`, + `ALTER TABLE user_portal ALTER COLUMN portal_receiver TYPE TEXT`, + ) + }} +} diff --git a/database/upgrades/upgrades.go b/database/upgrades/upgrades.go index af54f6c..9bcc1c2 100644 --- a/database/upgrades/upgrades.go +++ b/database/upgrades/upgrades.go @@ -39,7 +39,7 @@ type upgrade struct { fn upgradeFunc } -const NumberOfUpgrades = 21 +const NumberOfUpgrades = 24 var upgrades [NumberOfUpgrades]upgrade @@ -68,6 +68,16 @@ func SetVersion(tx *sql.Tx, version int) error { return err } +func execMany(tx *sql.Tx, queries ...string) error { + for _, query := range queries { + _, err := tx.Exec(query) + if err != nil { + return err + } + } + return nil +} + func Run(log log.Logger, dialectName string, db *sql.DB) error { var dialect Dialect switch strings.ToLower(dialectName) { @@ -89,13 +99,14 @@ func Run(log log.Logger, dialectName string, db *sql.DB) error { } log.Infofln("Database currently on v%d, latest: v%d", version, NumberOfUpgrades) - for i, upgrade := range upgrades[version:] { - log.Infofln("Upgrading database to v%d: %s", version+i+1, upgrade.message) - tx, err := db.Begin() + for i, upgradeItem := range upgrades[version:] { + log.Infofln("Upgrading database to v%d: %s", version+i+1, upgradeItem.message) + var tx *sql.Tx + tx, err = db.Begin() if err != nil { return err } - err = upgrade.fn(tx, context{dialect, db, log}) + err = upgradeItem.fn(tx, context{dialect, db, log}) if err != nil { return err } diff --git a/go.mod b/go.mod index 3269800..5ddc241 100644 --- a/go.mod +++ b/go.mod @@ -6,13 +6,13 @@ require ( github.com/Rhymen/go-whatsapp v0.1.0 github.com/gorilla/websocket v1.4.2 github.com/lib/pq v1.10.2 - github.com/mattn/go-sqlite3 v1.14.7 + github.com/mattn/go-sqlite3 v1.14.8 github.com/prometheus/client_golang v1.11.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e gopkg.in/yaml.v2 v2.4.0 maunium.net/go/mauflag v1.0.0 maunium.net/go/maulogger/v2 v2.3.0 - maunium.net/go/mautrix v0.9.19 + maunium.net/go/mautrix v0.9.20 ) -replace github.com/Rhymen/go-whatsapp => github.com/tulir/go-whatsapp v0.5.9 +replace github.com/Rhymen/go-whatsapp => github.com/tulir/go-whatsapp v0.5.10 diff --git a/go.sum b/go.sum index d5fea11..bd24420 100644 --- a/go.sum +++ b/go.sum @@ -80,8 +80,8 @@ github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA= -github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.8 h1:gDp86IdQsN/xWjIEmr9MF6o9mpksUgh0fu+9ByFxzIU= +github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -138,8 +138,8 @@ github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/sjson v1.1.5 h1:wsUceI/XDyZk3J1FUvuuYlK62zJv2HO2Pzb8A5EWdUE= github.com/tidwall/sjson v1.1.5/go.mod h1:VuJzsZnTowhSxWdOgsAnb886i4AjEyTkk7tNtsL7EYE= -github.com/tulir/go-whatsapp v0.5.9 h1:IGM2O877eQx8uZLi2kt/5ZtkwrZ9dh9dSVkyO0S7wCM= -github.com/tulir/go-whatsapp v0.5.9/go.mod h1:7J3IIL3bEQiBJGtiZst1N4PgXHlWIartdVQLe6lcx9A= +github.com/tulir/go-whatsapp v0.5.10 h1:mcN9GuSZ4kLg9RyNfPOhBasapkPFA8nA4wFLEwA4AfE= +github.com/tulir/go-whatsapp v0.5.10/go.mod h1:7J3IIL3bEQiBJGtiZst1N4PgXHlWIartdVQLe6lcx9A= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -217,5 +217,5 @@ maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfk maunium.net/go/maulogger/v2 v2.2.4/go.mod h1:TYWy7wKwz/tIXTpsx8G3mZseIRiC5DoMxSZazOHy68A= maunium.net/go/maulogger/v2 v2.3.0 h1:TMCcO65fLk6+pJXo7sl38tzjzW0KBFgc6JWJMBJp4GE= maunium.net/go/maulogger/v2 v2.3.0/go.mod h1:TYWy7wKwz/tIXTpsx8G3mZseIRiC5DoMxSZazOHy68A= -maunium.net/go/mautrix v0.9.19 h1:8ZoDuijJOKxgEOMDoBN2B6at0Ba7EJpsqWA/5jV7ELw= -maunium.net/go/mautrix v0.9.19/go.mod h1:7IzKfWvpQtN+W2Lzxc0rLvIxFM3ryKX6Ys3S/ZoWbg8= +maunium.net/go/mautrix v0.9.20 h1:MUKnH3etwsoi1K/1jhUg1NG8qcupzrj3w56rDnDF7mc= +maunium.net/go/mautrix v0.9.20/go.mod h1:7IzKfWvpQtN+W2Lzxc0rLvIxFM3ryKX6Ys3S/ZoWbg8= diff --git a/portal.go b/portal.go index 1d8e43a..104a3e6 100644 --- a/portal.go +++ b/portal.go @@ -353,7 +353,6 @@ func (portal *Portal) markHandled(source *User, message *waProto.WebMessageInfo, msg.Sender = message.GetParticipant() } } - msg.Content = message.Message msg.Sent = isSent msg.Insert() @@ -2081,6 +2080,43 @@ func parseGeoURI(uri string) (lat, long float64, err error) { return } +func fallbackQuoteContent() *waProto.Message { + blankString := "" + return &waProto.Message{ + Conversation: &blankString, + } +} + +func loadQuoteContent(sender *User, msg *database.Message) *waProto.Message { + resp, err := sender.Conn.LoadMessagesBefore(msg.Chat.JID, msg.JID, msg.Sender == sender.JID, 1) + if err != nil { + return fallbackQuoteContent() + } + msgList, ok := resp.Content.([]interface{}) + if !ok || len(msgList) == 0 { + return fallbackQuoteContent() + } + wmi, ok := msgList[0].(*waProto.WebMessageInfo) + if !ok { + return fallbackQuoteContent() + } + sender.log.Debugln("Loaded message before %s: %s", msg.JID, wmi.GetKey().GetId()) + resp, err = sender.Conn.LoadMessagesAfter(msg.Chat.JID, wmi.GetKey().GetId(), wmi.GetKey().GetFromMe(), 1) + if err != nil { + return fallbackQuoteContent() + } + msgList, ok = resp.Content.([]interface{}) + if !ok || len(msgList) == 0 { + return fallbackQuoteContent() + } + wmi, ok = msgList[0].(*waProto.WebMessageInfo) + if !ok { + return fallbackQuoteContent() + } + sender.log.Debugln("Loaded message %s", wmi.GetKey().GetId()) + return wmi.GetMessage() +} + func (portal *Portal) convertMatrixMessage(sender *User, evt *event.Event) (*waProto.WebMessageInfo, *User) { content, ok := evt.Content.Parsed.(*event.MessageEventContent) if !ok { @@ -2107,10 +2143,10 @@ func (portal *Portal) convertMatrixMessage(sender *User, evt *event.Event) (*waP if len(replyToID) > 0 { content.RemoveReplyFallback() msg := portal.bridge.DB.Message.GetByMXID(replyToID) - if msg != nil && msg.Content != nil { + if msg != nil { ctxInfo.StanzaId = &msg.JID ctxInfo.Participant = &msg.Sender - ctxInfo.QuotedMessage = msg.Content + ctxInfo.QuotedMessage = loadQuoteContent(sender, msg) } } relaybotFormatted := false