Remove content from message table. Fixes #320

This commit is contained in:
Tulir Asokan 2021-08-19 19:17:19 +03:00
parent b2878d5d34
commit 4fe179d0d7
8 changed files with 177 additions and 52 deletions

View file

@ -17,14 +17,11 @@
package database
import (
"bytes"
"database/sql"
"encoding/json"
"strings"
"time"
"github.com/Rhymen/go-whatsapp"
waProto "github.com/Rhymen/go-whatsapp/binary/proto"
log "maunium.net/go/maulogger/v2"
@ -44,7 +41,7 @@ func (mq *MessageQuery) New() *Message {
}
func (mq *MessageQuery) GetAll(chat PortalKey) (messages []*Message) {
rows, err := mq.db.Query("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent, content FROM message WHERE chat_jid=$1 AND chat_receiver=$2", chat.JID, chat.Receiver)
rows, err := mq.db.Query("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent FROM message WHERE chat_jid=$1 AND chat_receiver=$2", chat.JID, chat.Receiver)
if err != nil || rows == nil {
return nil
}
@ -56,12 +53,12 @@ func (mq *MessageQuery) GetAll(chat PortalKey) (messages []*Message) {
}
func (mq *MessageQuery) GetByJID(chat PortalKey, jid whatsapp.MessageID) *Message {
return mq.get("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent, content "+
return mq.get("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent "+
"FROM message WHERE chat_jid=$1 AND chat_receiver=$2 AND jid=$3", chat.JID, chat.Receiver, jid)
}
func (mq *MessageQuery) GetByMXID(mxid id.EventID) *Message {
return mq.get("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent, content "+
return mq.get("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent "+
"FROM message WHERE mxid=$1", mxid)
}
@ -70,7 +67,7 @@ func (mq *MessageQuery) GetLastInChat(chat PortalKey) *Message {
}
func (mq *MessageQuery) GetLastInChatBefore(chat PortalKey, maxTimestamp int64) *Message {
msg := mq.get("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent, content "+
msg := mq.get("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent "+
"FROM message WHERE chat_jid=$1 AND chat_receiver=$2 AND timestamp<=$3 AND sent=true ORDER BY timestamp DESC LIMIT 1",
chat.JID, chat.Receiver, maxTimestamp)
if msg == nil || msg.Timestamp == 0 {
@ -98,7 +95,6 @@ type Message struct {
Sender whatsapp.JID
Timestamp int64
Sent bool
Content *waProto.Message
}
func (msg *Message) IsFakeMXID() bool {
@ -106,8 +102,7 @@ func (msg *Message) IsFakeMXID() bool {
}
func (msg *Message) Scan(row Scannable) *Message {
var content []byte
err := row.Scan(&msg.Chat.JID, &msg.Chat.Receiver, &msg.JID, &msg.MXID, &msg.Sender, &msg.Timestamp, &msg.Sent, &content)
err := row.Scan(&msg.Chat.JID, &msg.Chat.Receiver, &msg.JID, &msg.MXID, &msg.Sender, &msg.Timestamp, &msg.Sent)
if err != nil {
if err != sql.ErrNoRows {
msg.log.Errorln("Database scan failed:", err)
@ -115,36 +110,14 @@ func (msg *Message) Scan(row Scannable) *Message {
return nil
}
msg.decodeBinaryContent(content)
return msg
}
func (msg *Message) decodeBinaryContent(content []byte) {
msg.Content = &waProto.Message{}
reader := bytes.NewReader(content)
dec := json.NewDecoder(reader)
err := dec.Decode(msg.Content)
if err != nil {
msg.log.Warnln("Failed to decode message content:", err)
}
}
func (msg *Message) encodeBinaryContent() []byte {
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
err := enc.Encode(msg.Content)
if err != nil {
msg.log.Warnln("Failed to encode message content:", err)
}
return buf.Bytes()
}
func (msg *Message) Insert() {
_, err := msg.db.Exec(`INSERT INTO message
(chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent, content)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
msg.Chat.JID, msg.Chat.Receiver, msg.JID, msg.MXID, msg.Sender, msg.Timestamp, msg.Sent, msg.encodeBinaryContent())
(chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent)
VALUES ($1, $2, $3, $4, $5, $6, $7)`,
msg.Chat.JID, msg.Chat.Receiver, msg.JID, msg.MXID, msg.Sender, msg.Timestamp, msg.Sent)
if err != nil {
msg.log.Warnfln("Failed to insert %s@%s: %v", msg.Chat, msg.JID, err)
}

View file

@ -0,0 +1,44 @@
package upgrades
import (
"database/sql"
)
func init() {
upgrades[21] = upgrade{"Remove message content from local database", func(tx *sql.Tx, ctx context) error {
if ctx.dialect == SQLite {
_, err := tx.Exec("ALTER TABLE message RENAME TO old_message")
if err != nil {
return err
}
_, err = tx.Exec(`CREATE TABLE IF NOT EXISTS message (
chat_jid TEXT,
chat_receiver TEXT,
jid TEXT,
mxid TEXT NOT NULL UNIQUE,
sender TEXT NOT NULL,
timestamp BIGINT NOT NULL,
sent BOOLEAN NOT NULL,
PRIMARY KEY (chat_jid, chat_receiver, jid),
FOREIGN KEY (chat_jid, chat_receiver) REFERENCES portal(jid, receiver) ON DELETE CASCADE
)`)
if err != nil {
return err
}
_, err = tx.Exec("INSERT INTO message SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent FROM old_message")
return err
} else {
_, err := tx.Exec(`ALTER TABLE message DROP COLUMN content`)
if err != nil {
return err
}
_, err = tx.Exec(`ALTER TABLE message ALTER COLUMN timestamp DROP DEFAULT`)
if err != nil {
return err
}
_, err = tx.Exec(`ALTER TABLE message ALTER COLUMN sent DROP DEFAULT`)
return err
}
}}
}

View file

@ -0,0 +1,13 @@
package upgrades
import (
"database/sql"
"maunium.net/go/mautrix/crypto/sql_store_upgrade"
)
func init() {
upgrades[23] = upgrade{"Replace VARCHAR(255) with TEXT in the crypto database", func(tx *sql.Tx, ctx context) error {
return sql_store_upgrade.Upgrades[4](tx, ctx.dialect.String())
}}
}

View file

@ -0,0 +1,48 @@
package upgrades
import (
"database/sql"
)
func init() {
upgrades[22] = upgrade{"Replace VARCHAR(255) with TEXT in the database", func(tx *sql.Tx, ctx context) error {
if ctx.dialect == SQLite {
// SQLite doesn't enforce varchar sizes anyway
return nil
}
return execMany(tx,
`ALTER TABLE message ALTER COLUMN chat_jid TYPE TEXT`,
`ALTER TABLE message ALTER COLUMN chat_receiver TYPE TEXT`,
`ALTER TABLE message ALTER COLUMN jid TYPE TEXT`,
`ALTER TABLE message ALTER COLUMN mxid TYPE TEXT`,
`ALTER TABLE message ALTER COLUMN sender TYPE TEXT`,
`ALTER TABLE portal ALTER COLUMN jid TYPE TEXT`,
`ALTER TABLE portal ALTER COLUMN receiver TYPE TEXT`,
`ALTER TABLE portal ALTER COLUMN mxid TYPE TEXT`,
`ALTER TABLE portal ALTER COLUMN name TYPE TEXT`,
`ALTER TABLE portal ALTER COLUMN topic TYPE TEXT`,
`ALTER TABLE portal ALTER COLUMN avatar TYPE TEXT`,
`ALTER TABLE portal ALTER COLUMN avatar_url TYPE TEXT`,
`ALTER TABLE puppet ALTER COLUMN jid TYPE TEXT`,
`ALTER TABLE puppet ALTER COLUMN avatar TYPE TEXT`,
`ALTER TABLE puppet ALTER COLUMN displayname TYPE TEXT`,
`ALTER TABLE puppet ALTER COLUMN custom_mxid TYPE TEXT`,
`ALTER TABLE puppet ALTER COLUMN access_token TYPE TEXT`,
`ALTER TABLE puppet ALTER COLUMN next_batch TYPE TEXT`,
`ALTER TABLE puppet ALTER COLUMN avatar_url TYPE TEXT`,
`ALTER TABLE "user" ALTER COLUMN mxid TYPE TEXT`,
`ALTER TABLE "user" ALTER COLUMN jid TYPE TEXT`,
`ALTER TABLE "user" ALTER COLUMN management_room TYPE TEXT`,
`ALTER TABLE "user" ALTER COLUMN client_id TYPE TEXT`,
`ALTER TABLE "user" ALTER COLUMN client_token TYPE TEXT`,
`ALTER TABLE "user" ALTER COLUMN server_token TYPE TEXT`,
`ALTER TABLE user_portal ALTER COLUMN user_jid TYPE TEXT`,
`ALTER TABLE user_portal ALTER COLUMN portal_jid TYPE TEXT`,
`ALTER TABLE user_portal ALTER COLUMN portal_receiver TYPE TEXT`,
)
}}
}

View file

@ -39,7 +39,7 @@ type upgrade struct {
fn upgradeFunc
}
const NumberOfUpgrades = 21
const NumberOfUpgrades = 24
var upgrades [NumberOfUpgrades]upgrade
@ -68,6 +68,16 @@ func SetVersion(tx *sql.Tx, version int) error {
return err
}
func execMany(tx *sql.Tx, queries ...string) error {
for _, query := range queries {
_, err := tx.Exec(query)
if err != nil {
return err
}
}
return nil
}
func Run(log log.Logger, dialectName string, db *sql.DB) error {
var dialect Dialect
switch strings.ToLower(dialectName) {
@ -89,13 +99,14 @@ func Run(log log.Logger, dialectName string, db *sql.DB) error {
}
log.Infofln("Database currently on v%d, latest: v%d", version, NumberOfUpgrades)
for i, upgrade := range upgrades[version:] {
log.Infofln("Upgrading database to v%d: %s", version+i+1, upgrade.message)
tx, err := db.Begin()
for i, upgradeItem := range upgrades[version:] {
log.Infofln("Upgrading database to v%d: %s", version+i+1, upgradeItem.message)
var tx *sql.Tx
tx, err = db.Begin()
if err != nil {
return err
}
err = upgrade.fn(tx, context{dialect, db, log})
err = upgradeItem.fn(tx, context{dialect, db, log})
if err != nil {
return err
}

6
go.mod
View file

@ -6,13 +6,13 @@ require (
github.com/Rhymen/go-whatsapp v0.1.0
github.com/gorilla/websocket v1.4.2
github.com/lib/pq v1.10.2
github.com/mattn/go-sqlite3 v1.14.7
github.com/mattn/go-sqlite3 v1.14.8
github.com/prometheus/client_golang v1.11.0
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
gopkg.in/yaml.v2 v2.4.0
maunium.net/go/mauflag v1.0.0
maunium.net/go/maulogger/v2 v2.3.0
maunium.net/go/mautrix v0.9.19
maunium.net/go/mautrix v0.9.20
)
replace github.com/Rhymen/go-whatsapp => github.com/tulir/go-whatsapp v0.5.9
replace github.com/Rhymen/go-whatsapp => github.com/tulir/go-whatsapp v0.5.10

12
go.sum
View file

@ -80,8 +80,8 @@ github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8=
github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.8 h1:gDp86IdQsN/xWjIEmr9MF6o9mpksUgh0fu+9ByFxzIU=
github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@ -138,8 +138,8 @@ github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=
github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/sjson v1.1.5 h1:wsUceI/XDyZk3J1FUvuuYlK62zJv2HO2Pzb8A5EWdUE=
github.com/tidwall/sjson v1.1.5/go.mod h1:VuJzsZnTowhSxWdOgsAnb886i4AjEyTkk7tNtsL7EYE=
github.com/tulir/go-whatsapp v0.5.9 h1:IGM2O877eQx8uZLi2kt/5ZtkwrZ9dh9dSVkyO0S7wCM=
github.com/tulir/go-whatsapp v0.5.9/go.mod h1:7J3IIL3bEQiBJGtiZst1N4PgXHlWIartdVQLe6lcx9A=
github.com/tulir/go-whatsapp v0.5.10 h1:mcN9GuSZ4kLg9RyNfPOhBasapkPFA8nA4wFLEwA4AfE=
github.com/tulir/go-whatsapp v0.5.10/go.mod h1:7J3IIL3bEQiBJGtiZst1N4PgXHlWIartdVQLe6lcx9A=
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@ -217,5 +217,5 @@ maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfk
maunium.net/go/maulogger/v2 v2.2.4/go.mod h1:TYWy7wKwz/tIXTpsx8G3mZseIRiC5DoMxSZazOHy68A=
maunium.net/go/maulogger/v2 v2.3.0 h1:TMCcO65fLk6+pJXo7sl38tzjzW0KBFgc6JWJMBJp4GE=
maunium.net/go/maulogger/v2 v2.3.0/go.mod h1:TYWy7wKwz/tIXTpsx8G3mZseIRiC5DoMxSZazOHy68A=
maunium.net/go/mautrix v0.9.19 h1:8ZoDuijJOKxgEOMDoBN2B6at0Ba7EJpsqWA/5jV7ELw=
maunium.net/go/mautrix v0.9.19/go.mod h1:7IzKfWvpQtN+W2Lzxc0rLvIxFM3ryKX6Ys3S/ZoWbg8=
maunium.net/go/mautrix v0.9.20 h1:MUKnH3etwsoi1K/1jhUg1NG8qcupzrj3w56rDnDF7mc=
maunium.net/go/mautrix v0.9.20/go.mod h1:7IzKfWvpQtN+W2Lzxc0rLvIxFM3ryKX6Ys3S/ZoWbg8=

View file

@ -353,7 +353,6 @@ func (portal *Portal) markHandled(source *User, message *waProto.WebMessageInfo,
msg.Sender = message.GetParticipant()
}
}
msg.Content = message.Message
msg.Sent = isSent
msg.Insert()
@ -2081,6 +2080,43 @@ func parseGeoURI(uri string) (lat, long float64, err error) {
return
}
func fallbackQuoteContent() *waProto.Message {
blankString := ""
return &waProto.Message{
Conversation: &blankString,
}
}
func loadQuoteContent(sender *User, msg *database.Message) *waProto.Message {
resp, err := sender.Conn.LoadMessagesBefore(msg.Chat.JID, msg.JID, msg.Sender == sender.JID, 1)
if err != nil {
return fallbackQuoteContent()
}
msgList, ok := resp.Content.([]interface{})
if !ok || len(msgList) == 0 {
return fallbackQuoteContent()
}
wmi, ok := msgList[0].(*waProto.WebMessageInfo)
if !ok {
return fallbackQuoteContent()
}
sender.log.Debugln("Loaded message before %s: %s", msg.JID, wmi.GetKey().GetId())
resp, err = sender.Conn.LoadMessagesAfter(msg.Chat.JID, wmi.GetKey().GetId(), wmi.GetKey().GetFromMe(), 1)
if err != nil {
return fallbackQuoteContent()
}
msgList, ok = resp.Content.([]interface{})
if !ok || len(msgList) == 0 {
return fallbackQuoteContent()
}
wmi, ok = msgList[0].(*waProto.WebMessageInfo)
if !ok {
return fallbackQuoteContent()
}
sender.log.Debugln("Loaded message %s", wmi.GetKey().GetId())
return wmi.GetMessage()
}
func (portal *Portal) convertMatrixMessage(sender *User, evt *event.Event) (*waProto.WebMessageInfo, *User) {
content, ok := evt.Content.Parsed.(*event.MessageEventContent)
if !ok {
@ -2107,10 +2143,10 @@ func (portal *Portal) convertMatrixMessage(sender *User, evt *event.Event) (*waP
if len(replyToID) > 0 {
content.RemoveReplyFallback()
msg := portal.bridge.DB.Message.GetByMXID(replyToID)
if msg != nil && msg.Content != nil {
if msg != nil {
ctxInfo.StanzaId = &msg.JID
ctxInfo.Participant = &msg.Sender
ctxInfo.QuotedMessage = msg.Content
ctxInfo.QuotedMessage = loadQuoteContent(sender, msg)
}
}
relaybotFormatted := false