From be5e44378ff5d36985aaa444acafef23c34cedeb Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 8 Jun 2023 09:21:43 +0100 Subject: [PATCH 01/94] Add a metric for connection failures --- metrics.go | 12 ++++++++++++ user.go | 3 +++ 2 files changed, 15 insertions(+) diff --git a/metrics.go b/metrics.go index 5613b50..6a93b04 100644 --- a/metrics.go +++ b/metrics.go @@ -52,6 +52,7 @@ type MetricsHandler struct { countCollection prometheus.Histogram disconnections *prometheus.CounterVec incomingRetryReceipts *prometheus.CounterVec + connectionFailures *prometheus.CounterVec puppetCount prometheus.Gauge userCount prometheus.Gauge messageCount prometheus.Gauge @@ -101,6 +102,10 @@ func NewMetricsHandler(address string, log log.Logger, db *database.Database) *M Name: "whatsapp_disconnections", Help: "Number of times a Matrix user has been disconnected from WhatsApp", }, []string{"user_id"}), + connectionFailures: promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "whatsapp_connection_failures", + Help: "Number of times a connection has failed to whatsapp", + }, []string{"reason"}), incomingRetryReceipts: promauto.NewCounterVec(prometheus.CounterOpts{ Name: "whatsapp_incoming_retry_receipts", Help: "Number of times a remote WhatsApp user has requested a retry from the bridge. retry_count = 5 is usually the last attempt (and very likely means a failed message)", @@ -173,6 +178,13 @@ func (mh *MetricsHandler) TrackDisconnection(userID id.UserID) { mh.disconnections.With(prometheus.Labels{"user_id": string(userID)}).Inc() } +func (mh *MetricsHandler) TrackConnectionFailure(reason string) { + if !mh.running { + return + } + mh.connectionFailures.With(prometheus.Labels{"reason": reason}).Inc() +} + func (mh *MetricsHandler) TrackRetryReceipt(count int, found bool) { if !mh.running { return diff --git a/user.go b/user.go index 3633be9..7957785 100644 --- a/user.go +++ b/user.go @@ -842,13 +842,16 @@ func (user *User) HandleEvent(event interface{}) { case *events.ConnectFailure: user.BridgeState.Send(status.BridgeState{StateEvent: status.StateUnknownError, Message: fmt.Sprintf("Unknown connection failure: %s", v.Reason)}) user.bridge.Metrics.TrackConnectionState(user.JID, false) + user.bridge.Metrics.TrackConnectionFailure(fmt.Sprintf("status-%d", v.Reason)) case *events.ClientOutdated: user.log.Errorfln("Got a client outdated connect failure. The bridge is likely out of date, please update immediately.") user.BridgeState.Send(status.BridgeState{StateEvent: status.StateUnknownError, Message: "Connect failure: 405 client outdated"}) user.bridge.Metrics.TrackConnectionState(user.JID, false) + user.bridge.Metrics.TrackConnectionFailure("client-outdated") case *events.TemporaryBan: user.BridgeState.Send(status.BridgeState{StateEvent: status.StateBadCredentials, Message: v.String()}) user.bridge.Metrics.TrackConnectionState(user.JID, false) + user.bridge.Metrics.TrackConnectionFailure("temporary-ban") case *events.Disconnected: // Don't send the normal transient disconnect state if we're already in a different transient disconnect state. // TODO remove this if/when the phone offline state is moved to a sub-state of CONNECTED From b97b3755dc4cc02adcf12f61db8cca0783a7e17f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 16 Jun 2023 12:58:57 +0300 Subject: [PATCH 02/94] Add index on message timestamps --- database/upgrades/00-latest-revision.sql | 4 +++- database/upgrades/57-message-timestamp-index.sql | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 database/upgrades/57-message-timestamp-index.sql diff --git a/database/upgrades/00-latest-revision.sql b/database/upgrades/00-latest-revision.sql index 4e1b5ea..dd799f1 100644 --- a/database/upgrades/00-latest-revision.sql +++ b/database/upgrades/00-latest-revision.sql @@ -1,4 +1,4 @@ --- v0 -> v56 (compatible with v45+): Latest revision +-- v0 -> v57 (compatible with v45+): Latest revision CREATE TABLE "user" ( mxid TEXT PRIMARY KEY, @@ -82,6 +82,8 @@ CREATE TABLE message ( FOREIGN KEY (chat_jid, chat_receiver) REFERENCES portal(jid, receiver) ON DELETE CASCADE ); +CREATE INDEX message_timestamp_idx ON message (chat_jid, chat_receiver, timestamp); + CREATE TABLE poll_option_id ( msg_mxid TEXT, opt_id TEXT, diff --git a/database/upgrades/57-message-timestamp-index.sql b/database/upgrades/57-message-timestamp-index.sql new file mode 100644 index 0000000..c6ebe13 --- /dev/null +++ b/database/upgrades/57-message-timestamp-index.sql @@ -0,0 +1,2 @@ +-- v57 (compatible with v45+): Add index for message timestamp to make read receipt handling faster +CREATE INDEX message_timestamp_idx ON message (chat_jid, chat_receiver, timestamp); From 5ebc8313d996420bb43ffa7121ffd028624d8e1a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 16 Jun 2023 14:30:41 +0300 Subject: [PATCH 03/94] Bump version to v0.8.6 --- CHANGELOG.md | 7 ++++++- go.mod | 20 ++++++++++---------- go.sum | 42 +++++++++++++++++++++--------------------- main.go | 2 +- portal.go | 2 +- 5 files changed, 39 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 257676d..9586c9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ -# v0.8.6 (unreleased) +# v0.8.6 (2023-06-16) * Implemented intentional mentions for outgoing messages. +* Added support for appservice websockets. +* Added additional index on message table to make bridging outgoing read + receipts and messages faster in chats with lots of messages. +* Fixed handling WhatsApp poll messages that only allow one choice. +* Fixed bridging new groups immediately when they're created. # v0.8.5 (2023-05-16) diff --git a/go.mod b/go.mod index c2fef41..bb4507f 100644 --- a/go.mod +++ b/go.mod @@ -8,17 +8,17 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.17 - github.com/prometheus/client_golang v1.15.1 + github.com/prometheus/client_golang v1.16.0 github.com/rs/zerolog v1.29.1 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.14.4 - go.mau.fi/whatsmeow v0.0.0-20230608204524-7aedaa1de108 - golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea - golang.org/x/image v0.7.0 - golang.org/x/net v0.10.0 + go.mau.fi/whatsmeow v0.0.0-20230614142319-2114a3c181bd + golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 + golang.org/x/image v0.8.0 + golang.org/x/net v0.11.0 google.golang.org/protobuf v1.30.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.15.3-0.20230609124302-54a73ab22ef9 + maunium.net/go/mautrix v0.15.3 ) require ( @@ -33,7 +33,7 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect @@ -41,9 +41,9 @@ require ( github.com/yuin/goldmark v1.5.4 // indirect go.mau.fi/libsignal v0.1.0 // indirect go.mau.fi/zeroconfig v0.1.2 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/crypto v0.10.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect maunium.net/go/mauflag v1.0.0 // indirect diff --git a/go.sum b/go.sum index cc7a65b..27f85c9 100644 --- a/go.sum +++ b/go.sum @@ -38,14 +38,14 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= -github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -53,7 +53,7 @@ github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -68,26 +68,26 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= -go.mau.fi/whatsmeow v0.0.0-20230608204524-7aedaa1de108 h1:kDOgPHj0urv2vsXyE8dt+KgyC18jxzQW48HGXa53pzc= -go.mau.fi/whatsmeow v0.0.0-20230608204524-7aedaa1de108/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= +go.mau.fi/whatsmeow v0.0.0-20230614142319-2114a3c181bd h1:AsFfDeXdpMSd9gVLWO2vh7IPAN4hyf0b0fN1CNtTJEI= +go.mau.fi/whatsmeow v0.0.0-20230614142319-2114a3c181bd/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea h1:vLCWI/yYrdEHyN2JzIzPO3aaQJHQdp89IZBA/+azVC4= -golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= -golang.org/x/image v0.7.0 h1:gzS29xtG1J5ybQlv0PuyfE3nmc6R4qB73m6LUUmvFuw= -golang.org/x/image v0.7.0/go.mod h1:nd/q4ef1AKKYl/4kft7g+6UyGbdiqWqTP1ZAbRoV7Rg= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/image v0.8.0 h1:agUcRXV/+w6L9ryntYYsF2x9fQTMd4T8fiiYXAVW6Jg= +golang.org/x/image v0.8.0/go.mod h1:PwLxp3opCYg4WR2WO9P0L6ESnsD6bLTWcw8zanLMVFM= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -100,8 +100,8 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -109,8 +109,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -131,5 +131,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.15.3-0.20230609124302-54a73ab22ef9 h1:MixixPn9FMv99V/6wdwDB18HR/1m69JDAc4VcEar8Wc= -maunium.net/go/mautrix v0.15.3-0.20230609124302-54a73ab22ef9/go.mod h1:h4NwfKqE4YxGTLSgn/gawKzXAb2sF4qx8agL6QEFtGg= +maunium.net/go/mautrix v0.15.3 h1:C9BHSUM0gYbuZmAtopuLjIcH5XHLb/ZjTEz7nN+0jN0= +maunium.net/go/mautrix v0.15.3/go.mod h1:zLrQqdxJlLkurRCozTc9CL6FySkgZlO/kpCYxBILSLE= diff --git a/main.go b/main.go index 9656be1..d2710ae 100644 --- a/main.go +++ b/main.go @@ -277,7 +277,7 @@ func main() { Name: "mautrix-whatsapp", URL: "https://github.com/mautrix/whatsapp", Description: "A Matrix-WhatsApp puppeting bridge.", - Version: "0.8.5", + Version: "0.8.6", ProtocolName: "WhatsApp", BeeperServiceName: "whatsapp", BeeperNetworkName: "whatsapp", diff --git a/portal.go b/portal.go index 1e9217b..be3bc20 100644 --- a/portal.go +++ b/portal.go @@ -3018,7 +3018,7 @@ func (portal *Portal) uploadMedia(intent *appservice.IntentAPI, data []byte, con } var mxc id.ContentURI if portal.bridge.Config.Homeserver.AsyncMedia { - uploaded, err := intent.UnstableUploadAsync(req) + uploaded, err := intent.UploadAsync(req) if err != nil { return err } From 8ab06edaca57ba0d726fe1013915d1fa3681436b Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Fri, 16 Jun 2023 14:40:03 -0600 Subject: [PATCH 04/94] deps/whatsomeow: upgrade to add retries on network failures Signed-off-by: Sumner Evans --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bb4507f..194ee44 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/rs/zerolog v1.29.1 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.14.4 - go.mau.fi/whatsmeow v0.0.0-20230614142319-2114a3c181bd + go.mau.fi/whatsmeow v0.0.0-20230616194828-be0edabb0bf3 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 golang.org/x/image v0.8.0 golang.org/x/net v0.11.0 diff --git a/go.sum b/go.sum index 27f85c9..48b53ef 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= -go.mau.fi/whatsmeow v0.0.0-20230614142319-2114a3c181bd h1:AsFfDeXdpMSd9gVLWO2vh7IPAN4hyf0b0fN1CNtTJEI= -go.mau.fi/whatsmeow v0.0.0-20230614142319-2114a3c181bd/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= +go.mau.fi/whatsmeow v0.0.0-20230616194828-be0edabb0bf3 h1:CByCFmtYMsj/4tO+j9YvLLcUa5xnnrDyJXFbl52quhU= +go.mau.fi/whatsmeow v0.0.0-20230616194828-be0edabb0bf3/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= From 2d33bb1673ce5eeeb0d0b637a57f8a9dc73776f8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 19 Jun 2023 02:10:43 +0300 Subject: [PATCH 05/94] Add non-MSC2716 backfill support --- CHANGELOG.md | 5 ++ config/bridge.go | 1 + config/upgrade.go | 1 + example-config.yaml | 21 +++++---- go.mod | 2 +- go.sum | 4 +- historysync.go | 109 +++++++++++++++++++++++++++++++++++--------- main.go | 15 ------ portal.go | 25 ++++++++-- 9 files changed, 130 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9586c9e..4ee24a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# v0.9.0 (unreleased) + +* Removed MSC2716 support. +* Added legacy backfill support. + # v0.8.6 (2023-06-16) * Implemented intentional mentions for outgoing messages. diff --git a/config/bridge.go b/config/bridge.go index 6f3d75f..7b16341 100644 --- a/config/bridge.go +++ b/config/bridge.go @@ -68,6 +68,7 @@ type BridgeConfig struct { StorageQuota uint32 `yaml:"storage_quota_mb"` } MaxInitialConversations int `yaml:"max_initial_conversations"` + MessageCount int `yaml:"message_count"` UnreadHoursThreshold int `yaml:"unread_hours_threshold"` Immediate struct { diff --git a/config/upgrade.go b/config/upgrade.go index d1f4e95..a249a6f 100644 --- a/config/upgrade.go +++ b/config/upgrade.go @@ -56,6 +56,7 @@ func DoUpgrade(helper *up.Helper) { helper.Copy(up.Str, "bridge", "history_sync", "media_requests", "request_method") helper.Copy(up.Int, "bridge", "history_sync", "media_requests", "request_local_time") helper.Copy(up.Int, "bridge", "history_sync", "max_initial_conversations") + helper.Copy(up.Int, "bridge", "history_sync", "message_count") helper.Copy(up.Int, "bridge", "history_sync", "unread_hours_threshold") helper.Copy(up.Int, "bridge", "history_sync", "immediate", "worker_count") helper.Copy(up.Int, "bridge", "history_sync", "immediate", "max_events") diff --git a/example-config.yaml b/example-config.yaml index a9fe4ac..0960098 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -127,12 +127,8 @@ bridge: portal_message_buffer: 128 # Settings for handling history sync payloads. history_sync: - # Enable backfilling history sync payloads from WhatsApp using batch sending? - # This requires a server with MSC2716 support, which is currently an experimental feature in synapse. - # It can be enabled by setting experimental_features -> msc2716_enabled to true in homeserver.yaml. - # Note that prior to Synapse 1.49, there were some bugs with the implementation, especially if using event persistence workers. - # There are also still some issues in Synapse's federation implementation. - backfill: false + # Enable backfilling history sync payloads from WhatsApp? + backfill: true # Should the bridge create portals for chats in the history sync payload? # This has no effect unless backfill is enabled. create_portals: true @@ -171,16 +167,22 @@ bridge: # be sent (in minutes after midnight)? request_local_time: 120 # The maximum number of initial conversations that should be synced. - # Other conversations will be backfilled on demand when the start PM - # provisioning endpoint is used or when a message comes in from that - # chat. + # Other conversations will be backfilled on demand when receiving a message or when initiating a direct chat. max_initial_conversations: -1 + # Number of messages to backfill in each conversation + message_count: 50 # If this value is greater than 0, then if the conversation's last # message was more than this number of hours ago, then the conversation # will automatically be marked it as read. # Conversations that have a last message that is less than this number # of hours ago will have their unread status synced from WhatsApp. unread_hours_threshold: 0 + + ############################################################################### + # The settings below are only applicable for backfilling using batch sending, # + # which is no longer supported in Synapse. # + ############################################################################### + # Settings for immediate backfills. These backfills should generally be # small and their main purpose is to populate each of the initial chats # (as configured by max_initial_conversations) with a few messages so @@ -220,6 +222,7 @@ bridge: - start_days_ago: -1 max_batch_events: 500 batch_delay: 10 + # Should puppet avatars be fetched from the server even if an avatar is already set? user_avatar_sync: true # Should Matrix users leaving groups be bridged to WhatsApp? diff --git a/go.mod b/go.mod index 194ee44..7092376 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( golang.org/x/net v0.11.0 google.golang.org/protobuf v1.30.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.15.3 + maunium.net/go/mautrix v0.15.4-0.20230618223441-8d500be4cbb2 ) require ( diff --git a/go.sum b/go.sum index 48b53ef..4ece6e2 100644 --- a/go.sum +++ b/go.sum @@ -131,5 +131,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.15.3 h1:C9BHSUM0gYbuZmAtopuLjIcH5XHLb/ZjTEz7nN+0jN0= -maunium.net/go/mautrix v0.15.3/go.mod h1:zLrQqdxJlLkurRCozTc9CL6FySkgZlO/kpCYxBILSLE= +maunium.net/go/mautrix v0.15.4-0.20230618223441-8d500be4cbb2 h1:a7xytO4aYZchg/j8vFqYFkxS75eIgJwMsSsqY7FO6kg= +maunium.net/go/mautrix v0.15.4-0.20230618223441-8d500be4cbb2/go.mod h1:zLrQqdxJlLkurRCozTc9CL6FySkgZlO/kpCYxBILSLE= diff --git a/historysync.go b/historysync.go index a6dcf98..888f178 100644 --- a/historysync.go +++ b/historysync.go @@ -60,25 +60,28 @@ func (user *User) handleHistorySyncsLoop() { return } - // Start the backfill queue. - user.BackfillQueue = &BackfillQueue{ - BackfillQuery: user.bridge.DB.Backfill, - reCheckChannels: []chan bool{}, - log: user.log.Sub("BackfillQueue"), + batchSend := user.bridge.SpecVersions.Supports(mautrix.BeeperFeatureBatchSending) + if batchSend { + // Start the backfill queue. + user.BackfillQueue = &BackfillQueue{ + BackfillQuery: user.bridge.DB.Backfill, + reCheckChannels: []chan bool{}, + log: user.log.Sub("BackfillQueue"), + } + + forwardAndImmediate := []database.BackfillType{database.BackfillImmediate, database.BackfillForward} + + // Immediate backfills can be done in parallel + for i := 0; i < user.bridge.Config.Bridge.HistorySync.Immediate.WorkerCount; i++ { + go user.HandleBackfillRequestsLoop(forwardAndImmediate, []database.BackfillType{}) + } + + // Deferred backfills should be handled synchronously so as not to + // overload the homeserver. Users can configure their backfill stages + // to be more or less aggressive with backfilling at this stage. + go user.HandleBackfillRequestsLoop([]database.BackfillType{database.BackfillDeferred}, forwardAndImmediate) } - forwardAndImmediate := []database.BackfillType{database.BackfillImmediate, database.BackfillForward} - - // Immediate backfills can be done in parallel - for i := 0; i < user.bridge.Config.Bridge.HistorySync.Immediate.WorkerCount; i++ { - go user.HandleBackfillRequestsLoop(forwardAndImmediate, []database.BackfillType{}) - } - - // Deferred backfills should be handled synchronously so as not to - // overload the homeserver. Users can configure their backfill stages - // to be more or less aggressive with backfilling at this stage. - go user.HandleBackfillRequestsLoop([]database.BackfillType{database.BackfillDeferred}, forwardAndImmediate) - if user.bridge.Config.Bridge.HistorySync.MediaRequests.AutoRequestMedia && user.bridge.Config.Bridge.HistorySync.MediaRequests.RequestMethod == config.MediaRequestMethodLocalTime { go user.dailyMediaRequestLoop() @@ -92,9 +95,13 @@ func (user *User) handleHistorySyncsLoop() { if evt == nil { return } - user.handleHistorySync(user.BackfillQueue, evt.Data) + user.storeHistorySync(evt.Data) case <-user.enqueueBackfillsTimer.C: - user.enqueueAllBackfills() + if batchSend { + user.enqueueAllBackfills() + } else { + user.backfillAll() + } } } } @@ -125,6 +132,66 @@ func (user *User) enqueueAllBackfills() { } } +func (user *User) backfillAll() { + conversations := user.bridge.DB.HistorySync.GetNMostRecentConversations(user.MXID, -1) + if len(conversations) > 0 { + user.zlog.Info(). + Int("conversation_count", len(conversations)). + Msg("Probably received all history sync blobs, now backfilling conversations") + // Find the portals for all the conversations. + for i, conv := range conversations { + jid, err := types.ParseJID(conv.ConversationID) + if err != nil { + user.zlog.Warn().Err(err). + Str("conversation_id", conv.ConversationID). + Msg("Failed to parse chat JID in history sync") + continue + } + portal := user.GetPortalByJID(jid) + if portal.MXID != "" { + user.zlog.Debug(). + Str("portal_jid", portal.Key.JID.String()). + Msg("Chat already has a room, deleting messages from database") + user.bridge.DB.HistorySync.DeleteAllMessagesForPortal(user.MXID, portal.Key) + } else if i < user.bridge.Config.Bridge.HistorySync.MaxInitialConversations { + err = portal.CreateMatrixRoom(user, nil, true, true) + if err != nil { + user.zlog.Err(err).Msg("Failed to create Matrix room for backfill") + } + } + } + } +} + +func (portal *Portal) legacyBackfill(user *User) { + defer portal.latestEventBackfillLock.Unlock() + // This should only be called from CreateMatrixRoom which locks latestEventBackfillLock before creating the room. + if portal.latestEventBackfillLock.TryLock() { + panic("legacyBackfill() called without locking latestEventBackfillLock") + } + // TODO use portal.zlog instead of user.zlog + log := user.zlog.With(). + Str("portal_jid", portal.Key.JID.String()). + Str("action", "legacy backfill"). + Logger() + messages := user.bridge.DB.HistorySync.GetMessagesBetween(user.MXID, portal.Key.JID.String(), nil, nil, portal.bridge.Config.Bridge.HistorySync.MessageCount) + log.Debug().Int("message_count", len(messages)).Msg("Got messages to backfill from database") + for i := len(messages) - 1; i >= 0; i-- { + msgEvt, err := user.Client.ParseWebMessage(portal.Key.JID, messages[i]) + if err != nil { + log.Warn().Err(err). + Int("msg_index", i). + Str("msg_id", messages[i].GetKey().GetId()). + Uint64("msg_time_seconds", messages[i].GetMessageTimestamp()). + Msg("Dropping historical message due to parse error") + continue + } + portal.handleMessage(user, msgEvt) + } + log.Debug().Msg("Backfill complete, deleting leftover messages from database") + user.bridge.DB.HistorySync.DeleteAllMessagesForPortal(user.MXID, portal.Key) +} + func (user *User) dailyMediaRequestLoop() { // Calculate when to do the first set of media retry requests now := time.Now() @@ -358,12 +425,12 @@ func (user *User) shouldCreatePortalForHistorySync(conv *database.HistorySyncCon } } -func (user *User) handleHistorySync(backfillQueue *BackfillQueue, evt *waProto.HistorySync) { +func (user *User) storeHistorySync(evt *waProto.HistorySync) { if evt == nil || evt.SyncType == nil { return } log := user.bridge.ZLog.With(). - Str("method", "User.handleHistorySync"). + Str("method", "User.storeHistorySync"). Str("user_id", user.MXID.String()). Str("sync_type", evt.GetSyncType().String()). Uint32("chunk_order", evt.GetChunkOrder()). diff --git a/main.go b/main.go index d2710ae..0d41b86 100644 --- a/main.go +++ b/main.go @@ -33,7 +33,6 @@ import ( "go.mau.fi/whatsmeow/store/sqlstore" "go.mau.fi/whatsmeow/types" - "maunium.net/go/mautrix" "maunium.net/go/mautrix/bridge" "maunium.net/go/mautrix/bridge/commands" "maunium.net/go/mautrix/bridge/status" @@ -248,20 +247,6 @@ func (br *WABridge) GetConfigPtr() interface{} { return br.Config } -const unstableFeatureBatchSending = "org.matrix.msc2716" - -func (br *WABridge) CheckFeatures(versions *mautrix.RespVersions) (string, bool) { - if br.Config.Bridge.HistorySync.Backfill { - supported, known := versions.UnstableFeatures[unstableFeatureBatchSending] - if !known { - return "Backfilling is enabled in bridge config, but homeserver does not support MSC2716 batch sending", false - } else if !supported { - return "Backfilling is enabled in bridge config, but MSC2716 batch sending is not enabled on homeserver", false - } - } - return "", true -} - func main() { br := &WABridge{ usersByMXID: make(map[id.UserID]*User), diff --git a/portal.go b/portal.go index be3bc20..b3cffe2 100644 --- a/portal.go +++ b/portal.go @@ -1697,6 +1697,16 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, i if !portal.shouldSetDMRoomMetadata() { req.Name = "" } + legacyBackfill := user.bridge.Config.Bridge.HistorySync.Backfill && backfill && !user.bridge.SpecVersions.Supports(mautrix.BeeperFeatureBatchSending) + var backfillStarted bool + if legacyBackfill { + portal.latestEventBackfillLock.Lock() + defer func() { + if !backfillStarted { + portal.latestEventBackfillLock.Unlock() + } + }() + } resp, err := intent.CreateRoom(req) if err != nil { return err @@ -1758,10 +1768,15 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, i } if user.bridge.Config.Bridge.HistorySync.Backfill && backfill { - portals := []*Portal{portal} - user.EnqueueImmediateBackfills(portals) - user.EnqueueDeferredBackfills(portals) - user.BackfillQueue.ReCheck() + if legacyBackfill { + backfillStarted = true + go portal.legacyBackfill(user) + } else { + portals := []*Portal{portal} + user.EnqueueImmediateBackfills(portals) + user.EnqueueDeferredBackfills(portals) + user.BackfillQueue.ReCheck() + } } return nil } @@ -4491,7 +4506,7 @@ func (portal *Portal) Cleanup(puppetsOnly bool) { return } intent := portal.MainIntent() - if portal.bridge.SpecVersions.UnstableFeatures["com.beeper.room_yeeting"] { + if portal.bridge.SpecVersions.Supports(mautrix.BeeperFeatureRoomYeeting) { err := intent.BeeperDeleteRoom(portal.MXID) if err == nil || errors.Is(err, mautrix.MNotFound) { return From 11beb10b97684806246fa1630fcf0ec8a332c7fc Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 19 Jun 2023 14:38:18 +0300 Subject: [PATCH 06/94] Remove MSC2716 backfill support on Synapse --- config/bridge.go | 8 +- config/config.go | 12 --- config/upgrade.go | 2 - example-config.yaml | 21 ++--- historysync.go | 189 +++++++------------------------------------- 5 files changed, 37 insertions(+), 195 deletions(-) diff --git a/config/bridge.go b/config/bridge.go index 7b16341..2393cef 100644 --- a/config/bridge.go +++ b/config/bridge.go @@ -57,12 +57,10 @@ type BridgeConfig struct { IdentityChangeNotices bool `yaml:"identity_change_notices"` HistorySync struct { - CreatePortals bool `yaml:"create_portals"` - Backfill bool `yaml:"backfill"` + Backfill bool `yaml:"backfill"` - DoublePuppetBackfill bool `yaml:"double_puppet_backfill"` - RequestFullSync bool `yaml:"request_full_sync"` - FullSyncConfig struct { + RequestFullSync bool `yaml:"request_full_sync"` + FullSyncConfig struct { DaysLimit uint32 `yaml:"days_limit"` SizeLimit uint32 `yaml:"size_mb_limit"` StorageQuota uint32 `yaml:"storage_quota_mb"` diff --git a/config/config.go b/config/config.go index 326f859..bb7ec25 100644 --- a/config/config.go +++ b/config/config.go @@ -45,15 +45,3 @@ func (config *Config) CanAutoDoublePuppet(userID id.UserID) bool { _, hasSecret := config.Bridge.LoginSharedSecretMap[homeserver] return hasSecret } - -func (config *Config) CanDoublePuppetBackfill(userID id.UserID) bool { - if !config.Bridge.HistorySync.DoublePuppetBackfill { - return false - } - _, homeserver, _ := userID.Parse() - // Batch sending can only use local users, so don't allow double puppets on other servers. - if homeserver != config.Homeserver.Domain && config.Homeserver.Software != bridgeconfig.SoftwareHungry { - return false - } - return true -} diff --git a/config/upgrade.go b/config/upgrade.go index a249a6f..440c07e 100644 --- a/config/upgrade.go +++ b/config/upgrade.go @@ -45,9 +45,7 @@ func DoUpgrade(helper *up.Helper) { helper.Copy(up.Int, "bridge", "portal_message_buffer") helper.Copy(up.Bool, "bridge", "call_start_notices") helper.Copy(up.Bool, "bridge", "identity_change_notices") - helper.Copy(up.Bool, "bridge", "history_sync", "create_portals") helper.Copy(up.Bool, "bridge", "history_sync", "backfill") - helper.Copy(up.Bool, "bridge", "history_sync", "double_puppet_backfill") helper.Copy(up.Bool, "bridge", "history_sync", "request_full_sync") helper.Copy(up.Int|up.Null, "bridge", "history_sync", "full_sync_config", "days_limit") helper.Copy(up.Int|up.Null, "bridge", "history_sync", "full_sync_config", "size_mb_limit") diff --git a/example-config.yaml b/example-config.yaml index 0960098..b1b4c2d 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -95,7 +95,7 @@ whatsapp: os_name: Mautrix-WhatsApp bridge # Browser name that determines the logo shown in the mobile app. # Must be "unknown" for a generic icon or a valid browser name if you want a specific icon. - # List of valid browser names: https://github.com/tulir/whatsmeow/blob/8b34d886d543b72e5f4699cf5b2797f68d598f78/binary/proto/def.proto#L38-L51 + # List of valid browser names: https://github.com/tulir/whatsmeow/blob/efc632c008604016ddde63bfcfca8de4e5304da9/binary/proto/def.proto#L43-L64 browser_name: unknown # Bridge config @@ -129,14 +129,12 @@ bridge: history_sync: # Enable backfilling history sync payloads from WhatsApp? backfill: true - # Should the bridge create portals for chats in the history sync payload? - # This has no effect unless backfill is enabled. - create_portals: true - # Use double puppets for backfilling? - # In order to use this, the double puppets must be in the appservice's user ID namespace - # (because the bridge can't use the double puppet access token with batch sending). - # This only affects double puppets on the local server, double puppets on other servers will never be used. - double_puppet_backfill: false + # The maximum number of initial conversations that should be synced. + # Other conversations will be backfilled on demand when receiving a message or when initiating a direct chat. + max_initial_conversations: -1 + # Maximum number of messages to backfill in each conversation. + # Set to -1 to disable limit. + message_count: 50 # Should the bridge request a full sync from the phone when logging in? # This bumps the size of history syncs from 3 months to 1 year. request_full_sync: false @@ -166,11 +164,6 @@ bridge: # If request_method is "local_time", what time should the requests # be sent (in minutes after midnight)? request_local_time: 120 - # The maximum number of initial conversations that should be synced. - # Other conversations will be backfilled on demand when receiving a message or when initiating a direct chat. - max_initial_conversations: -1 - # Number of messages to backfill in each conversation - message_count: 50 # If this value is greater than 0, then if the conversation's last # message was more than this number of hours ago, then the conversation # will automatically be marked it as read. diff --git a/historysync.go b/historysync.go index 888f178..f7d9e2a 100644 --- a/historysync.go +++ b/historysync.go @@ -29,7 +29,6 @@ import ( "maunium.net/go/mautrix" "maunium.net/go/mautrix/appservice" - "maunium.net/go/mautrix/bridge/bridgeconfig" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" "maunium.net/go/mautrix/util/dbutil" @@ -242,8 +241,8 @@ func (user *User) backfillInChunks(req *database.Backfill, conv *database.Histor portal.backfillLock.Lock() defer portal.backfillLock.Unlock() - if !user.shouldCreatePortalForHistorySync(conv, portal) { - return + if len(portal.MXID) > 0 && !user.bridge.AS.StateStore.IsInRoom(portal.MXID, user.MXID) { + portal.ensureUserInvited(user) } backfillState := user.bridge.DB.Backfill.GetBackfillState(user.MXID, &portal.Key) @@ -253,19 +252,17 @@ func (user *User) backfillInChunks(req *database.Backfill, conv *database.Histor backfillState.SetProcessingBatch(true) defer backfillState.SetProcessingBatch(false) - var forwardPrevID id.EventID var timeEnd *time.Time - var isLatestEvents, shouldMarkAsRead, shouldAtomicallyMarkAsRead bool + var forward, shouldMarkAsRead bool portal.latestEventBackfillLock.Lock() if req.BackfillType == database.BackfillForward { // TODO this overrides the TimeStart set when enqueuing the backfill // maybe the enqueue should instead include the prev event ID lastMessage := portal.bridge.DB.Message.GetLastInChat(portal.Key) - forwardPrevID = lastMessage.MXID start := lastMessage.Timestamp.Add(1 * time.Second) req.TimeStart = &start // Sending events at the end of the room (= latest events) - isLatestEvents = true + forward = true } else { firstMessage := portal.bridge.DB.Message.GetFirstInChat(portal.Key) if firstMessage != nil { @@ -274,10 +271,10 @@ func (user *User) backfillInChunks(req *database.Backfill, conv *database.Histor user.log.Debugfln("Limiting backfill to end at %v", end) } else { // Portal is empty -> events are latest - isLatestEvents = true + forward = true } } - if !isLatestEvents { + if !forward { // We'll use normal batch sending, so no need to keep blocking new message processing portal.latestEventBackfillLock.Unlock() } else { @@ -288,7 +285,6 @@ func (user *User) backfillInChunks(req *database.Backfill, conv *database.Histor isUnread := conv.MarkedAsUnread || conv.UnreadCount > 0 isTooOld := user.bridge.Config.Bridge.HistorySync.UnreadHoursThreshold > 0 && conv.LastMessageTimestamp.Before(time.Now().Add(time.Duration(-user.bridge.Config.Bridge.HistorySync.UnreadHoursThreshold)*time.Hour)) shouldMarkAsRead = !isUnread || isTooOld - shouldAtomicallyMarkAsRead = shouldMarkAsRead && user.bridge.Config.Homeserver.Software == bridgeconfig.SoftwareHungry } allMsgs := user.bridge.DB.HistorySync.GetMessagesBetween(user.MXID, conv.ConversationID, req.TimeStart, timeEnd, req.MaxTotalEvents) @@ -347,7 +343,6 @@ func (user *User) backfillInChunks(req *database.Backfill, conv *database.Histor user.log.Infofln("Backfilling %d messages in %s, %d messages at a time (queue ID: %d)", len(allMsgs), portal.Key.JID, req.MaxBatchEvents, req.QueueID) toBackfill := allMsgs[0:] - var insertionEventIds []id.EventID for len(toBackfill) > 0 { var msgs []*waProto.WebMessageInfo if len(toBackfill) <= req.MaxBatchEvents || req.MaxBatchEvents < 0 { @@ -361,19 +356,10 @@ func (user *User) backfillInChunks(req *database.Backfill, conv *database.Histor if len(msgs) > 0 { time.Sleep(time.Duration(req.BatchDelay) * time.Second) user.log.Debugfln("Backfilling %d messages in %s (queue ID: %d)", len(msgs), portal.Key.JID, req.QueueID) - resp := portal.backfill(user, msgs, req.BackfillType == database.BackfillForward, isLatestEvents, shouldAtomicallyMarkAsRead, forwardPrevID) - if resp != nil && (resp.BaseInsertionEventID != "" || !isLatestEvents) { - insertionEventIds = append(insertionEventIds, resp.BaseInsertionEventID) - } + portal.backfill(user, msgs, forward, shouldMarkAsRead) } } user.log.Debugfln("Finished backfilling %d messages in %s (queue ID: %d)", len(allMsgs), portal.Key.JID, req.QueueID) - if len(insertionEventIds) > 0 { - portal.sendPostBackfillDummy( - time.Unix(int64(allMsgs[0].GetMessageTimestamp()), 0), - insertionEventIds[0]) - } - user.log.Debugfln("Deleting %d history sync messages after backfilling (queue ID: %d)", len(allMsgs), req.QueueID) err := user.bridge.DB.HistorySync.DeleteMessages(user.MXID, conv.ConversationID, allMsgs) if err != nil { user.log.Warnfln("Failed to delete %d history sync messages after backfilling (queue ID: %d): %v", len(allMsgs), req.QueueID, err) @@ -399,30 +385,6 @@ func (user *User) backfillInChunks(req *database.Backfill, conv *database.Histor backfillState.Upsert() portal.updateBackfillStatus(backfillState) } - - if isLatestEvents && !shouldAtomicallyMarkAsRead { - if shouldMarkAsRead { - user.markSelfReadFull(portal) - } else if conv.MarkedAsUnread && user.bridge.Config.Bridge.SyncManualMarkedUnread { - user.markUnread(portal, true) - } - } -} - -func (user *User) shouldCreatePortalForHistorySync(conv *database.HistorySyncConversation, portal *Portal) bool { - if len(portal.MXID) > 0 { - if !user.bridge.AS.StateStore.IsInRoom(portal.MXID, user.MXID) { - portal.ensureUserInvited(user) - } - // Portal exists, let backfill continue - return true - } else if !user.bridge.Config.Bridge.HistorySync.CreatePortals { - user.log.Debugfln("Not creating portal for %s: creating rooms from history sync is disabled", portal.Key.JID) - return false - } else { - // Portal doesn't exist, but should be created - return true - } } func (user *User) storeHistorySync(evt *waProto.HistorySync) { @@ -627,69 +589,20 @@ func (portal *Portal) deterministicEventID(sender types.JID, messageID types.Mes var ( PortalCreationDummyEvent = event.Type{Type: "fi.mau.dummy.portal_created", Class: event.MessageEventType} - PreBackfillDummyEvent = event.Type{Type: "fi.mau.dummy.pre_backfill", Class: event.MessageEventType} - - HistorySyncMarker = event.Type{Type: "org.matrix.msc2716.marker", Class: event.MessageEventType} BackfillStatusEvent = event.Type{Type: "com.beeper.backfill_status", Class: event.StateEventType} ) -func (portal *Portal) backfill(source *User, messages []*waProto.WebMessageInfo, isForward, isLatest, atomicMarkAsRead bool, prevEventID id.EventID) *mautrix.RespBatchSend { - var req mautrix.ReqBatchSend +func (portal *Portal) backfill(source *User, messages []*waProto.WebMessageInfo, isForward, atomicMarkAsRead bool) *mautrix.RespBeeperBatchSend { + var req mautrix.ReqBeeperBatchSend var infos []*wrappedInfo - if !isForward { - if portal.FirstEventID != "" || portal.NextBatchID != "" { - req.PrevEventID = portal.FirstEventID - req.BatchID = portal.NextBatchID - } else { - portal.log.Warnfln("Can't backfill %d messages through %s to chat: first event ID not known", len(messages), source.MXID) - return nil - } - } else { - req.PrevEventID = prevEventID - } - req.BeeperNewMessages = isLatest && req.BatchID == "" + req.Forward = isForward if atomicMarkAsRead { - req.BeeperMarkReadBy = source.MXID + req.MarkReadBy = source.MXID } - beforeFirstMessageTimestampMillis := (int64(messages[len(messages)-1].GetMessageTimestamp()) * 1000) - 1 - req.StateEventsAtStart = make([]*event.Event, 0) - - addedMembers := make(map[id.UserID]struct{}) - addMember := func(puppet *Puppet) { - if portal.bridge.Config.Homeserver.Software == bridgeconfig.SoftwareHungry { - // Hungryserv doesn't need state_events_at_start, it can figure out memberships automatically - return - } else if _, alreadyAdded := addedMembers[puppet.MXID]; alreadyAdded { - return - } - mxid := puppet.MXID.String() - content := event.MemberEventContent{ - Membership: event.MembershipJoin, - Displayname: puppet.Displayname, - AvatarURL: puppet.AvatarURL.CUString(), - } - inviteContent := content - inviteContent.Membership = event.MembershipInvite - req.StateEventsAtStart = append(req.StateEventsAtStart, &event.Event{ - Type: event.StateMember, - Sender: portal.MainIntent().UserID, - StateKey: &mxid, - Timestamp: beforeFirstMessageTimestampMillis, - Content: event.Content{Parsed: &inviteContent}, - }, &event.Event{ - Type: event.StateMember, - Sender: puppet.MXID, - StateKey: &mxid, - Timestamp: beforeFirstMessageTimestampMillis, - Content: event.Content{Parsed: &content}, - }) - addedMembers[puppet.MXID] = struct{}{} - } - - portal.log.Infofln("Processing history sync with %d messages (forward: %t, latest: %t, prev: %s, batch: %s)", len(messages), isForward, isLatest, req.PrevEventID, req.BatchID) + portal.log.Infofln("Processing history sync with %d messages (forward: %t)", len(messages), isForward) // The messages are ordered newest to oldest, so iterate them in reverse order. for i := len(messages) - 1; i >= 0; i-- { webMsg := messages[i] @@ -720,19 +633,12 @@ func (portal *Portal) backfill(source *User, messages []*waProto.WebMessageInfo, if puppet == nil { continue } - intent := puppet.IntentFor(portal) - if intent.IsCustomPuppet && !portal.bridge.Config.CanDoublePuppetBackfill(puppet.CustomMXID) { - intent = puppet.DefaultIntent() - } - converted := portal.convertMessage(intent, source, &msgEvt.Info, msgEvt.Message, true) + converted := portal.convertMessage(puppet.IntentFor(portal), source, &msgEvt.Info, msgEvt.Message, true) if converted == nil { portal.log.Debugfln("Skipping unsupported message %s in backfill", msgEvt.Info.ID) continue } - if !intent.IsCustomPuppet && !portal.bridge.StateStore.IsInRoom(portal.MXID, puppet.MXID) { - addMember(puppet) - } if converted.ReplyTo != nil { portal.SetReply(converted.Content, converted.ReplyTo, true) } @@ -747,15 +653,7 @@ func (portal *Portal) backfill(source *User, messages []*waProto.WebMessageInfo, return nil } - if len(req.BatchID) == 0 || isForward { - portal.log.Debugln("Sending a dummy event to avoid forward extremity errors with backfill") - _, err := portal.MainIntent().SendMessageEvent(portal.MXID, PreBackfillDummyEvent, struct{}{}) - if err != nil { - portal.log.Warnln("Error sending pre-backfill dummy event:", err) - } - } - - resp, err := portal.MainIntent().BatchSend(portal.MXID, &req) + resp, err := portal.MainIntent().BeeperBatchSend(portal.MXID, &req) if err != nil { portal.log.Errorln("Error batch sending messages:", err) return nil @@ -766,12 +664,7 @@ func (portal *Portal) backfill(source *User, messages []*waProto.WebMessageInfo, return nil } - // Do the following block in the transaction - { - portal.finishBatch(txn, resp.EventIDs, infos) - portal.NextBatchID = resp.NextBatchID - portal.Update(txn) - } + portal.finishBatch(txn, resp.EventIDs, infos) err = txn.Commit() if err != nil { @@ -846,19 +739,16 @@ func (portal *Portal) appendBatchEvents(source *User, converted *ConvertedMessag *infoArray = append(*infoArray, nil) } } - // Sending reactions in the same batch requires deterministic event IDs, so only do it on hungryserv - if portal.bridge.Config.Homeserver.Software == bridgeconfig.SoftwareHungry { - for _, reaction := range raw.GetReactions() { - reactionEvent, reactionInfo := portal.wrapBatchReaction(source, reaction, mainEvt.ID, info.Timestamp) - if reactionEvent != nil { - *eventsArray = append(*eventsArray, reactionEvent) - *infoArray = append(*infoArray, &wrappedInfo{ - MessageInfo: reactionInfo, - SenderMXID: reactionEvent.Sender, - ReactionTarget: info.ID, - Type: database.MsgReaction, - }) - } + for _, reaction := range raw.GetReactions() { + reactionEvent, reactionInfo := portal.wrapBatchReaction(source, reaction, mainEvt.ID, info.Timestamp) + if reactionEvent != nil { + *eventsArray = append(*eventsArray, reactionEvent) + *infoArray = append(*infoArray, &wrappedInfo{ + MessageInfo: reactionInfo, + SenderMXID: reactionEvent.Sender, + ReactionTarget: info.ID, + Type: database.MsgReaction, + }) } } return nil @@ -923,13 +813,8 @@ func (portal *Portal) wrapBatchEvent(info *types.MessageInfo, intent *appservice return nil, err } intent.AddDoublePuppetValue(&wrappedContent) - var eventID id.EventID - if portal.bridge.Config.Homeserver.Software == bridgeconfig.SoftwareHungry { - eventID = portal.deterministicEventID(info.Sender, info.ID, partName) - } - return &event.Event{ - ID: eventID, + ID: portal.deterministicEventID(info.Sender, info.ID, partName), Sender: intent.UserID, Type: newEventType, Timestamp: info.Timestamp.UnixMilli(), @@ -956,33 +841,13 @@ func (portal *Portal) finishBatch(txn dbutil.Transaction, eventIDs []id.EventID, portal.log.Infofln("Successfully sent %d events", len(eventIDs)) } -func (portal *Portal) sendPostBackfillDummy(lastTimestamp time.Time, insertionEventId id.EventID) { - resp, err := portal.MainIntent().SendMessageEvent(portal.MXID, HistorySyncMarker, map[string]interface{}{ - "org.matrix.msc2716.marker.insertion": insertionEventId, - //"m.marker.insertion": insertionEventId, - }) - if err != nil { - portal.log.Errorln("Error sending post-backfill dummy event:", err) - return - } - msg := portal.bridge.DB.Message.New() - msg.Chat = portal.Key - msg.MXID = resp.EventID - msg.SenderMXID = portal.MainIntent().UserID - msg.JID = types.MessageID(resp.EventID) - msg.Timestamp = lastTimestamp.Add(1 * time.Second) - msg.Sent = true - msg.Type = database.MsgFake - msg.Insert(nil) -} - func (portal *Portal) updateBackfillStatus(backfillState *database.BackfillState) { backfillStatus := "backfilling" if backfillState.BackfillComplete { backfillStatus = "complete" } - _, err := portal.MainIntent().SendStateEvent(portal.MXID, BackfillStatusEvent, "", map[string]interface{}{ + _, err := portal.bridge.Bot.SendStateEvent(portal.MXID, BackfillStatusEvent, "", map[string]interface{}{ "status": backfillStatus, "first_timestamp": backfillState.FirstExpectedTimestamp * 1000, }) From a57f4a67beb090173a9c81b6c0896e416dab687a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 19 Jun 2023 14:48:26 +0300 Subject: [PATCH 07/94] Check more things via /versions flags --- go.mod | 2 +- go.sum | 4 ++-- portal.go | 6 +++--- puppet.go | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 7092376..384adc8 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( golang.org/x/net v0.11.0 google.golang.org/protobuf v1.30.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.15.4-0.20230618223441-8d500be4cbb2 + maunium.net/go/mautrix v0.15.4-0.20230619114441-3e840e962e24 ) require ( diff --git a/go.sum b/go.sum index 4ece6e2..21fe37b 100644 --- a/go.sum +++ b/go.sum @@ -131,5 +131,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.15.4-0.20230618223441-8d500be4cbb2 h1:a7xytO4aYZchg/j8vFqYFkxS75eIgJwMsSsqY7FO6kg= -maunium.net/go/mautrix v0.15.4-0.20230618223441-8d500be4cbb2/go.mod h1:zLrQqdxJlLkurRCozTc9CL6FySkgZlO/kpCYxBILSLE= +maunium.net/go/mautrix v0.15.4-0.20230619114441-3e840e962e24 h1:caWWJEYs2G4eLbQ5y7RIrCPUgPU8m1nj6jeyGZAuYVM= +maunium.net/go/mautrix v0.15.4-0.20230619114441-3e840e962e24/go.mod h1:zLrQqdxJlLkurRCozTc9CL6FySkgZlO/kpCYxBILSLE= diff --git a/portal.go b/portal.go index b3cffe2..35194d5 100644 --- a/portal.go +++ b/portal.go @@ -1667,7 +1667,7 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, i }, }) } - autoJoinInvites := portal.bridge.Config.Homeserver.Software == bridgeconfig.SoftwareHungry + autoJoinInvites := portal.bridge.SpecVersions.Supports(mautrix.BeeperFeatureAutojoinInvites) if autoJoinInvites { portal.log.Debugfln("Hungryserv mode: adding all group members in create request") if groupInfo != nil { @@ -1916,7 +1916,7 @@ func (portal *Portal) addReplyMention(content *event.MessageEventContent, sender } } -func (portal *Portal) SetReply(content *event.MessageEventContent, replyTo *ReplyInfo, isBackfill bool) bool { +func (portal *Portal) SetReply(content *event.MessageEventContent, replyTo *ReplyInfo, isHungryBackfill bool) bool { if replyTo == nil { return false } @@ -1942,7 +1942,7 @@ func (portal *Portal) SetReply(content *event.MessageEventContent, replyTo *Repl } message := portal.bridge.DB.Message.GetByJID(key, replyTo.MessageID) if message == nil || message.IsFakeMXID() { - if isBackfill && portal.bridge.Config.Homeserver.Software == bridgeconfig.SoftwareHungry { + if isHungryBackfill { content.RelatesTo = (&event.RelatesTo{}).SetReplyTo(targetPortal.deterministicEventID(replyTo.Sender, replyTo.MessageID, "")) portal.addReplyMention(content, replyTo.Sender, "") return true diff --git a/puppet.go b/puppet.go index 7f2d46e..e1e45d3 100644 --- a/puppet.go +++ b/puppet.go @@ -26,9 +26,9 @@ import ( log "maunium.net/go/maulogger/v2" + "maunium.net/go/mautrix" "maunium.net/go/mautrix/appservice" "maunium.net/go/mautrix/bridge" - "maunium.net/go/mautrix/bridge/bridgeconfig" "maunium.net/go/mautrix/id" "maunium.net/go/mautrix-whatsapp/config" @@ -264,7 +264,7 @@ func (puppet *Puppet) UpdateName(contact types.ContactInfo, forcePortalSync bool } func (puppet *Puppet) UpdateContactInfo() bool { - if puppet.bridge.Config.Homeserver.Software != bridgeconfig.SoftwareHungry { + if !puppet.bridge.SpecVersions.Supports(mautrix.BeeperFeatureArbitraryProfileMeta) { return false } From 5892169dd07c4b803d336fc07a0e332cf30f2178 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 19 Jun 2023 21:06:35 +0300 Subject: [PATCH 08/94] Ignore @lid chats in history syncs --- historysync.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/historysync.go b/historysync.go index f7d9e2a..ec9369c 100644 --- a/historysync.go +++ b/historysync.go @@ -430,6 +430,9 @@ func (user *User) storeHistorySync(evt *waProto.HistorySync) { } else if jid.Server == types.BroadcastServer { log.Debug().Str("chat_jid", jid.String()).Msg("Skipping broadcast list in history sync") continue + } else if jid.Server == types.HiddenUserServer { + log.Debug().Str("chat_jid", jid.String()).Msg("Skipping hidden user JID chat in history sync") + continue } totalMessageCount += len(conv.GetMessages()) portal := user.GetPortalByJID(jid) From c850f6f3739a1420cc03ed69439d30d903aff586 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 19 Jun 2023 21:12:36 +0300 Subject: [PATCH 09/94] Fix things in legacy backfill --- backfillqueue.go | 2 +- database/historysync.go | 13 ++++++++++++- historysync.go | 13 +++++++++++-- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/backfillqueue.go b/backfillqueue.go index 51e50c2..ab79644 100644 --- a/backfillqueue.go +++ b/backfillqueue.go @@ -65,7 +65,7 @@ func (user *User) HandleBackfillRequestsLoop(backfillTypes []database.BackfillTy req := user.BackfillQueue.GetNextBackfill(user.MXID, backfillTypes, waitForBackfillTypes, reCheckChannel) user.log.Infofln("Handling backfill request %s", req) - conv := user.bridge.DB.HistorySync.GetConversation(user.MXID, req.Portal) + conv := user.bridge.DB.HistorySync.GetConversation(user.MXID, *req.Portal) if conv == nil { user.log.Debugfln("Could not find history sync conversation data for %s", req.Portal.String()) req.MarkDone() diff --git a/database/historysync.go b/database/historysync.go index 214440b..4d9d0bf 100644 --- a/database/historysync.go +++ b/database/historysync.go @@ -183,7 +183,7 @@ func (hsq *HistorySyncQuery) GetNMostRecentConversations(userID id.UserID, n int return } -func (hsq *HistorySyncQuery) GetConversation(userID id.UserID, portalKey *PortalKey) (conversation *HistorySyncConversation) { +func (hsq *HistorySyncQuery) GetConversation(userID id.UserID, portalKey PortalKey) (conversation *HistorySyncConversation) { rows, err := hsq.db.Query(getConversationByPortal, userID, portalKey.JID, portalKey.Receiver) defer rows.Close() if err != nil || rows == nil { @@ -323,3 +323,14 @@ func (hsq *HistorySyncQuery) DeleteAllMessagesForPortal(userID id.UserID, portal hsq.log.Warnfln("Failed to delete historical messages for %s/%s: %v", userID, portalKey.JID, err) } } + +func (hsq *HistorySyncQuery) DeleteConversation(userID id.UserID, jid string) { + // This will also clear history_sync_message as there's a foreign key constraint + _, err := hsq.db.Exec(` + DELETE FROM history_sync_conversation + WHERE user_mxid=$1 AND conversation_id=$2 + `, userID, jid) + if err != nil { + hsq.log.Warnfln("Failed to delete historical messages for %s/%s: %v", userID, jid, err) + } +} diff --git a/historysync.go b/historysync.go index ec9369c..e09d1b4 100644 --- a/historysync.go +++ b/historysync.go @@ -151,7 +151,7 @@ func (user *User) backfillAll() { user.zlog.Debug(). Str("portal_jid", portal.Key.JID.String()). Msg("Chat already has a room, deleting messages from database") - user.bridge.DB.HistorySync.DeleteAllMessagesForPortal(user.MXID, portal.Key) + user.bridge.DB.HistorySync.DeleteConversation(user.MXID, portal.Key.JID.String()) } else if i < user.bridge.Config.Bridge.HistorySync.MaxInitialConversations { err = portal.CreateMatrixRoom(user, nil, true, true) if err != nil { @@ -173,6 +173,7 @@ func (portal *Portal) legacyBackfill(user *User) { Str("portal_jid", portal.Key.JID.String()). Str("action", "legacy backfill"). Logger() + conv := user.bridge.DB.HistorySync.GetConversation(user.MXID, portal.Key) messages := user.bridge.DB.HistorySync.GetMessagesBetween(user.MXID, portal.Key.JID.String(), nil, nil, portal.bridge.Config.Bridge.HistorySync.MessageCount) log.Debug().Int("message_count", len(messages)).Msg("Got messages to backfill from database") for i := len(messages) - 1; i >= 0; i-- { @@ -187,8 +188,16 @@ func (portal *Portal) legacyBackfill(user *User) { } portal.handleMessage(user, msgEvt) } + if conv != nil { + isUnread := conv.MarkedAsUnread || conv.UnreadCount > 0 + isTooOld := user.bridge.Config.Bridge.HistorySync.UnreadHoursThreshold > 0 && conv.LastMessageTimestamp.Before(time.Now().Add(time.Duration(-user.bridge.Config.Bridge.HistorySync.UnreadHoursThreshold)*time.Hour)) + shouldMarkAsRead := !isUnread || isTooOld + if shouldMarkAsRead { + user.markSelfReadFull(portal) + } + } log.Debug().Msg("Backfill complete, deleting leftover messages from database") - user.bridge.DB.HistorySync.DeleteAllMessagesForPortal(user.MXID, portal.Key) + user.bridge.DB.HistorySync.DeleteConversation(user.MXID, portal.Key.JID.String()) } func (user *User) dailyMediaRequestLoop() { From ccd43b2080c3855094dff65e45000d4eacda1f5e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 19 Jun 2023 21:20:52 +0300 Subject: [PATCH 10/94] Move media request config and fix line wrapping --- example-config.yaml | 61 +++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/example-config.yaml b/example-config.yaml index b1b4c2d..b247c05 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -148,27 +148,10 @@ bridge: size_mb_limit: null # This is presumably the local storage quota, which may affect what the phone includes in the history sync blob. storage_quota_mb: null - # Settings for media requests. If the media expired, then it will not - # be on the WA servers. - # Media can always be requested by reacting with the ♻️ (recycle) emoji. - # These settings determine if the media requests should be done - # automatically during or after backfill. - media_requests: - # Should expired media be automatically requested from the server as - # part of the backfill process? - auto_request_media: true - # Whether to request the media immediately after the media message - # is backfilled ("immediate") or at a specific time of the day - # ("local_time"). - request_method: immediate - # If request_method is "local_time", what time should the requests - # be sent (in minutes after midnight)? - request_local_time: 120 - # If this value is greater than 0, then if the conversation's last - # message was more than this number of hours ago, then the conversation - # will automatically be marked it as read. - # Conversations that have a last message that is less than this number - # of hours ago will have their unread status synced from WhatsApp. + # If this value is greater than 0, then if the conversation's last message was more than + # this number of hours ago, then the conversation will automatically be marked it as read. + # Conversations that have a last message that is less than this number of hours ago will + # have their unread status synced from WhatsApp. unread_hours_threshold: 0 ############################################################################### @@ -176,24 +159,32 @@ bridge: # which is no longer supported in Synapse. # ############################################################################### - # Settings for immediate backfills. These backfills should generally be - # small and their main purpose is to populate each of the initial chats - # (as configured by max_initial_conversations) with a few messages so - # that you can continue conversations without loosing context. + # Settings for media requests. If the media expired, then it will not be on the WA servers. + # Media can always be requested by reacting with the ♻️ (recycle) emoji. + # These settings determine if the media requests should be done automatically during or after backfill. + media_requests: + # Should expired media be automatically requested from the server as part of the backfill process? + auto_request_media: true + # Whether to request the media immediately after the media message is backfilled ("immediate") + # or at a specific time of the day ("local_time"). + request_method: immediate + # If request_method is "local_time", what time should the requests be sent (in minutes after midnight)? + request_local_time: 120 + # Settings for immediate backfills. These backfills should generally be small and their main purpose is + # to populate each of the initial chats (as configured by max_initial_conversations) with a few messages + # so that you can continue conversations without losing context. immediate: - # The number of concurrent backfill workers to create for immediate - # backfills. Note that using more than one worker could cause the - # room list to jump around since there are no guarantees about the - # order in which the backfills will complete. + # The number of concurrent backfill workers to create for immediate backfills. + # Note that using more than one worker could cause the room list to jump around + # since there are no guarantees about the order in which the backfills will complete. worker_count: 1 # The maximum number of events to backfill initially. max_events: 10 - # Settings for deferred backfills. The purpose of these backfills are - # to fill in the rest of the chat history that was not covered by the - # immediate backfills. These backfills generally should happen at a - # slower pace so as not to overload the homeserver. - # Each deferred backfill config should define a "stage" of backfill - # (i.e. the last week of messages). The fields are as follows: + # Settings for deferred backfills. The purpose of these backfills are to fill in the rest of + # the chat history that was not covered by the immediate backfills. + # These backfills generally should happen at a slower pace so as not to overload the homeserver. + # Each deferred backfill config should define a "stage" of backfill (i.e. the last week of messages). + # The fields are as follows: # - start_days_ago: the number of days ago to start backfilling from. # To indicate the start of time, use -1. For example, for a week ago, use 7. # - max_batch_events: the number of events to send per batch. From eddd6c87bf9808d8d6a519b346851375bd9e7e42 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 21 Jun 2023 10:35:39 +0300 Subject: [PATCH 11/94] Remove confusing backfill command --- commands.go | 43 ------------------------------------------- 1 file changed, 43 deletions(-) diff --git a/commands.go b/commands.go index b7bc65c..a328533 100644 --- a/commands.go +++ b/commands.go @@ -41,8 +41,6 @@ import ( "maunium.net/go/mautrix/bridge/status" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" - - "maunium.net/go/mautrix-whatsapp/database" ) type WrappedCommandEvent struct { @@ -71,7 +69,6 @@ func (br *WABridge) RegisterCommands() { cmdPing, cmdDeletePortal, cmdDeleteAllPortals, - cmdBackfill, cmdList, cmdSearch, cmdOpen, @@ -787,46 +784,6 @@ func fnDeleteAllPortals(ce *WrappedCommandEvent) { }() } -var cmdBackfill = &commands.FullHandler{ - Func: wrapCommand(fnBackfill), - Name: "backfill", - Help: commands.HelpMeta{ - Section: HelpSectionPortalManagement, - Description: "Backfill all messages the portal.", - Args: "[_batch size_] [_batch delay_]", - }, - RequiresPortal: true, -} - -func fnBackfill(ce *WrappedCommandEvent) { - if !ce.Bridge.Config.Bridge.HistorySync.Backfill { - ce.Reply("Backfill is not enabled for this bridge.") - return - } - batchSize := 100 - batchDelay := 5 - if len(ce.Args) >= 1 { - var err error - batchSize, err = strconv.Atoi(ce.Args[0]) - if err != nil || batchSize < 1 { - ce.Reply("\"%s\" isn't a valid batch size", ce.Args[0]) - return - } - } - if len(ce.Args) >= 2 { - var err error - batchDelay, err = strconv.Atoi(ce.Args[0]) - if err != nil || batchSize < 0 { - ce.Reply("\"%s\" isn't a valid batch delay", ce.Args[1]) - return - } - } - backfillMessages := ce.Portal.bridge.DB.Backfill.NewWithValues(ce.User.MXID, database.BackfillImmediate, 0, &ce.Portal.Key, nil, batchSize, -1, batchDelay) - backfillMessages.Insert() - - ce.User.BackfillQueue.ReCheck() -} - func matchesQuery(str string, query string) bool { if query == "" { return true From 8d93f7c0368e785154e2a8c858e5379bceee3e64 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 22 Jun 2023 12:48:00 +0300 Subject: [PATCH 12/94] Fix handling max_initial_conversations being -1 in legacy backfill --- historysync.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/historysync.go b/historysync.go index e09d1b4..742c9b2 100644 --- a/historysync.go +++ b/historysync.go @@ -137,6 +137,7 @@ func (user *User) backfillAll() { user.zlog.Info(). Int("conversation_count", len(conversations)). Msg("Probably received all history sync blobs, now backfilling conversations") + limit := user.bridge.Config.Bridge.HistorySync.MaxInitialConversations // Find the portals for all the conversations. for i, conv := range conversations { jid, err := types.ParseJID(conv.ConversationID) @@ -152,7 +153,7 @@ func (user *User) backfillAll() { Str("portal_jid", portal.Key.JID.String()). Msg("Chat already has a room, deleting messages from database") user.bridge.DB.HistorySync.DeleteConversation(user.MXID, portal.Key.JID.String()) - } else if i < user.bridge.Config.Bridge.HistorySync.MaxInitialConversations { + } else if limit < 0 || i < limit { err = portal.CreateMatrixRoom(user, nil, true, true) if err != nil { user.zlog.Err(err).Msg("Failed to create Matrix room for backfill") From 1ee0841add2097a4f62d4dcbbf118adcb321769b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 22 Jun 2023 12:49:49 +0300 Subject: [PATCH 13/94] Fix function name --- database/historysync.go | 2 +- historysync.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/database/historysync.go b/database/historysync.go index 4d9d0bf..5727a9a 100644 --- a/database/historysync.go +++ b/database/historysync.go @@ -166,7 +166,7 @@ func (hsc *HistorySyncConversation) Scan(row dbutil.Scannable) *HistorySyncConve return hsc } -func (hsq *HistorySyncQuery) GetNMostRecentConversations(userID id.UserID, n int) (conversations []*HistorySyncConversation) { +func (hsq *HistorySyncQuery) GetRecentConversations(userID id.UserID, n int) (conversations []*HistorySyncConversation) { nPtr := &n // Negative limit on SQLite means unlimited, but Postgres prefers a NULL limit. if n < 0 && hsq.db.Dialect == dbutil.Postgres { diff --git a/historysync.go b/historysync.go index 742c9b2..f5535a2 100644 --- a/historysync.go +++ b/historysync.go @@ -108,7 +108,7 @@ func (user *User) handleHistorySyncsLoop() { const EnqueueBackfillsDelay = 30 * time.Second func (user *User) enqueueAllBackfills() { - nMostRecent := user.bridge.DB.HistorySync.GetNMostRecentConversations(user.MXID, user.bridge.Config.Bridge.HistorySync.MaxInitialConversations) + nMostRecent := user.bridge.DB.HistorySync.GetRecentConversations(user.MXID, user.bridge.Config.Bridge.HistorySync.MaxInitialConversations) if len(nMostRecent) > 0 { user.log.Infofln("%v has passed since the last history sync blob, enqueueing backfills for %d chats", EnqueueBackfillsDelay, len(nMostRecent)) // Find the portals for all the conversations. @@ -132,7 +132,7 @@ func (user *User) enqueueAllBackfills() { } func (user *User) backfillAll() { - conversations := user.bridge.DB.HistorySync.GetNMostRecentConversations(user.MXID, -1) + conversations := user.bridge.DB.HistorySync.GetRecentConversations(user.MXID, -1) if len(conversations) > 0 { user.zlog.Info(). Int("conversation_count", len(conversations)). From 2c53b2f193b2ae185e10851bb5ef64633a899134 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 23 Jun 2023 15:50:02 +0300 Subject: [PATCH 14/94] Update mautrix-go --- config/upgrade.go | 1 + example-config.yaml | 4 ++++ go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/config/upgrade.go b/config/upgrade.go index 440c07e..e4023e3 100644 --- a/config/upgrade.go +++ b/config/upgrade.go @@ -134,6 +134,7 @@ func DoUpgrade(helper *up.Helper) { helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "delete_prev_on_new_session") helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "delete_on_device_delete") helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "periodically_delete_expired") + helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "delete_outdated_inbound") helper.Copy(up.Str, "bridge", "encryption", "verification_levels", "receive") helper.Copy(up.Str, "bridge", "encryption", "verification_levels", "send") helper.Copy(up.Str, "bridge", "encryption", "verification_levels", "share") diff --git a/example-config.yaml b/example-config.yaml index b247c05..f5a347a 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -372,6 +372,10 @@ bridge: delete_on_device_delete: false # Periodically delete megolm sessions when 2x max_age has passed since receiving the session. periodically_delete_expired: false + # Delete inbound megolm sessions that don't have the received_at field used for + # automatic ratcheting and expired session deletion. This is meant as a migration + # to delete old keys prior to the bridge update. + delete_outdated_inbound: false # What level of device verification should be required from users? # # Valid levels: diff --git a/go.mod b/go.mod index 384adc8..f393b7d 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( golang.org/x/net v0.11.0 google.golang.org/protobuf v1.30.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.15.4-0.20230619114441-3e840e962e24 + maunium.net/go/mautrix v0.15.4-0.20230623121006-d8b15c18dc3f ) require ( diff --git a/go.sum b/go.sum index 21fe37b..fb2ed0f 100644 --- a/go.sum +++ b/go.sum @@ -131,5 +131,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.15.4-0.20230619114441-3e840e962e24 h1:caWWJEYs2G4eLbQ5y7RIrCPUgPU8m1nj6jeyGZAuYVM= -maunium.net/go/mautrix v0.15.4-0.20230619114441-3e840e962e24/go.mod h1:zLrQqdxJlLkurRCozTc9CL6FySkgZlO/kpCYxBILSLE= +maunium.net/go/mautrix v0.15.4-0.20230623121006-d8b15c18dc3f h1:On42TnPHerbuxWmKVSamncYD0MPgOkKDSqtL7eOyHJc= +maunium.net/go/mautrix v0.15.4-0.20230623121006-d8b15c18dc3f/go.mod h1:zLrQqdxJlLkurRCozTc9CL6FySkgZlO/kpCYxBILSLE= From 7f7845051c533ed52d6b02d2b49279dd1d413bde Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 26 Jun 2023 13:20:59 +0300 Subject: [PATCH 15/94] Update Docker image to Alpine 3.18 --- Dockerfile | 4 ++-- Dockerfile.ci | 2 +- Dockerfile.dev | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6278562..b2943ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1-alpine3.17 AS builder +FROM golang:1-alpine3.18 AS builder RUN apk add --no-cache git ca-certificates build-base su-exec olm-dev @@ -6,7 +6,7 @@ COPY . /build WORKDIR /build RUN go build -o /usr/bin/mautrix-whatsapp -FROM alpine:3.17 +FROM alpine:3.18 ENV UID=1337 \ GID=1337 diff --git a/Dockerfile.ci b/Dockerfile.ci index b3f736c..2860a1d 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM alpine:3.17 +FROM alpine:3.18 ENV UID=1337 \ GID=1337 diff --git a/Dockerfile.dev b/Dockerfile.dev index 5d4413d..4edc6ad 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM golang:1-alpine3.17 +FROM golang:1-alpine3.18 RUN apk add --no-cache git ca-certificates build-base su-exec olm-dev bash jq yq curl From 2b8b4332b087088a43b827024cdc18e518f706da Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Wed, 28 Jun 2023 08:54:49 -0600 Subject: [PATCH 16/94] deps/mautrix: upgrade to reduce logs of requests Signed-off-by: Sumner Evans --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f393b7d..0f76c05 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( golang.org/x/net v0.11.0 google.golang.org/protobuf v1.30.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.15.4-0.20230623121006-d8b15c18dc3f + maunium.net/go/mautrix v0.15.4-0.20230627151820-13edf5920c4d ) require ( diff --git a/go.sum b/go.sum index fb2ed0f..0a131b5 100644 --- a/go.sum +++ b/go.sum @@ -131,5 +131,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.15.4-0.20230623121006-d8b15c18dc3f h1:On42TnPHerbuxWmKVSamncYD0MPgOkKDSqtL7eOyHJc= -maunium.net/go/mautrix v0.15.4-0.20230623121006-d8b15c18dc3f/go.mod h1:zLrQqdxJlLkurRCozTc9CL6FySkgZlO/kpCYxBILSLE= +maunium.net/go/mautrix v0.15.4-0.20230627151820-13edf5920c4d h1:twnvyn4XnhnXfeH/A5HuHjvZqDW701dRXUxvST4sSVE= +maunium.net/go/mautrix v0.15.4-0.20230627151820-13edf5920c4d/go.mod h1:zLrQqdxJlLkurRCozTc9CL6FySkgZlO/kpCYxBILSLE= From b33affe97df0e870aa12865d9d857a3108726f2f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 27 Jun 2023 11:17:41 +0300 Subject: [PATCH 17/94] Log if finding reply target fails --- historysync.go | 2 +- portal.go | 27 ++++++++++++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/historysync.go b/historysync.go index f5535a2..c274074 100644 --- a/historysync.go +++ b/historysync.go @@ -653,7 +653,7 @@ func (portal *Portal) backfill(source *User, messages []*waProto.WebMessageInfo, continue } if converted.ReplyTo != nil { - portal.SetReply(converted.Content, converted.ReplyTo, true) + portal.SetReply(msgEvt.Info.ID, converted.Content, converted.ReplyTo, true) } err = portal.appendBatchEvents(source, converted, &msgEvt.Info, webMsg, &req.Events, &infos) if err != nil { diff --git a/portal.go b/portal.go index 35194d5..ee76094 100644 --- a/portal.go +++ b/portal.go @@ -42,6 +42,7 @@ import ( "time" "github.com/chai2010/webp" + "github.com/rs/zerolog" "github.com/tidwall/gjson" "golang.org/x/exp/slices" "golang.org/x/image/draw" @@ -194,6 +195,7 @@ func (br *WABridge) newBlankPortal(key database.PortalKey) *Portal { portal := &Portal{ bridge: br, log: br.Log.Sub(fmt.Sprintf("Portal/%s", key)), + zlog: br.ZLog.With().Str("portal_key", key.String()).Logger(), messages: make(chan PortalMessage, br.Config.Bridge.PortalMessageBuffer), matrixMessages: make(chan PortalMatrixMessage, br.Config.Bridge.PortalMessageBuffer), @@ -256,7 +258,9 @@ type Portal struct { *database.Portal bridge *WABridge - log log.Logger + // Deprecated: use zerolog + log log.Logger + zlog zerolog.Logger roomCreateLock sync.Mutex encryptLock sync.Mutex @@ -772,7 +776,7 @@ func (portal *Portal) handleMessage(source *User, evt *events.Message) { portal.MarkDisappearing(nil, existingMsg.MXID, converted.ExpiresIn, evt.Info.Timestamp) converted.Content.SetEdit(existingMsg.MXID) } else if converted.ReplyTo != nil { - portal.SetReply(converted.Content, converted.ReplyTo, false) + portal.SetReply(evt.Info.ID, converted.Content, converted.ReplyTo, false) } dbMsgType := database.MsgNormal if editTargetMsg != nil { @@ -1916,10 +1920,15 @@ func (portal *Portal) addReplyMention(content *event.MessageEventContent, sender } } -func (portal *Portal) SetReply(content *event.MessageEventContent, replyTo *ReplyInfo, isHungryBackfill bool) bool { +func (portal *Portal) SetReply(msgID string, content *event.MessageEventContent, replyTo *ReplyInfo, isHungryBackfill bool) bool { if replyTo == nil { return false } + log := portal.zlog.With(). + Str("message_id", msgID). + Object("reply_to", replyTo). + Str("action", "SetReply"). + Logger() key := portal.Key targetPortal := portal defer func() { @@ -1946,6 +1955,8 @@ func (portal *Portal) SetReply(content *event.MessageEventContent, replyTo *Repl content.RelatesTo = (&event.RelatesTo{}).SetReplyTo(targetPortal.deterministicEventID(replyTo.Sender, replyTo.MessageID, "")) portal.addReplyMention(content, replyTo.Sender, "") return true + } else { + log.Warn().Msg("Failed to find reply target") } return false } @@ -1956,14 +1967,14 @@ func (portal *Portal) SetReply(content *event.MessageEventContent, replyTo *Repl } evt, err := targetPortal.MainIntent().GetEvent(targetPortal.MXID, message.MXID) if err != nil { - portal.log.Warnln("Failed to get reply target:", err) + log.Warn().Err(err).Msg("Failed to get reply target event") return true } _ = evt.Content.ParseRaw(evt.Type) if evt.Type == event.EventEncrypted { decryptedEvt, err := portal.bridge.Crypto.Decrypt(evt) if err != nil { - portal.log.Warnln("Failed to decrypt reply target:", err) + log.Warn().Err(err).Msg("Failed to decrypt reply target event") } else { evt = decryptedEvt } @@ -2100,6 +2111,12 @@ type ReplyInfo struct { Sender types.JID } +func (r ReplyInfo) MarshalZerologObject(e *zerolog.Event) { + e.Str("message_id", r.MessageID) + e.Str("chat_jid", r.Chat.String()) + e.Str("sender_jid", r.Sender.String()) +} + type Replyable interface { GetStanzaId() string GetParticipant() string From de5de426cbff2e50175d6ff27e2ded085514de72 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 28 Jun 2023 23:29:54 +0300 Subject: [PATCH 18/94] Update dependencies --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 0f76c05..5fc3818 100644 --- a/go.mod +++ b/go.mod @@ -12,13 +12,13 @@ require ( github.com/rs/zerolog v1.29.1 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.14.4 - go.mau.fi/whatsmeow v0.0.0-20230616194828-be0edabb0bf3 - golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 + go.mau.fi/whatsmeow v0.0.0-20230628202737-27f697dce47a + golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df golang.org/x/image v0.8.0 golang.org/x/net v0.11.0 - google.golang.org/protobuf v1.30.0 + google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.15.4-0.20230627151820-13edf5920c4d + maunium.net/go/mautrix v0.15.4-0.20230628151140-e99578a15474 ) require ( diff --git a/go.sum b/go.sum index 0a131b5..1d1a809 100644 --- a/go.sum +++ b/go.sum @@ -68,16 +68,16 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= -go.mau.fi/whatsmeow v0.0.0-20230616194828-be0edabb0bf3 h1:CByCFmtYMsj/4tO+j9YvLLcUa5xnnrDyJXFbl52quhU= -go.mau.fi/whatsmeow v0.0.0-20230616194828-be0edabb0bf3/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= +go.mau.fi/whatsmeow v0.0.0-20230628202737-27f697dce47a h1:pj23OyiqQp+9RWjJZMARi54wdirCOXiIw/A3QSTfiSc= +go.mau.fi/whatsmeow v0.0.0-20230628202737-27f697dce47a/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.8.0 h1:agUcRXV/+w6L9ryntYYsF2x9fQTMd4T8fiiYXAVW6Jg= golang.org/x/image v0.8.0/go.mod h1:PwLxp3opCYg4WR2WO9P0L6ESnsD6bLTWcw8zanLMVFM= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -119,8 +119,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= @@ -131,5 +131,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.15.4-0.20230627151820-13edf5920c4d h1:twnvyn4XnhnXfeH/A5HuHjvZqDW701dRXUxvST4sSVE= -maunium.net/go/mautrix v0.15.4-0.20230627151820-13edf5920c4d/go.mod h1:zLrQqdxJlLkurRCozTc9CL6FySkgZlO/kpCYxBILSLE= +maunium.net/go/mautrix v0.15.4-0.20230628151140-e99578a15474 h1:Pzopg6NL7qaNdG2iUf8dVnu9+tVEa82b44KJKmpb/NY= +maunium.net/go/mautrix v0.15.4-0.20230628151140-e99578a15474/go.mod h1:zLrQqdxJlLkurRCozTc9CL6FySkgZlO/kpCYxBILSLE= From c00fa4dd246b4123b37144cfe33ecf6f4735c010 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 5 Jul 2023 12:09:31 +0300 Subject: [PATCH 19/94] Send all ogg audio from WhatsApp as voice messages For some reason WhatsApp removes the `ptt` flag when a message is forwarded, but still renders them as voice messages. --- portal.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portal.go b/portal.go index ee76094..7ca3415 100644 --- a/portal.go +++ b/portal.go @@ -3014,7 +3014,7 @@ func (portal *Portal) convertMediaMessageContent(intent *appservice.IntentAPI, m "duration": int(audioMessage.GetSeconds()) * 1000, "waveform": waveform, } - if audioMessage.GetPtt() { + if audioMessage.GetPtt() || audioMessage.GetMimetype() == "audio/ogg; codecs/opus" { extraContent["org.matrix.msc3245.voice"] = map[string]interface{}{} } } From ff609124b25717a23cd1f4d01de86a7ea59c6c5e Mon Sep 17 00:00:00 2001 From: vurpo Date: Sat, 8 Jul 2023 13:14:22 +0300 Subject: [PATCH 20/94] Update mautrix-go (#630) (adds Matrix media retries) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5fc3818..22ad005 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( golang.org/x/net v0.11.0 google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.15.4-0.20230628151140-e99578a15474 + maunium.net/go/mautrix v0.15.4-0.20230707135424-4639f0c9c471 ) require ( diff --git a/go.sum b/go.sum index 1d1a809..06fd240 100644 --- a/go.sum +++ b/go.sum @@ -131,5 +131,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.15.4-0.20230628151140-e99578a15474 h1:Pzopg6NL7qaNdG2iUf8dVnu9+tVEa82b44KJKmpb/NY= -maunium.net/go/mautrix v0.15.4-0.20230628151140-e99578a15474/go.mod h1:zLrQqdxJlLkurRCozTc9CL6FySkgZlO/kpCYxBILSLE= +maunium.net/go/mautrix v0.15.4-0.20230707135424-4639f0c9c471 h1:jo8zt1l3QZOzclEyf/9ZK06Eqcf/aoBoMXEEp5B4S/E= +maunium.net/go/mautrix v0.15.4-0.20230707135424-4639f0c9c471/go.mod h1:zLrQqdxJlLkurRCozTc9CL6FySkgZlO/kpCYxBILSLE= From 22812c74551660613067bfffc2bab2d033c6feeb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 10 Jul 2023 12:37:54 +0300 Subject: [PATCH 21/94] Sleep if logged in user is disconnected when handling message --- portal.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/portal.go b/portal.go index 7ca3415..628ae76 100644 --- a/portal.go +++ b/portal.go @@ -4094,7 +4094,7 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event, timing ms := metricSender{portal: portal, timings: &timings} allowRelay := evt.Type != TypeMSC3381PollResponse && evt.Type != TypeMSC3381V2PollResponse && evt.Type != TypeMSC3381PollStart - if err := portal.canBridgeFrom(sender, allowRelay); err != nil { + if err := portal.canBridgeFrom(sender, allowRelay, true); err != nil { go ms.sendMessageMetrics(evt, err, "Ignoring", true) return } else if portal.Key.JID == types.StatusBroadcastJID && portal.bridge.Config.Bridge.DisableStatusBroadcastSend { @@ -4188,7 +4188,7 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event, timing } func (portal *Portal) HandleMatrixReaction(sender *User, evt *event.Event) { - if err := portal.canBridgeFrom(sender, false); err != nil { + if err := portal.canBridgeFrom(sender, false, true); err != nil { go portal.sendMessageMetrics(evt, err, "Ignoring", nil) return } else if portal.Key.JID.Server == types.BroadcastServer { @@ -4280,7 +4280,7 @@ func (portal *Portal) upsertReaction(txn dbutil.Transaction, intent *appservice. } func (portal *Portal) HandleMatrixRedaction(sender *User, evt *event.Event) { - if err := portal.canBridgeFrom(sender, true); err != nil { + if err := portal.canBridgeFrom(sender, true, true); err != nil { go portal.sendMessageMetrics(evt, err, "Ignoring", nil) return } @@ -4449,12 +4449,16 @@ func (portal *Portal) HandleMatrixTyping(newTyping []id.UserID) { portal.setTyping(stoppedTyping, types.ChatPresencePaused) } -func (portal *Portal) canBridgeFrom(sender *User, allowRelay bool) error { +func (portal *Portal) canBridgeFrom(sender *User, allowRelay, reconnectWait bool) error { if !sender.IsLoggedIn() { if allowRelay && portal.HasRelaybot() { return nil } else if sender.Session != nil { return errUserNotConnected + } else if reconnectWait { + // If a message was received exactly during a disconnection, wait a second for the socket to reconnect + time.Sleep(1 * time.Second) + return portal.canBridgeFrom(sender, allowRelay, false) } else { return errUserNotLoggedIn } From 158486cc18760f072265d5a25e274aa9ce83c7c3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 10 Jul 2023 12:38:23 +0300 Subject: [PATCH 22/94] Update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ee24a5..57bf1b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ * Removed MSC2716 support. * Added legacy backfill support. +* Updated Docker image to Alpine 3.18. +* Changed all ogg audio messages from WhatsApp to be bridged as voice messages + to Matrix, as WhatsApp removes the voice message flag when forwarding for + some reason. # v0.8.6 (2023-06-16) From dc00b55dff4c81da3e8a83c1413ac7eef580f825 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 16 Jul 2023 12:30:30 +0300 Subject: [PATCH 23/94] Bump version to v0.9.0 --- CHANGELOG.md | 6 +++++- go.mod | 16 ++++++++-------- go.sum | 32 ++++++++++++++++---------------- main.go | 2 +- 4 files changed, 30 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57bf1b7..9db0fbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# v0.9.0 (unreleased) +# v0.9.0 (2023-07-16) * Removed MSC2716 support. * Added legacy backfill support. @@ -6,6 +6,10 @@ * Changed all ogg audio messages from WhatsApp to be bridged as voice messages to Matrix, as WhatsApp removes the voice message flag when forwarding for some reason. +* Added Prometheus metric for WhatsApp connection failures + (thanks to [@Half-Shot] in [#620]). + +[#620]: https://github.com/mautrix/whatsapp/pull/620 # v0.8.6 (2023-06-16) diff --git a/go.mod b/go.mod index 22ad005..94b689a 100644 --- a/go.mod +++ b/go.mod @@ -12,13 +12,13 @@ require ( github.com/rs/zerolog v1.29.1 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.14.4 - go.mau.fi/whatsmeow v0.0.0-20230628202737-27f697dce47a - golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df - golang.org/x/image v0.8.0 - golang.org/x/net v0.11.0 + go.mau.fi/whatsmeow v0.0.0-20230716092751-4bd753344e7b + golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 + golang.org/x/image v0.9.0 + golang.org/x/net v0.12.0 google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.15.4-0.20230707135424-4639f0c9c471 + maunium.net/go/mautrix v0.15.4 ) require ( @@ -41,9 +41,9 @@ require ( github.com/yuin/goldmark v1.5.4 // indirect go.mau.fi/libsignal v0.1.0 // indirect go.mau.fi/zeroconfig v0.1.2 // indirect - golang.org/x/crypto v0.10.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/text v0.10.0 // indirect + golang.org/x/crypto v0.11.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect maunium.net/go/mauflag v1.0.0 // indirect diff --git a/go.sum b/go.sum index 06fd240..2ad036d 100644 --- a/go.sum +++ b/go.sum @@ -68,26 +68,26 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= -go.mau.fi/whatsmeow v0.0.0-20230628202737-27f697dce47a h1:pj23OyiqQp+9RWjJZMARi54wdirCOXiIw/A3QSTfiSc= -go.mau.fi/whatsmeow v0.0.0-20230628202737-27f697dce47a/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= +go.mau.fi/whatsmeow v0.0.0-20230716092751-4bd753344e7b h1:L6nTucSoWAY5le8liGQh+ZOov0NvLjy4nCnYK10lwvE= +go.mau.fi/whatsmeow v0.0.0-20230716092751-4bd753344e7b/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= -golang.org/x/image v0.8.0 h1:agUcRXV/+w6L9ryntYYsF2x9fQTMd4T8fiiYXAVW6Jg= -golang.org/x/image v0.8.0/go.mod h1:PwLxp3opCYg4WR2WO9P0L6ESnsD6bLTWcw8zanLMVFM= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/image v0.9.0 h1:QrzfX26snvCM20hIhBwuHI/ThTg18b/+kcKdXHvnR+g= +golang.org/x/image v0.9.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -100,8 +100,8 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -109,8 +109,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -131,5 +131,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.15.4-0.20230707135424-4639f0c9c471 h1:jo8zt1l3QZOzclEyf/9ZK06Eqcf/aoBoMXEEp5B4S/E= -maunium.net/go/mautrix v0.15.4-0.20230707135424-4639f0c9c471/go.mod h1:zLrQqdxJlLkurRCozTc9CL6FySkgZlO/kpCYxBILSLE= +maunium.net/go/mautrix v0.15.4 h1:Ug3n2Mo+9Yb94AjZTWJQSNHmShaksEzZi85EPl3S3P0= +maunium.net/go/mautrix v0.15.4/go.mod h1:dBaDmsnOOBM4a+gKcgefXH73pHGXm+MCJzCs1dXFgrw= diff --git a/main.go b/main.go index 0d41b86..71f98f1 100644 --- a/main.go +++ b/main.go @@ -262,7 +262,7 @@ func main() { Name: "mautrix-whatsapp", URL: "https://github.com/mautrix/whatsapp", Description: "A Matrix-WhatsApp puppeting bridge.", - Version: "0.8.6", + Version: "0.9.0", ProtocolName: "WhatsApp", BeeperServiceName: "whatsapp", BeeperNetworkName: "whatsapp", From a7d9251abd881a2972ef8b3057ca65075a1f9520 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 18 Jul 2023 22:09:55 +0300 Subject: [PATCH 24/94] Add support for logging in by entering code on phone --- commands.go | 32 +++++++++++++++++++++++++++++--- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/commands.go b/commands.go index a328533..b0d4b95 100644 --- a/commands.go +++ b/commands.go @@ -23,6 +23,7 @@ import ( "fmt" "html" "math" + "regexp" "sort" "strconv" "strings" @@ -431,9 +432,12 @@ var cmdLogin = &commands.FullHandler{ Help: commands.HelpMeta{ Section: commands.HelpSectionAuth, Description: "Link the bridge to your WhatsApp account as a web client.", + Args: "[_phone number_]", }, } +var looksLikeAPhoneRegex = regexp.MustCompile(`^\+[0-9]+$`) + func fnLogin(ce *WrappedCommandEvent) { if ce.User.Session != nil { if ce.User.IsConnected() { @@ -444,13 +448,33 @@ func fnLogin(ce *WrappedCommandEvent) { return } + var phoneNumber string + if len(ce.Args) > 0 { + phoneNumber = strings.TrimSpace(strings.Join(ce.Args, " ")) + if !looksLikeAPhoneRegex.MatchString(phoneNumber) { + ce.Reply("When specifying a phone number, it must be provided in international format without spaces or other extra characters") + return + } + } + qrChan, err := ce.User.Login(context.Background()) if err != nil { - ce.User.log.Errorf("Failed to log in:", err) + ce.ZLog.Err(err).Msg("Failed to start login") ce.Reply("Failed to log in: %v", err) return } + if phoneNumber != "" { + pairingCode, err := ce.User.Client.PairPhone(phoneNumber, true) + if err != nil { + ce.ZLog.Err(err).Msg("Failed to start phone code login") + ce.Reply("Failed to start phone code login: %v", err) + go ce.User.DeleteConnection() + return + } + ce.Reply("Scan the code below or enter the following code on your phone to log in: **%s**", pairingCode) + } + var qrEventID id.EventID for item := range qrChan { switch item.Event { @@ -458,7 +482,7 @@ func fnLogin(ce *WrappedCommandEvent) { jid := ce.User.Client.Store.ID ce.Reply("Successfully logged in as +%s (device #%d)", jid.User, jid.Device) case whatsmeow.QRChannelTimeout.Event: - ce.Reply("QR code timed out. Please restart the login.") + ce.Reply("Login timed out. Please restart the login.") case whatsmeow.QRChannelErrUnexpectedEvent.Event: ce.Reply("Failed to log in: unexpected connection event from server") case whatsmeow.QRChannelClientOutdated.Event: @@ -471,7 +495,9 @@ func fnLogin(ce *WrappedCommandEvent) { qrEventID = ce.User.sendQR(ce, item.Code, qrEventID) } } - _, _ = ce.Bot.RedactEvent(ce.RoomID, qrEventID) + if qrEventID != "" { + _, _ = ce.Bot.RedactEvent(ce.RoomID, qrEventID) + } } func (user *User) sendQR(ce *WrappedCommandEvent, code string, prevEvent id.EventID) id.EventID { diff --git a/go.mod b/go.mod index 94b689a..f9f0449 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/rs/zerolog v1.29.1 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.14.4 - go.mau.fi/whatsmeow v0.0.0-20230716092751-4bd753344e7b + go.mau.fi/whatsmeow v0.0.0-20230718190209-efef6f1cec8e golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 golang.org/x/image v0.9.0 golang.org/x/net v0.12.0 diff --git a/go.sum b/go.sum index 2ad036d..90b9d9a 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= -go.mau.fi/whatsmeow v0.0.0-20230716092751-4bd753344e7b h1:L6nTucSoWAY5le8liGQh+ZOov0NvLjy4nCnYK10lwvE= -go.mau.fi/whatsmeow v0.0.0-20230716092751-4bd753344e7b/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= +go.mau.fi/whatsmeow v0.0.0-20230718190209-efef6f1cec8e h1:i2atPgn2MRLGxisk+EZM1b1RfPh+4dZxSc8OdyvzutY= +go.mau.fi/whatsmeow v0.0.0-20230718190209-efef6f1cec8e/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= From 5c8c736c7f99e08938273c506980b650c7fc90de Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 21 Jul 2023 17:47:57 +0300 Subject: [PATCH 25/94] Update changelog and mautrix-go --- CHANGELOG.md | 5 +++++ go.mod | 9 +-------- go.sum | 4 ++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9db0fbb..b51dc0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# unreleased + +* Added support for logging in by entering a 8-letter code on the phone instead + of scanning a QR code. + # v0.9.0 (2023-07-16) * Removed MSC2716 support. diff --git a/go.mod b/go.mod index f9f0449..5cddd37 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( golang.org/x/net v0.12.0 google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.15.4 + maunium.net/go/mautrix v0.15.5-0.20230722173405-ab9d6e5eb5b9 ) require ( @@ -48,10 +48,3 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect maunium.net/go/mauflag v1.0.0 // indirect ) - -// Exclude some things that cause go.sum to explode -exclude ( - cloud.google.com/go v0.65.0 - github.com/prometheus/client_golang v1.12.1 - google.golang.org/appengine v1.6.6 -) diff --git a/go.sum b/go.sum index 90b9d9a..ca089ef 100644 --- a/go.sum +++ b/go.sum @@ -131,5 +131,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.15.4 h1:Ug3n2Mo+9Yb94AjZTWJQSNHmShaksEzZi85EPl3S3P0= -maunium.net/go/mautrix v0.15.4/go.mod h1:dBaDmsnOOBM4a+gKcgefXH73pHGXm+MCJzCs1dXFgrw= +maunium.net/go/mautrix v0.15.5-0.20230722173405-ab9d6e5eb5b9 h1:FIdA1S8jI8CHNTYv04mTtkwyoE796Sm6g/OWdBsUm6Y= +maunium.net/go/mautrix v0.15.5-0.20230722173405-ab9d6e5eb5b9/go.mod h1:dBaDmsnOOBM4a+gKcgefXH73pHGXm+MCJzCs1dXFgrw= From d19ae64ccacda05a058a1ff46e3149c9b22ee853 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 28 Jul 2023 22:19:56 +0300 Subject: [PATCH 26/94] Update mautrix-go --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5cddd37..451e47d 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( golang.org/x/net v0.12.0 google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.15.5-0.20230722173405-ab9d6e5eb5b9 + maunium.net/go/mautrix v0.15.5-0.20230728182848-1ef656165098 ) require ( diff --git a/go.sum b/go.sum index ca089ef..bbf4da5 100644 --- a/go.sum +++ b/go.sum @@ -131,5 +131,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.15.5-0.20230722173405-ab9d6e5eb5b9 h1:FIdA1S8jI8CHNTYv04mTtkwyoE796Sm6g/OWdBsUm6Y= -maunium.net/go/mautrix v0.15.5-0.20230722173405-ab9d6e5eb5b9/go.mod h1:dBaDmsnOOBM4a+gKcgefXH73pHGXm+MCJzCs1dXFgrw= +maunium.net/go/mautrix v0.15.5-0.20230728182848-1ef656165098 h1:1MA35IEVXHzgFnJEcgAoawNcDedRZjXoHdwylhPURLs= +maunium.net/go/mautrix v0.15.5-0.20230728182848-1ef656165098/go.mod h1:dBaDmsnOOBM4a+gKcgefXH73pHGXm+MCJzCs1dXFgrw= From e1189bad4b971323e07afd5bc2ae37e8ebf0cf90 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 2 Aug 2023 17:24:28 +0300 Subject: [PATCH 27/94] Log conversation settings when saving history sync payloads --- historysync.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/historysync.go b/historysync.go index c274074..d3d8e0c 100644 --- a/historysync.go +++ b/historysync.go @@ -22,6 +22,7 @@ import ( "fmt" "time" + "github.com/rs/zerolog" "maunium.net/go/mautrix/util/variationselector" waProto "go.mau.fi/whatsmeow/binary/proto" @@ -529,6 +530,14 @@ func (user *User) storeHistorySync(evt *waProto.HistorySync) { Int("lowest_time_index", minTimeIndex). Time("highest_time", maxTime). Int("highest_time_index", maxTimeIndex). + Dict("metadata", zerolog.Dict(). + Uint32("ephemeral_expiration", conv.GetEphemeralExpiration()). + Bool("marked_unread", conv.GetMarkedAsUnread()). + Bool("archived", conv.GetArchived()). + Uint32("pinned", conv.GetPinned()). + Uint64("mute_end", conv.GetMuteEndTime()). + Uint32("unread_count", conv.GetUnreadCount()), + ). Msg("Saved messages from history sync conversation") } log.Info(). From ce5707315e265f3414d3a718448e857785315a40 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 2 Aug 2023 18:43:11 +0300 Subject: [PATCH 28/94] Update whatsmeow to retry on 500 connect failures --- go.mod | 2 +- go.sum | 4 ++-- user.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 451e47d..9652cd8 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/rs/zerolog v1.29.1 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.14.4 - go.mau.fi/whatsmeow v0.0.0-20230718190209-efef6f1cec8e + go.mau.fi/whatsmeow v0.0.0-20230802154236-954a5c8cefe0 golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 golang.org/x/image v0.9.0 golang.org/x/net v0.12.0 diff --git a/go.sum b/go.sum index bbf4da5..3a45b2d 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= -go.mau.fi/whatsmeow v0.0.0-20230718190209-efef6f1cec8e h1:i2atPgn2MRLGxisk+EZM1b1RfPh+4dZxSc8OdyvzutY= -go.mau.fi/whatsmeow v0.0.0-20230718190209-efef6f1cec8e/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= +go.mau.fi/whatsmeow v0.0.0-20230802154236-954a5c8cefe0 h1:Ukvl6H0a41fkzL7frv1207u3R52quHNe/zVM41Kdyjs= +go.mau.fi/whatsmeow v0.0.0-20230802154236-954a5c8cefe0/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/user.go b/user.go index 7de1819..ca11028 100644 --- a/user.go +++ b/user.go @@ -842,7 +842,7 @@ func (user *User) HandleEvent(event interface{}) { user.sendMarkdownBridgeAlert("The bridge was started in another location. Use `reconnect` to reconnect this one.") } case *events.ConnectFailure: - user.BridgeState.Send(status.BridgeState{StateEvent: status.StateUnknownError, Message: fmt.Sprintf("Unknown connection failure: %s", v.Reason)}) + user.BridgeState.Send(status.BridgeState{StateEvent: status.StateUnknownError, Message: fmt.Sprintf("Unknown connection failure: %s (%s)", v.Reason, v.Message)}) user.bridge.Metrics.TrackConnectionState(user.JID, false) user.bridge.Metrics.TrackConnectionFailure(fmt.Sprintf("status-%d", v.Reason)) case *events.ClientOutdated: From 93aac0a5fb05aed55dccab7d85921a8ac2a81f83 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 3 Aug 2023 13:30:38 +0300 Subject: [PATCH 29/94] Implicitly enable timer when receiving disappearing messages in DMs --- historysync.go | 2 +- portal.go | 29 +++++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/historysync.go b/historysync.go index d3d8e0c..aba3646 100644 --- a/historysync.go +++ b/historysync.go @@ -188,7 +188,7 @@ func (portal *Portal) legacyBackfill(user *User) { Msg("Dropping historical message due to parse error") continue } - portal.handleMessage(user, msgEvt) + portal.handleMessage(user, msgEvt, true) } if conv != nil { isUnread := conv.MarkedAsUnread || conv.UnreadCount > 0 diff --git a/portal.go b/portal.go index 628ae76..f238a39 100644 --- a/portal.go +++ b/portal.go @@ -314,7 +314,7 @@ func (portal *Portal) handleMessageLoopItem(msg PortalMessage) { defer portal.latestEventBackfillLock.Unlock() switch { case msg.evt != nil: - portal.handleMessage(msg.source, msg.evt) + portal.handleMessage(msg.source, msg.evt, false) case msg.receipt != nil: portal.handleReceipt(msg.receipt, msg.source) case msg.undecryptable != nil: @@ -609,6 +609,23 @@ func (portal *Portal) convertMessage(intent *appservice.IntentAPI, source *User, } } +func (portal *Portal) implicitlyEnableDisappearingMessages(timer time.Duration) { + portal.ExpirationTime = uint32(timer.Seconds()) + portal.Update(nil) + intent := portal.MainIntent() + if portal.Encrypted { + intent = portal.bridge.Bot + } + duration := formatDuration(time.Duration(portal.ExpirationTime) * time.Second) + _, err := portal.sendMessage(intent, event.EventMessage, &event.MessageEventContent{ + MsgType: event.MsgNotice, + Body: fmt.Sprintf("Automatically enabled disappearing message timer (%s) because incoming message is disappearing", duration), + }, nil, 0) + if err != nil { + portal.zlog.Warn().Err(err).Msg("Failed to send notice about implicit disappearing timer") + } +} + func (portal *Portal) UpdateGroupDisappearingMessages(sender *types.JID, timestamp time.Time, timer uint32) { if portal.ExpirationTime == timer { return @@ -713,7 +730,7 @@ func (portal *Portal) handleFakeMessage(msg fakeMessage) { } } -func (portal *Portal) handleMessage(source *User, evt *events.Message) { +func (portal *Portal) handleMessage(source *User, evt *events.Message, historical bool) { if len(portal.MXID) == 0 { portal.log.Warnln("handleMessage called even though portal.MXID is empty") return @@ -761,6 +778,14 @@ func (portal *Portal) handleMessage(source *User, evt *events.Message) { } converted := portal.convertMessage(intent, source, &evt.Info, evt.Message, false) if converted != nil { + if !historical && portal.IsPrivateChat() && evt.Info.Sender.Device == 0 && converted.ExpiresIn > 0 && portal.ExpirationTime == 0 { + portal.zlog.Info(). + Str("timer", converted.ExpiresIn.String()). + Str("sender_jid", evt.Info.Sender.String()). + Str("message_id", evt.Info.ID). + Msg("Implicitly enabling disappearing messages as incoming message is disappearing") + portal.implicitlyEnableDisappearingMessages(converted.ExpiresIn) + } if evt.Info.IsIncomingBroadcast() { if converted.Extra == nil { converted.Extra = map[string]interface{}{} From c19b5b18b01a442b33c653657c7712452dbb0746 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 3 Aug 2023 16:44:48 +0300 Subject: [PATCH 30/94] Remove debug retry receipt provisioning API --- provisioning.go | 52 ------------------------------------------------- 1 file changed, 52 deletions(-) diff --git a/provisioning.go b/provisioning.go index 7abef77..c9060a8 100644 --- a/provisioning.go +++ b/provisioning.go @@ -24,7 +24,6 @@ import ( "fmt" "net" "net/http" - "strconv" "strings" "time" @@ -32,7 +31,6 @@ import ( "github.com/gorilla/websocket" "go.mau.fi/whatsmeow/appstate" - waBinary "go.mau.fi/whatsmeow/binary" "go.mau.fi/whatsmeow/types" "go.mau.fi/whatsmeow" @@ -61,7 +59,6 @@ func (prov *ProvisioningAPI) Init() { r.HandleFunc("/v1/disconnect", prov.Disconnect).Methods(http.MethodPost) r.HandleFunc("/v1/reconnect", prov.Reconnect).Methods(http.MethodPost) r.HandleFunc("/v1/debug/appstate/{name}", prov.SyncAppState).Methods(http.MethodPost) - r.HandleFunc("/v1/debug/retry", prov.SendRetryReceipt).Methods(http.MethodPost) r.HandleFunc("/v1/contacts", prov.ListContacts).Methods(http.MethodGet) r.HandleFunc("/v1/groups", prov.ListGroups).Methods(http.MethodGet, http.MethodPost) r.HandleFunc("/v1/resolve_identifier/{number}", prov.ResolveIdentifier).Methods(http.MethodGet) @@ -191,55 +188,6 @@ func (prov *ProvisioningAPI) Reconnect(w http.ResponseWriter, r *http.Request) { } } -type debugRetryReceiptContent struct { - ID types.MessageID `json:"id"` - From types.JID `json:"from"` - Recipient types.JID `json:"recipient"` - Participant types.JID `json:"participant"` - Timestamp int64 `json:"timestamp"` - Count int `json:"count"` - - ForceIncludeIdentity bool `json:"force_include_identity"` -} - -func (prov *ProvisioningAPI) SendRetryReceipt(w http.ResponseWriter, r *http.Request) { - var req debugRetryReceiptContent - user := r.Context().Value("user").(*User) - if user == nil || user.Client == nil { - jsonResponse(w, http.StatusNotFound, Error{ - Error: "User is not connected to WhatsApp", - ErrCode: "no session", - }) - return - } else if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - jsonResponse(w, http.StatusBadRequest, Error{ - Error: "Failed to parse request JSON", - ErrCode: "bad json", - }) - } else { - node := &waBinary.Node{ - Attrs: waBinary.Attrs{ - "id": string(req.ID), - "from": req.From, - "t": strconv.FormatInt(req.Timestamp, 10), - }, - } - if !req.Recipient.IsEmpty() { - node.Attrs["recipient"] = req.Recipient - } - if !req.Participant.IsEmpty() { - node.Attrs["participant"] = req.Participant - } - if req.Count > 0 { - node.Content = []waBinary.Node{{ - Tag: "enc", - Attrs: waBinary.Attrs{"count": strconv.Itoa(req.Count)}, - }} - } - user.Client.DangerousInternals().SendRetryReceipt(node, req.ForceIncludeIdentity) - } -} - func (prov *ProvisioningAPI) SyncAppState(w http.ResponseWriter, r *http.Request) { user := r.Context().Value("user").(*User) if user == nil || user.Client == nil { From 27290b7ebd5cfe3632117fc01f3c8f92b16cb9ce Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 3 Aug 2023 16:44:54 +0300 Subject: [PATCH 31/94] Enable automatically re-requesting undecryptable messages from phone --- go.mod | 4 ++-- go.sum | 8 ++++---- portal.go | 9 +++++++-- user.go | 1 + 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 9652cd8..f274bdf 100644 --- a/go.mod +++ b/go.mod @@ -12,13 +12,13 @@ require ( github.com/rs/zerolog v1.29.1 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.14.4 - go.mau.fi/whatsmeow v0.0.0-20230802154236-954a5c8cefe0 + go.mau.fi/whatsmeow v0.0.0-20230803134242-fe6785536744 golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 golang.org/x/image v0.9.0 golang.org/x/net v0.12.0 google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.15.5-0.20230728182848-1ef656165098 + maunium.net/go/mautrix v0.15.5-0.20230729114956-ff77fa8ec663 ) require ( diff --git a/go.sum b/go.sum index 3a45b2d..ef37cb4 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= -go.mau.fi/whatsmeow v0.0.0-20230802154236-954a5c8cefe0 h1:Ukvl6H0a41fkzL7frv1207u3R52quHNe/zVM41Kdyjs= -go.mau.fi/whatsmeow v0.0.0-20230802154236-954a5c8cefe0/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= +go.mau.fi/whatsmeow v0.0.0-20230803134242-fe6785536744 h1:DzsCDn0VqGohyjkztZ9mXqPqSV33ZZ94nSLeK6zZB3I= +go.mau.fi/whatsmeow v0.0.0-20230803134242-fe6785536744/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -131,5 +131,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.15.5-0.20230728182848-1ef656165098 h1:1MA35IEVXHzgFnJEcgAoawNcDedRZjXoHdwylhPURLs= -maunium.net/go/mautrix v0.15.5-0.20230728182848-1ef656165098/go.mod h1:dBaDmsnOOBM4a+gKcgefXH73pHGXm+MCJzCs1dXFgrw= +maunium.net/go/mautrix v0.15.5-0.20230729114956-ff77fa8ec663 h1:tsXiuaWnBe9Xa/ms8tUhsJ4x7tPbQJ94fugqBIC0vsw= +maunium.net/go/mautrix v0.15.5-0.20230729114956-ff77fa8ec663/go.mod h1:dBaDmsnOOBM4a+gKcgefXH73pHGXm+MCJzCs1dXFgrw= diff --git a/portal.go b/portal.go index f238a39..96d3db6 100644 --- a/portal.go +++ b/portal.go @@ -746,10 +746,15 @@ func (portal *Portal) handleMessage(source *User, evt *events.Message, historica existingMsg := portal.bridge.DB.Message.GetByJID(portal.Key, msgID) if existingMsg != nil { if existingMsg.Error == database.MsgErrDecryptionFailed { + resolveType := "sender" + if evt.UnavailableRequestID != "" { + resolveType = "phone" + } Segment.Track(source.MXID, "WhatsApp undecryptable message resolved", map[string]interface{}{ - "messageID": evt.Info.ID, + "messageID": evt.Info.ID, + "resolveType": resolveType, }) - portal.log.Debugfln("Got decryptable version of previously undecryptable message %s (%s)", msgID, msgType) + portal.log.Debugfln("Got decryptable version of previously undecryptable message %s (%s) via %s", msgID, msgType, resolveType) } else { portal.log.Debugfln("Not handling %s (%s): message is duplicate", msgID, msgType) return diff --git a/user.go b/user.go index ca11028..c10ff9f 100644 --- a/user.go +++ b/user.go @@ -496,6 +496,7 @@ func (user *User) createClient(sess *store.Device) { user.Client = whatsmeow.NewClient(sess, &waLogger{user.log.Sub("Client")}) user.Client.AddEventHandler(user.HandleEvent) user.Client.SetForceActiveDeliveryReceipts(user.bridge.Config.Bridge.ForceActiveDeliveryReceipts) + user.Client.AutomaticMessageRerequestFromPhone = true user.Client.GetMessageForRetry = func(requester, to types.JID, id types.MessageID) *waProto.Message { Segment.Track(user.MXID, "WhatsApp incoming retry (message not found)", map[string]interface{}{ "requester": user.obfuscateJID(requester), From 262940f3a0ce8f77a27a9577c45502bc2f695e42 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 3 Aug 2023 19:47:42 +0300 Subject: [PATCH 32/94] Use new GenerateMessageID function --- commands.go | 2 +- portal.go | 2 +- user.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/commands.go b/commands.go index b0d4b95..7ba89ce 100644 --- a/commands.go +++ b/commands.go @@ -380,7 +380,7 @@ func fnCreate(ce *WrappedCommandEvent) { } // TODO check m.space.parent to create rooms directly in communities - messageID := whatsmeow.GenerateMessageID() + messageID := ce.User.Client.GenerateMessageID() ce.Log.Infofln("Creating group for %s with name %s and participants %+v (create key: %s)", ce.RoomID, roomNameEvent.Name, participants, messageID) ce.User.createKeyDedup = messageID resp, err := ce.User.Client.CreateGroup(whatsmeow.ReqCreateGroup{ diff --git a/portal.go b/portal.go index 96d3db6..93cc53f 100644 --- a/portal.go +++ b/portal.go @@ -4108,7 +4108,7 @@ func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, ev func (portal *Portal) generateMessageInfo(sender *User) *types.MessageInfo { return &types.MessageInfo{ - ID: whatsmeow.GenerateMessageID(), + ID: sender.Client.GenerateMessageID(), Timestamp: time.Now(), MessageSource: types.MessageSource{ Sender: sender.JID, diff --git a/user.go b/user.go index c10ff9f..5ccfbe5 100644 --- a/user.go +++ b/user.go @@ -677,7 +677,7 @@ const PhoneMinPingInterval = 24 * time.Hour func (user *User) sendHackyPhonePing() { user.PhoneLastPinged = time.Now() - msgID := whatsmeow.GenerateMessageID() + msgID := user.Client.GenerateMessageID() keyIDs := make([]*waProto.AppStateSyncKeyId, 0, 1) lastKeyID, err := user.GetLastAppStateKeyID() if lastKeyID != nil { From 558ccc7ab69e992e056740434d71644a09e5168d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 4 Aug 2023 16:09:54 +0300 Subject: [PATCH 33/94] Update whatsmeow --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f274bdf..96381a9 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/rs/zerolog v1.29.1 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.14.4 - go.mau.fi/whatsmeow v0.0.0-20230803134242-fe6785536744 + go.mau.fi/whatsmeow v0.0.0-20230804130905-60a6b1c2cf06 golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 golang.org/x/image v0.9.0 golang.org/x/net v0.12.0 diff --git a/go.sum b/go.sum index ef37cb4..1686068 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= -go.mau.fi/whatsmeow v0.0.0-20230803134242-fe6785536744 h1:DzsCDn0VqGohyjkztZ9mXqPqSV33ZZ94nSLeK6zZB3I= -go.mau.fi/whatsmeow v0.0.0-20230803134242-fe6785536744/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= +go.mau.fi/whatsmeow v0.0.0-20230804130905-60a6b1c2cf06 h1:jxr/yWZZGX732faE5xWFc7rejb7PSAS0EaZcaNIBJYc= +go.mau.fi/whatsmeow v0.0.0-20230804130905-60a6b1c2cf06/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= From cc900971dfaed17c5a60ab72c611bb6da791fe21 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 4 Aug 2023 23:57:27 +0300 Subject: [PATCH 34/94] Update whatsmeow again --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 96381a9..f969ea8 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/rs/zerolog v1.29.1 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.14.4 - go.mau.fi/whatsmeow v0.0.0-20230804130905-60a6b1c2cf06 + go.mau.fi/whatsmeow v0.0.0-20230804205607-3bb2b63aac52 golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 golang.org/x/image v0.9.0 golang.org/x/net v0.12.0 diff --git a/go.sum b/go.sum index 1686068..30441af 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= -go.mau.fi/whatsmeow v0.0.0-20230804130905-60a6b1c2cf06 h1:jxr/yWZZGX732faE5xWFc7rejb7PSAS0EaZcaNIBJYc= -go.mau.fi/whatsmeow v0.0.0-20230804130905-60a6b1c2cf06/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= +go.mau.fi/whatsmeow v0.0.0-20230804205607-3bb2b63aac52 h1:+gKteSXTZhhQjyOTdG2d8lh4v6t5Xpt4lRwQNoZaTqY= +go.mau.fi/whatsmeow v0.0.0-20230804205607-3bb2b63aac52/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= From e07ed17ccdff16000064c98c4e2b83f9fe757cfa Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 5 Aug 2023 00:07:16 +0300 Subject: [PATCH 35/94] =?UTF-8?q?Update=20whatsmeow=20again=C2=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f969ea8..08a0943 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/rs/zerolog v1.29.1 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.14.4 - go.mau.fi/whatsmeow v0.0.0-20230804205607-3bb2b63aac52 + go.mau.fi/whatsmeow v0.0.0-20230804210635-04e4e3a38f73 golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 golang.org/x/image v0.9.0 golang.org/x/net v0.12.0 diff --git a/go.sum b/go.sum index 30441af..b74872b 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= -go.mau.fi/whatsmeow v0.0.0-20230804205607-3bb2b63aac52 h1:+gKteSXTZhhQjyOTdG2d8lh4v6t5Xpt4lRwQNoZaTqY= -go.mau.fi/whatsmeow v0.0.0-20230804205607-3bb2b63aac52/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= +go.mau.fi/whatsmeow v0.0.0-20230804210635-04e4e3a38f73 h1:JK9z+AWKvWXINSqz6wqfF67L/uUMEMJ+Mfw4sVMFnQg= +go.mau.fi/whatsmeow v0.0.0-20230804210635-04e4e3a38f73/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= From 45e5290f0c51dfecb737c9325329b8f69ba07c5d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 5 Aug 2023 18:59:17 +0300 Subject: [PATCH 36/94] Update mautrix-go to move stuff to util --- config/upgrade.go | 6 +++--- database/backfill.go | 2 +- database/database.go | 2 +- database/disappearingmessage.go | 2 +- database/historysync.go | 8 +++----- database/mediabackfillrequest.go | 2 +- database/message.go | 5 ++--- database/polloption.go | 2 +- database/portal.go | 5 ++--- database/puppet.go | 5 ++--- database/reaction.go | 5 ++--- database/upgrades/upgrades.go | 2 +- database/user.go | 5 ++--- disappear.go | 3 ++- go.mod | 5 +++-- go.sum | 12 +++++++----- historysync.go | 5 ++--- main.go | 2 +- portal.go | 33 ++++++++++++++++---------------- 19 files changed, 53 insertions(+), 58 deletions(-) diff --git a/config/upgrade.go b/config/upgrade.go index e4023e3..02ae8fe 100644 --- a/config/upgrade.go +++ b/config/upgrade.go @@ -19,9 +19,9 @@ package config import ( "strings" + up "go.mau.fi/util/configupgrade" + "go.mau.fi/util/random" "maunium.net/go/mautrix/bridge/bridgeconfig" - "maunium.net/go/mautrix/util" - up "maunium.net/go/mautrix/util/configupgrade" ) func DoUpgrade(helper *up.Helper) { @@ -163,7 +163,7 @@ func DoUpgrade(helper *up.Helper) { if secret, ok := helper.Get(up.Str, "appservice", "provisioning", "shared_secret"); ok && secret != "generate" { helper.Set(up.Str, secret, "bridge", "provisioning", "shared_secret") } else if secret, ok = helper.Get(up.Str, "bridge", "provisioning", "shared_secret"); !ok || secret == "generate" { - sharedSecret := util.RandomString(64) + sharedSecret := random.String(64) helper.Set(up.Str, sharedSecret, "bridge", "provisioning", "shared_secret") } else { helper.Copy(up.Str, "bridge", "provisioning", "shared_secret") diff --git a/database/backfill.go b/database/backfill.go index 6925382..273370b 100644 --- a/database/backfill.go +++ b/database/backfill.go @@ -25,10 +25,10 @@ import ( "sync" "time" + "go.mau.fi/util/dbutil" log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" ) type BackfillType int diff --git a/database/database.go b/database/database.go index 7f519e7..4bc749a 100644 --- a/database/database.go +++ b/database/database.go @@ -23,10 +23,10 @@ import ( "github.com/lib/pq" _ "github.com/mattn/go-sqlite3" + "go.mau.fi/util/dbutil" "go.mau.fi/whatsmeow/store" "go.mau.fi/whatsmeow/store/sqlstore" "maunium.net/go/maulogger/v2" - "maunium.net/go/mautrix/util/dbutil" "maunium.net/go/mautrix-whatsapp/database/upgrades" ) diff --git a/database/disappearingmessage.go b/database/disappearingmessage.go index 7489381..bae11e0 100644 --- a/database/disappearingmessage.go +++ b/database/disappearingmessage.go @@ -21,10 +21,10 @@ import ( "errors" "time" + "go.mau.fi/util/dbutil" log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" ) type DisappearingMessageQuery struct { diff --git a/database/historysync.go b/database/historysync.go index 5727a9a..2f88139 100644 --- a/database/historysync.go +++ b/database/historysync.go @@ -22,15 +22,13 @@ import ( "fmt" "time" - "google.golang.org/protobuf/proto" - - waProto "go.mau.fi/whatsmeow/binary/proto" - _ "github.com/mattn/go-sqlite3" + "go.mau.fi/util/dbutil" + waProto "go.mau.fi/whatsmeow/binary/proto" + "google.golang.org/protobuf/proto" log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" ) type HistorySyncQuery struct { diff --git a/database/mediabackfillrequest.go b/database/mediabackfillrequest.go index c206796..e71b986 100644 --- a/database/mediabackfillrequest.go +++ b/database/mediabackfillrequest.go @@ -21,10 +21,10 @@ import ( "errors" _ "github.com/mattn/go-sqlite3" + "go.mau.fi/util/dbutil" log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" ) type MediaBackfillRequestStatus int diff --git a/database/message.go b/database/message.go index ffe33c1..8f35de8 100644 --- a/database/message.go +++ b/database/message.go @@ -22,12 +22,11 @@ import ( "strings" "time" + "go.mau.fi/util/dbutil" + "go.mau.fi/whatsmeow/types" log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" - - "go.mau.fi/whatsmeow/types" ) type MessageQuery struct { diff --git a/database/polloption.go b/database/polloption.go index aedcd53..4af576f 100644 --- a/database/polloption.go +++ b/database/polloption.go @@ -21,7 +21,7 @@ import ( "strings" "github.com/lib/pq" - "maunium.net/go/mautrix/util/dbutil" + "go.mau.fi/util/dbutil" ) func scanPollOptionMapping(rows dbutil.Rows) (id string, hashArr [32]byte, err error) { diff --git a/database/portal.go b/database/portal.go index 304d535..b9d4a66 100644 --- a/database/portal.go +++ b/database/portal.go @@ -21,12 +21,11 @@ import ( "fmt" "time" + "go.mau.fi/util/dbutil" + "go.mau.fi/whatsmeow/types" log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" - - "go.mau.fi/whatsmeow/types" ) type PortalKey struct { diff --git a/database/puppet.go b/database/puppet.go index 17d7996..c490e7e 100644 --- a/database/puppet.go +++ b/database/puppet.go @@ -20,12 +20,11 @@ import ( "database/sql" "time" + "go.mau.fi/util/dbutil" + "go.mau.fi/whatsmeow/types" log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" - - "go.mau.fi/whatsmeow/types" ) type PuppetQuery struct { diff --git a/database/reaction.go b/database/reaction.go index f30f263..1769358 100644 --- a/database/reaction.go +++ b/database/reaction.go @@ -20,12 +20,11 @@ import ( "database/sql" "errors" + "go.mau.fi/util/dbutil" + "go.mau.fi/whatsmeow/types" log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" - - "go.mau.fi/whatsmeow/types" ) type ReactionQuery struct { diff --git a/database/upgrades/upgrades.go b/database/upgrades/upgrades.go index 00ac6bc..990f496 100644 --- a/database/upgrades/upgrades.go +++ b/database/upgrades/upgrades.go @@ -20,7 +20,7 @@ import ( "embed" "errors" - "maunium.net/go/mautrix/util/dbutil" + "go.mau.fi/util/dbutil" ) var Table dbutil.UpgradeTable diff --git a/database/user.go b/database/user.go index 8b77850..088e998 100644 --- a/database/user.go +++ b/database/user.go @@ -21,12 +21,11 @@ import ( "sync" "time" + "go.mau.fi/util/dbutil" + "go.mau.fi/whatsmeow/types" log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" - - "go.mau.fi/whatsmeow/types" ) type UserQuery struct { diff --git a/disappear.go b/disappear.go index c90fc05..c315bc2 100644 --- a/disappear.go +++ b/disappear.go @@ -20,9 +20,10 @@ import ( "fmt" "time" + "go.mau.fi/util/dbutil" + "maunium.net/go/mautrix" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" "maunium.net/go/mautrix-whatsapp/database" ) diff --git a/go.mod b/go.mod index 08a0943..245154e 100644 --- a/go.mod +++ b/go.mod @@ -9,16 +9,17 @@ require ( github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.17 github.com/prometheus/client_golang v1.16.0 - github.com/rs/zerolog v1.29.1 + github.com/rs/zerolog v1.30.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.14.4 + go.mau.fi/util v0.0.0-20230805161919-cf42c11d39c3 go.mau.fi/whatsmeow v0.0.0-20230804210635-04e4e3a38f73 golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 golang.org/x/image v0.9.0 golang.org/x/net v0.12.0 google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.15.5-0.20230729114956-ff77fa8ec663 + maunium.net/go/mautrix v0.15.5-0.20230805162445-fd1cdd8c9457 ) require ( diff --git a/go.sum b/go.sum index b74872b..368bc93 100644 --- a/go.sum +++ b/go.sum @@ -48,9 +48,9 @@ github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+Pymzi github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= -github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= +github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= @@ -68,6 +68,8 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= +go.mau.fi/util v0.0.0-20230805161919-cf42c11d39c3 h1:r3Hrayw0CfmkrDhse7SbClYzq0e7/5P3iKpyV+gW16w= +go.mau.fi/util v0.0.0-20230805161919-cf42c11d39c3/go.mod h1:tNxQ2KpD+QhP2MlMfJvFSGSJfDjg4OhIwP7bIK43X/I= go.mau.fi/whatsmeow v0.0.0-20230804210635-04e4e3a38f73 h1:JK9z+AWKvWXINSqz6wqfF67L/uUMEMJ+Mfw4sVMFnQg= go.mau.fi/whatsmeow v0.0.0-20230804210635-04e4e3a38f73/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= @@ -131,5 +133,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.15.5-0.20230729114956-ff77fa8ec663 h1:tsXiuaWnBe9Xa/ms8tUhsJ4x7tPbQJ94fugqBIC0vsw= -maunium.net/go/mautrix v0.15.5-0.20230729114956-ff77fa8ec663/go.mod h1:dBaDmsnOOBM4a+gKcgefXH73pHGXm+MCJzCs1dXFgrw= +maunium.net/go/mautrix v0.15.5-0.20230805162445-fd1cdd8c9457 h1:VBKrYpAojLXdvR0RKi6uLqUmmfQLAf96SRdHv1G9occ= +maunium.net/go/mautrix v0.15.5-0.20230805162445-fd1cdd8c9457/go.mod h1:gpOJUT7EgK7vZTbewOrhWzNNjjJqwVahGRfTVg3BQko= diff --git a/historysync.go b/historysync.go index aba3646..175746e 100644 --- a/historysync.go +++ b/historysync.go @@ -23,8 +23,8 @@ import ( "time" "github.com/rs/zerolog" - "maunium.net/go/mautrix/util/variationselector" - + "go.mau.fi/util/dbutil" + "go.mau.fi/util/variationselector" waProto "go.mau.fi/whatsmeow/binary/proto" "go.mau.fi/whatsmeow/types" @@ -32,7 +32,6 @@ import ( "maunium.net/go/mautrix/appservice" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" "maunium.net/go/mautrix-whatsapp/config" "maunium.net/go/mautrix-whatsapp/database" diff --git a/main.go b/main.go index 71f98f1..04675bc 100644 --- a/main.go +++ b/main.go @@ -27,6 +27,7 @@ import ( "google.golang.org/protobuf/proto" + "go.mau.fi/util/configupgrade" "go.mau.fi/whatsmeow" waProto "go.mau.fi/whatsmeow/binary/proto" "go.mau.fi/whatsmeow/store" @@ -38,7 +39,6 @@ import ( "maunium.net/go/mautrix/bridge/status" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/configupgrade" "maunium.net/go/mautrix-whatsapp/config" "maunium.net/go/mautrix-whatsapp/database" diff --git a/portal.go b/portal.go index 93cc53f..d9ad3f0 100644 --- a/portal.go +++ b/portal.go @@ -44,12 +44,20 @@ import ( "github.com/chai2010/webp" "github.com/rs/zerolog" "github.com/tidwall/gjson" + "go.mau.fi/util/dbutil" + "go.mau.fi/util/exerrors" + "go.mau.fi/util/exmime" + "go.mau.fi/util/ffmpeg" + "go.mau.fi/util/random" + "go.mau.fi/util/variationselector" + "go.mau.fi/whatsmeow" + waProto "go.mau.fi/whatsmeow/binary/proto" + "go.mau.fi/whatsmeow/types" + "go.mau.fi/whatsmeow/types/events" "golang.org/x/exp/slices" "golang.org/x/image/draw" "google.golang.org/protobuf/proto" - log "maunium.net/go/maulogger/v2" - "maunium.net/go/mautrix" "maunium.net/go/mautrix/appservice" "maunium.net/go/mautrix/bridge" @@ -58,15 +66,6 @@ import ( "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/format" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util" - "maunium.net/go/mautrix/util/dbutil" - "maunium.net/go/mautrix/util/ffmpeg" - "maunium.net/go/mautrix/util/variationselector" - - "go.mau.fi/whatsmeow" - waProto "go.mau.fi/whatsmeow/binary/proto" - "go.mau.fi/whatsmeow/types" - "go.mau.fi/whatsmeow/types/events" "maunium.net/go/mautrix-whatsapp/database" ) @@ -2335,7 +2334,7 @@ func (portal *Portal) convertListMessage(intent *appservice.IntentAPI, source *U body = fmt.Sprintf("%s\n\n%s", msg.GetTitle(), body) } } - randomID := util.RandomString(64) + randomID := random.String(64) body = fmt.Sprintf("%s\n%s", body, randomID) if msg.GetFooterText() != "" { body = fmt.Sprintf("%s\n\n%s", body, msg.GetFooterText()) @@ -2943,7 +2942,7 @@ func (portal *Portal) convertMediaMessageContent(intent *appservice.IntentAPI, m content.Body = mimeClass } - content.Body += util.ExtensionFromMimetype(msg.GetMimetype()) + content.Body += exmime.ExtensionFromMimetype(msg.GetMimetype()) } msgWithDuration, ok := msg.(MediaMessageWithDuration) @@ -3495,12 +3494,12 @@ func (portal *Portal) preprocessMatrixMedia(ctx context.Context, sender *User, r } data, err := portal.MainIntent().DownloadBytesContext(ctx, mxc) if err != nil { - return nil, util.NewDualError(errMediaDownloadFailed, err) + return nil, exerrors.NewDualError(errMediaDownloadFailed, err) } if file != nil { err = file.DecryptInPlace(data) if err != nil { - return nil, util.NewDualError(errMediaDecryptFailed, err) + return nil, exerrors.NewDualError(errMediaDecryptFailed, err) } } mimeType := content.GetInfo().MimeType @@ -3555,7 +3554,7 @@ func (portal *Portal) preprocessMatrixMedia(ctx context.Context, sender *User, r } if convertErr != nil { if content.Info.MimeType != mimeType || data == nil { - return nil, util.NewDualError(fmt.Errorf("%w (%s to %s)", errMediaConvertFailed, mimeType, content.Info.MimeType), convertErr) + return nil, exerrors.NewDualError(fmt.Errorf("%w (%s to %s)", errMediaConvertFailed, mimeType, content.Info.MimeType), convertErr) } else { // If the mime type didn't change and the errored conversion function returned the original data, just log a warning and continue portal.log.Warnfln("Failed to re-encode %s media: %v, continuing with original file", mimeType, convertErr) @@ -3563,7 +3562,7 @@ func (portal *Portal) preprocessMatrixMedia(ctx context.Context, sender *User, r } uploadResp, err := sender.Client.Upload(ctx, data, mediaType) if err != nil { - return nil, util.NewDualError(errMediaWhatsAppUploadFailed, err) + return nil, exerrors.NewDualError(errMediaWhatsAppUploadFailed, err) } // Audio doesn't have thumbnails From 6ad533c0ae0936afc2197b25e059b91801f84c9c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 5 Aug 2023 20:03:40 +0300 Subject: [PATCH 37/94] Add zerolog for message handling context --- portal.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/portal.go b/portal.go index d9ad3f0..bd157cf 100644 --- a/portal.go +++ b/portal.go @@ -4121,6 +4121,10 @@ func (portal *Portal) generateMessageInfo(sender *User) *types.MessageInfo { func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event, timings messageTimings) { start := time.Now() ms := metricSender{portal: portal, timings: &timings} + log := portal.zlog.With(). + Str("event_id", evt.ID.String()). + Str("action", "handle matrix message"). + Logger() allowRelay := evt.Type != TypeMSC3381PollResponse && evt.Type != TypeMSC3381V2PollResponse && evt.Type != TypeMSC3381PollStart if err := portal.canBridgeFrom(sender, allowRelay, true); err != nil { @@ -4179,6 +4183,7 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event, timing ctx, cancel = context.WithTimeout(ctx, deadline) defer cancel() } + ctx = log.WithContext(ctx) timings.preproc = time.Since(start) start = time.Now() From dcd4ca366cd1230275aa695f5ef5a17fa48b1b42 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 8 Aug 2023 15:11:30 +0300 Subject: [PATCH 38/94] Update whatsmeow and ignore events from @lid users --- database/user.go | 6 ++++-- formatting.go | 3 +++ go.mod | 2 +- go.sum | 4 ++-- portal.go | 30 ++++++++++++++++++++++++++---- puppet.go | 3 +++ user.go | 10 ++++++++++ 7 files changed, 49 insertions(+), 9 deletions(-) diff --git a/database/user.go b/database/user.go index 088e998..a4ebc35 100644 --- a/database/user.go +++ b/database/user.go @@ -122,14 +122,16 @@ func (user *User) usernamePtr() *string { func (user *User) agentPtr() *uint8 { if !user.JID.IsEmpty() { - return &user.JID.Agent + zero := uint8(0) + return &zero } return nil } func (user *User) devicePtr() *uint8 { if !user.JID.IsEmpty() { - return &user.JID.Device + device := uint8(user.JID.Device) + return &device } return nil } diff --git a/formatting.go b/formatting.go index 3a06575..a60eb99 100644 --- a/formatting.go +++ b/formatting.go @@ -141,6 +141,9 @@ func (formatter *Formatter) ParseWhatsApp(roomID id.RoomID, content *event.Messa continue } else if jid.Server == types.LegacyUserServer { jid.Server = types.DefaultUserServer + } else if jid.Server != types.DefaultUserServer { + // TODO lid support? + continue } mxid, displayname := formatter.getMatrixInfoByJID(roomID, jid) number := "@" + jid.User diff --git a/go.mod b/go.mod index 245154e..4d95023 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.14.4 go.mau.fi/util v0.0.0-20230805161919-cf42c11d39c3 - go.mau.fi/whatsmeow v0.0.0-20230804210635-04e4e3a38f73 + go.mau.fi/whatsmeow v0.0.0-20230808115051-056c25e4b485 golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 golang.org/x/image v0.9.0 golang.org/x/net v0.12.0 diff --git a/go.sum b/go.sum index 368bc93..7970dda 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= go.mau.fi/util v0.0.0-20230805161919-cf42c11d39c3 h1:r3Hrayw0CfmkrDhse7SbClYzq0e7/5P3iKpyV+gW16w= go.mau.fi/util v0.0.0-20230805161919-cf42c11d39c3/go.mod h1:tNxQ2KpD+QhP2MlMfJvFSGSJfDjg4OhIwP7bIK43X/I= -go.mau.fi/whatsmeow v0.0.0-20230804210635-04e4e3a38f73 h1:JK9z+AWKvWXINSqz6wqfF67L/uUMEMJ+Mfw4sVMFnQg= -go.mau.fi/whatsmeow v0.0.0-20230804210635-04e4e3a38f73/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= +go.mau.fi/whatsmeow v0.0.0-20230808115051-056c25e4b485 h1:AaWJS6eKFOQ606PAvrhFeNco4hZ0ullUdQGPJ+G6HCQ= +go.mau.fi/whatsmeow v0.0.0-20230808115051-056c25e4b485/go.mod h1:Iv3G4uv6+HWtqL7XSLRa2dSy077Bnji14IvqUbG+bRo= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/portal.go b/portal.go index bd157cf..a0f1f8f 100644 --- a/portal.go +++ b/portal.go @@ -352,6 +352,10 @@ func (portal *Portal) handleMatrixMessageLoopItem(msg PortalMatrixMessage) { } func (portal *Portal) handleReceipt(receipt *events.Receipt, source *User) { + if receipt.Sender.Server != types.DefaultUserServer { + // TODO handle lids + return + } // The order of the message ID array depends on the sender's platform, so we just have to find // the last message based on timestamp. Also, timestamps only have second precision, so if // there are many messages at the same second just mark them all as read, because we don't @@ -703,6 +707,11 @@ func (portal *Portal) handleFakeMessage(msg fakeMessage) { portal.log.Debugfln("Not handling %s (fake): message is duplicate", msg.ID) return } + if msg.Sender.Server != types.DefaultUserServer { + portal.log.Debugfln("Not handling %s (fake): message is from a lid user (%s)", msg.ID, msg.Sender) + // TODO handle lids + return + } intent := portal.bridge.GetPuppetByJID(msg.Sender).IntentFor(portal) if !intent.IsCustomPuppet && portal.IsPrivateChat() && msg.Sender.User == portal.Key.Receiver.User && portal.Key.Receiver != portal.Key.JID { portal.log.Debugfln("Not handling %s (fake): user doesn't have double puppeting enabled", msg.ID) @@ -1046,6 +1055,10 @@ func (portal *Portal) SyncParticipants(source *User, metadata *types.GroupInfo) participantMap := make(map[types.JID]bool) userIDs := make([]id.UserID, 0, len(metadata.Participants)) for _, participant := range metadata.Participants { + if participant.JID.IsEmpty() || participant.JID.Server != types.DefaultUserServer { + // TODO handle lids + continue + } portal.log.Debugfln("Syncing participant %s (admin: %t)", participant.JID, participant.IsAdmin) participantMap[participant.JID] = true puppet := portal.bridge.GetPuppetByJID(participant.JID) @@ -1167,7 +1180,7 @@ func (portal *Portal) UpdateAvatar(user *User, setBy types.JID, updateInfo bool) if len(portal.MXID) > 0 { intent := portal.MainIntent() - if !setBy.IsEmpty() { + if !setBy.IsEmpty() && setBy.Server == types.DefaultUserServer { intent = portal.bridge.GetPuppetByJID(setBy).IntentFor(portal) } _, err := intent.SetRoomAvatar(portal.MXID, portal.AvatarURL) @@ -1205,7 +1218,7 @@ func (portal *Portal) UpdateName(name string, setBy types.JID, updateInfo bool) portal.UpdateBridgeInfo() } else if len(portal.MXID) > 0 { intent := portal.MainIntent() - if !setBy.IsEmpty() { + if !setBy.IsEmpty() && setBy.Server == types.DefaultUserServer { intent = portal.bridge.GetPuppetByJID(setBy).IntentFor(portal) } _, err := intent.SetRoomName(portal.MXID, name) @@ -1234,7 +1247,7 @@ func (portal *Portal) UpdateTopic(topic string, setBy types.JID, updateInfo bool portal.TopicSet = false intent := portal.MainIntent() - if !setBy.IsEmpty() { + if !setBy.IsEmpty() && setBy.Server == types.DefaultUserServer { intent = portal.bridge.GetPuppetByJID(setBy).IntentFor(portal) } _, err := intent.SetRoomTopic(portal.MXID, topic) @@ -1405,6 +1418,10 @@ func (portal *Portal) ChangeAdminStatus(jids []types.JID, setAdmin bool) id.Even } changed := portal.applyPowerLevelFixes(levels) for _, jid := range jids { + if jid.Server != types.DefaultUserServer { + // TODO handle lids + continue + } puppet := portal.bridge.GetPuppetByJID(jid) changed = levels.EnsureUserLevel(puppet.MXID, newLevel) || changed @@ -1936,7 +1953,8 @@ func (portal *Portal) addReplyMention(content *event.MessageEventContent, sender if content.Mentions == nil || (sender.IsEmpty() && senderMXID == "") { return } - if senderMXID == "" { + // TODO handle lids + if senderMXID == "" && sender.Server == types.DefaultUserServer { if user := portal.bridge.GetUserByJID(sender); user != nil { senderMXID = user.MXID } else { @@ -3244,6 +3262,10 @@ func (portal *Portal) handleMediaRetry(retry *events.MediaRetry, source *User) { } else { puppet = portal.bridge.GetPuppetByJID(retry.SenderID) } + if puppet == nil { + // TODO handle lids? + return + } intent := puppet.IntentFor(portal) retryData, err := whatsmeow.DecryptMediaRetryNotification(retry, meta.Media.Key) diff --git a/puppet.go b/puppet.go index e1e45d3..0c3f9ac 100644 --- a/puppet.go +++ b/puppet.go @@ -330,6 +330,9 @@ func (puppet *Puppet) updatePortalName() { } func (puppet *Puppet) SyncContact(source *User, onlyIfNoName, shouldHavePushName bool, reason string) { + if puppet == nil { + return + } if onlyIfNoName && len(puppet.Displayname) > 0 && (!shouldHavePushName || puppet.NameQuality > config.NameQualityPhone) { source.EnqueuePuppetResync(puppet) return diff --git a/user.go b/user.go index 5ccfbe5..9c276e3 100644 --- a/user.go +++ b/user.go @@ -896,6 +896,9 @@ func (user *User) HandleEvent(event interface{}) { user.handleCallStart(v.CallCreator, v.CallID, v.Type, v.Timestamp) case *events.IdentityChange: puppet := user.bridge.GetPuppetByJID(v.JID) + if puppet == nil { + return + } portal := user.GetPortalByJID(v.JID) if len(portal.MXID) > 0 && user.bridge.Config.Bridge.IdentityChangeNotices { text := fmt.Sprintf("Your security code with %s changed.", puppet.Displayname) @@ -1221,6 +1224,9 @@ const WATypingTimeout = 15 * time.Second func (user *User) handleChatPresence(presence *events.ChatPresence) { puppet := user.bridge.GetPuppetByJID(presence.Sender) + if puppet == nil { + return + } portal := user.GetPortalByJID(presence.Chat) if puppet == nil || portal == nil || len(portal.MXID) == 0 { return @@ -1341,6 +1347,10 @@ func (user *User) handleGroupUpdate(evt *events.GroupInfo) { log.Debug().Msg("Ignoring group info update in chat with no portal") return } + if evt.Sender != nil && evt.Sender.Server == types.HiddenUserServer { + log.Debug().Str("sender", evt.Sender.String()).Msg("Ignoring group info update from @lid user") + return + } switch { case evt.Announce != nil: log.Debug().Msg("Group announcement mode (message send permission) changed") From 32d26ec0ab1b7221504148ae4144707651c69fa7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 9 Aug 2023 15:31:21 +0300 Subject: [PATCH 39/94] Add support for round video messages --- portal.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/portal.go b/portal.go index a0f1f8f..fa30ee7 100644 --- a/portal.go +++ b/portal.go @@ -410,7 +410,7 @@ func containsSupportedMessage(waMsg *waProto.Message) bool { return false } return waMsg.Conversation != nil || waMsg.ExtendedTextMessage != nil || waMsg.ImageMessage != nil || - waMsg.StickerMessage != nil || waMsg.AudioMessage != nil || waMsg.VideoMessage != nil || + waMsg.StickerMessage != nil || waMsg.AudioMessage != nil || waMsg.VideoMessage != nil || waMsg.PtvMessage != nil || waMsg.DocumentMessage != nil || waMsg.ContactMessage != nil || waMsg.LocationMessage != nil || waMsg.LiveLocationMessage != nil || waMsg.GroupInviteMessage != nil || waMsg.ContactsArrayMessage != nil || waMsg.HighlyStructuredMessage != nil || waMsg.TemplateMessage != nil || waMsg.TemplateButtonReplyMessage != nil || @@ -429,6 +429,8 @@ func getMessageType(waMsg *waProto.Message) string { return fmt.Sprintf("sticker %s", waMsg.GetStickerMessage().GetMimetype()) case waMsg.VideoMessage != nil: return fmt.Sprintf("video %s", waMsg.GetVideoMessage().GetMimetype()) + case waMsg.PtvMessage != nil: + return fmt.Sprintf("round video %s", waMsg.GetPtvMessage().GetMimetype()) case waMsg.AudioMessage != nil: return fmt.Sprintf("audio %s", waMsg.GetAudioMessage().GetMimetype()) case waMsg.DocumentMessage != nil: @@ -578,6 +580,8 @@ func (portal *Portal) convertMessage(intent *appservice.IntentAPI, source *User, return portal.convertMediaMessage(intent, source, info, waMsg.GetStickerMessage(), "sticker", isBackfill) case waMsg.VideoMessage != nil: return portal.convertMediaMessage(intent, source, info, waMsg.GetVideoMessage(), "video attachment", isBackfill) + case waMsg.PtvMessage != nil: + return portal.convertMediaMessage(intent, source, info, waMsg.GetPtvMessage(), "video message", isBackfill) case waMsg.AudioMessage != nil: typeName := "audio attachment" if waMsg.GetAudioMessage().GetPtt() { From 205d4e651b7786ae7b32a3cded4c66cfe1f5b2de Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 13 Aug 2023 00:54:51 +0300 Subject: [PATCH 40/94] Fix more places where lids aren't ignored --- portal.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/portal.go b/portal.go index fa30ee7..a8ea9dc 100644 --- a/portal.go +++ b/portal.go @@ -640,7 +640,7 @@ func (portal *Portal) UpdateGroupDisappearingMessages(sender *types.JID, timesta portal.ExpirationTime = timer portal.Update(nil) intent := portal.MainIntent() - if sender != nil { + if sender != nil && sender.Server == types.DefaultUserServer { intent = portal.bridge.GetPuppetByJID(sender.ToNonAD()).IntentFor(portal) } else { sender = &types.EmptyJID @@ -2766,6 +2766,10 @@ func (portal *Portal) HandleWhatsAppKick(source *User, senderJID types.JID, jids sender := portal.bridge.GetPuppetByJID(senderJID) senderIntent := sender.IntentFor(portal) for _, jid := range jids { + if jid.Server != types.DefaultUserServer { + // TODO handle lids + continue + } //if source != nil && source.JID.User == jid.User { // portal.log.Debugln("Ignoring self-kick by", source.MXID) // continue @@ -2793,6 +2797,10 @@ func (portal *Portal) HandleWhatsAppInvite(source *User, senderJID *types.JID, j intent = sender.IntentFor(portal) } for _, jid := range jids { + if jid.Server != types.DefaultUserServer { + // TODO handle lids + continue + } puppet := portal.bridge.GetPuppetByJID(jid) puppet.SyncContact(source, true, false, "handling whatsapp invite") resp, err := intent.SendStateEvent(portal.MXID, event.StateMember, puppet.MXID.String(), &event.MemberEventContent{ From 8fffc2840bbdbd8f6741ef03bba878474bb337e3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 13 Aug 2023 00:55:50 +0300 Subject: [PATCH 41/94] Update Go version and dependencies --- .github/workflows/go.yml | 2 +- go.mod | 24 +++++++++++----------- go.sum | 44 ++++++++++++++++++++-------------------- 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 3861af0..965969e 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -11,7 +11,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: "1.20" + go-version: "1.21" - name: Install libolm run: sudo apt-get install libolm-dev libolm3 diff --git a/go.mod b/go.mod index 4d95023..e6a393b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module maunium.net/go/mautrix-whatsapp -go 1.19 +go 1.20 require ( github.com/chai2010/webp v1.1.1 @@ -11,15 +11,15 @@ require ( github.com/prometheus/client_golang v1.16.0 github.com/rs/zerolog v1.30.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e - github.com/tidwall/gjson v1.14.4 - go.mau.fi/util v0.0.0-20230805161919-cf42c11d39c3 - go.mau.fi/whatsmeow v0.0.0-20230808115051-056c25e4b485 - golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 - golang.org/x/image v0.9.0 - golang.org/x/net v0.12.0 + github.com/tidwall/gjson v1.16.0 + go.mau.fi/util v0.0.0-20230805171708-199bf3eec776 + go.mau.fi/whatsmeow v0.0.0-20230809185103-1bdbbb5766d3 + golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb + golang.org/x/image v0.11.0 + golang.org/x/net v0.14.0 google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.15.5-0.20230805162445-fd1cdd8c9457 + maunium.net/go/mautrix v0.15.5-0.20230811131248-82eaf14f46cb ) require ( @@ -39,12 +39,12 @@ require ( github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/sjson v1.2.5 // indirect - github.com/yuin/goldmark v1.5.4 // indirect + github.com/yuin/goldmark v1.5.5 // indirect go.mau.fi/libsignal v0.1.0 // indirect go.mau.fi/zeroconfig v0.1.2 // indirect - golang.org/x/crypto v0.11.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect + golang.org/x/crypto v0.12.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect maunium.net/go/mauflag v1.0.0 // indirect diff --git a/go.sum b/go.sum index 7970dda..f0b79bc 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,8 @@ github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= -github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg= +github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= @@ -64,32 +64,32 @@ github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhso github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= -github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.5.5 h1:IJznPe8wOzfIKETmMkd06F8nXkmlhaHqFRM9l1hAGsU= +github.com/yuin/goldmark v1.5.5/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= -go.mau.fi/util v0.0.0-20230805161919-cf42c11d39c3 h1:r3Hrayw0CfmkrDhse7SbClYzq0e7/5P3iKpyV+gW16w= -go.mau.fi/util v0.0.0-20230805161919-cf42c11d39c3/go.mod h1:tNxQ2KpD+QhP2MlMfJvFSGSJfDjg4OhIwP7bIK43X/I= -go.mau.fi/whatsmeow v0.0.0-20230808115051-056c25e4b485 h1:AaWJS6eKFOQ606PAvrhFeNco4hZ0ullUdQGPJ+G6HCQ= -go.mau.fi/whatsmeow v0.0.0-20230808115051-056c25e4b485/go.mod h1:Iv3G4uv6+HWtqL7XSLRa2dSy077Bnji14IvqUbG+bRo= +go.mau.fi/util v0.0.0-20230805171708-199bf3eec776 h1:VrxDCO/gLFHLQywGUsJzertrvt2mUEMrZPf4hEL/s18= +go.mau.fi/util v0.0.0-20230805171708-199bf3eec776/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84= +go.mau.fi/whatsmeow v0.0.0-20230809185103-1bdbbb5766d3 h1:x3BXsTUP3dOaXKkrSpfqpeNhWABC+y+3FLXMea2MSjg= +go.mau.fi/whatsmeow v0.0.0-20230809185103-1bdbbb5766d3/go.mod h1:Iv3G4uv6+HWtqL7XSLRa2dSy077Bnji14IvqUbG+bRo= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= -golang.org/x/image v0.9.0 h1:QrzfX26snvCM20hIhBwuHI/ThTg18b/+kcKdXHvnR+g= -golang.org/x/image v0.9.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb h1:mIKbk8weKhSeLH2GmUTrvx8CjkyJmnU1wFmg59CUjFA= +golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo= +golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -102,8 +102,8 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -111,8 +111,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -133,5 +133,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.15.5-0.20230805162445-fd1cdd8c9457 h1:VBKrYpAojLXdvR0RKi6uLqUmmfQLAf96SRdHv1G9occ= -maunium.net/go/mautrix v0.15.5-0.20230805162445-fd1cdd8c9457/go.mod h1:gpOJUT7EgK7vZTbewOrhWzNNjjJqwVahGRfTVg3BQko= +maunium.net/go/mautrix v0.15.5-0.20230811131248-82eaf14f46cb h1:qoJvn/iMHOj25Sur2GyIWoq1Fzf7vfgMepC74k54/Cs= +maunium.net/go/mautrix v0.15.5-0.20230811131248-82eaf14f46cb/go.mod h1:XAjE9pTSGcr6vXaiNgQGiip7tddJ8FQV1a29u2QdBG4= From 4024d704b5c5b0f9fa0f7f6cbf051fb618221d65 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 17 Aug 2023 01:09:25 +0300 Subject: [PATCH 42/94] Bump version to v0.10.0 --- CHANGELOG.md | 9 ++++++++- go.mod | 4 ++-- go.sum | 8 ++++---- main.go | 2 +- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b51dc0a..6256ffc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,14 @@ -# unreleased +# v0.10.0 (2023-08-16) +* Bumped minimum Go version to 1.20. +* Added automatic re-requesting of undecryptable WhatsApp messages from primary + device. +* Added support for round video messages. * Added support for logging in by entering a 8-letter code on the phone instead of scanning a QR code. + * Note: due to a server-side change, code login may only work when `os_name` + and `browser_name` in the config are set in a specific way. A future release + will automatically change the values to always work with code login. # v0.9.0 (2023-07-16) diff --git a/go.mod b/go.mod index e6a393b..fa8651f 100644 --- a/go.mod +++ b/go.mod @@ -13,13 +13,13 @@ require ( github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.16.0 go.mau.fi/util v0.0.0-20230805171708-199bf3eec776 - go.mau.fi/whatsmeow v0.0.0-20230809185103-1bdbbb5766d3 + go.mau.fi/whatsmeow v0.0.0-20230816173759-58beaf3b5bd0 golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb golang.org/x/image v0.11.0 golang.org/x/net v0.14.0 google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.15.5-0.20230811131248-82eaf14f46cb + maunium.net/go/mautrix v0.16.0 ) require ( diff --git a/go.sum b/go.sum index f0b79bc..af708f9 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= go.mau.fi/util v0.0.0-20230805171708-199bf3eec776 h1:VrxDCO/gLFHLQywGUsJzertrvt2mUEMrZPf4hEL/s18= go.mau.fi/util v0.0.0-20230805171708-199bf3eec776/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84= -go.mau.fi/whatsmeow v0.0.0-20230809185103-1bdbbb5766d3 h1:x3BXsTUP3dOaXKkrSpfqpeNhWABC+y+3FLXMea2MSjg= -go.mau.fi/whatsmeow v0.0.0-20230809185103-1bdbbb5766d3/go.mod h1:Iv3G4uv6+HWtqL7XSLRa2dSy077Bnji14IvqUbG+bRo= +go.mau.fi/whatsmeow v0.0.0-20230816173759-58beaf3b5bd0 h1:BPjAk+ndCpkg+QPUi44ACgnhdjMA5nMa5DV4bK/5kXw= +go.mau.fi/whatsmeow v0.0.0-20230816173759-58beaf3b5bd0/go.mod h1:Iv3G4uv6+HWtqL7XSLRa2dSy077Bnji14IvqUbG+bRo= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -133,5 +133,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.15.5-0.20230811131248-82eaf14f46cb h1:qoJvn/iMHOj25Sur2GyIWoq1Fzf7vfgMepC74k54/Cs= -maunium.net/go/mautrix v0.15.5-0.20230811131248-82eaf14f46cb/go.mod h1:XAjE9pTSGcr6vXaiNgQGiip7tddJ8FQV1a29u2QdBG4= +maunium.net/go/mautrix v0.16.0 h1:iUqCzJE2yqBC1ddAK6eAn159My8rLb4X8g4SFtQh2Dk= +maunium.net/go/mautrix v0.16.0/go.mod h1:XAjE9pTSGcr6vXaiNgQGiip7tddJ8FQV1a29u2QdBG4= diff --git a/main.go b/main.go index 04675bc..abd65df 100644 --- a/main.go +++ b/main.go @@ -262,7 +262,7 @@ func main() { Name: "mautrix-whatsapp", URL: "https://github.com/mautrix/whatsapp", Description: "A Matrix-WhatsApp puppeting bridge.", - Version: "0.9.0", + Version: "0.10.0", ProtocolName: "WhatsApp", BeeperServiceName: "whatsapp", BeeperNetworkName: "whatsapp", From 4ad2a9e318b9dee29db7a2a82b38f1147e29ebc6 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 17 Aug 2023 11:38:45 +0300 Subject: [PATCH 43/94] Use hardcoded UA for pair code --- commands.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/commands.go b/commands.go index 7ba89ce..7025466 100644 --- a/commands.go +++ b/commands.go @@ -465,7 +465,7 @@ func fnLogin(ce *WrappedCommandEvent) { } if phoneNumber != "" { - pairingCode, err := ce.User.Client.PairPhone(phoneNumber, true) + pairingCode, err := ce.User.Client.PairPhone(phoneNumber, true, whatsmeow.PairClientChrome, "Chrome (Linux)") if err != nil { ce.ZLog.Err(err).Msg("Failed to start phone code login") ce.Reply("Failed to start phone code login: %v", err) diff --git a/go.mod b/go.mod index fa8651f..6182d09 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.16.0 go.mau.fi/util v0.0.0-20230805171708-199bf3eec776 - go.mau.fi/whatsmeow v0.0.0-20230816173759-58beaf3b5bd0 + go.mau.fi/whatsmeow v0.0.0-20230817083005-1c185f033d88 golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb golang.org/x/image v0.11.0 golang.org/x/net v0.14.0 diff --git a/go.sum b/go.sum index af708f9..7d5a8ca 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= go.mau.fi/util v0.0.0-20230805171708-199bf3eec776 h1:VrxDCO/gLFHLQywGUsJzertrvt2mUEMrZPf4hEL/s18= go.mau.fi/util v0.0.0-20230805171708-199bf3eec776/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84= -go.mau.fi/whatsmeow v0.0.0-20230816173759-58beaf3b5bd0 h1:BPjAk+ndCpkg+QPUi44ACgnhdjMA5nMa5DV4bK/5kXw= -go.mau.fi/whatsmeow v0.0.0-20230816173759-58beaf3b5bd0/go.mod h1:Iv3G4uv6+HWtqL7XSLRa2dSy077Bnji14IvqUbG+bRo= +go.mau.fi/whatsmeow v0.0.0-20230817083005-1c185f033d88 h1:OyWiw4j6s8unaOhRuIYO1pTnJLFREIRnflDDp+GNJ2U= +go.mau.fi/whatsmeow v0.0.0-20230817083005-1c185f033d88/go.mod h1:Iv3G4uv6+HWtqL7XSLRa2dSy077Bnji14IvqUbG+bRo= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= From 6178ced1fea5f7b53a6bdb3d57daeb1a17eb7607 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 21 Aug 2023 12:21:31 +0300 Subject: [PATCH 44/94] Make incoming call message clearer --- user.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/user.go b/user.go index 9c276e3..8c110b4 100644 --- a/user.go +++ b/user.go @@ -655,9 +655,9 @@ func (user *User) handleCallStart(sender types.JID, id, callType string, ts time return } portal := user.GetPortalByJID(sender) - text := "Incoming call" + text := "Incoming call. Use the WhatsApp app to answer." if callType != "" { - text = fmt.Sprintf("Incoming %s call", callType) + text = fmt.Sprintf("Incoming %s call. Use the WhatsApp app to answer.", callType) } portal.messages <- PortalMessage{ fake: &fakeMessage{ From b85fddee27392db6ca46446272605e81703a9853 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 21 Aug 2023 12:22:41 +0300 Subject: [PATCH 45/94] Update whatsmeow --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6182d09..2fb04ad 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.16.0 go.mau.fi/util v0.0.0-20230805171708-199bf3eec776 - go.mau.fi/whatsmeow v0.0.0-20230817083005-1c185f033d88 + go.mau.fi/whatsmeow v0.0.0-20230821092221-636dac24aed9 golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb golang.org/x/image v0.11.0 golang.org/x/net v0.14.0 diff --git a/go.sum b/go.sum index 7d5a8ca..45b12f9 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= go.mau.fi/util v0.0.0-20230805171708-199bf3eec776 h1:VrxDCO/gLFHLQywGUsJzertrvt2mUEMrZPf4hEL/s18= go.mau.fi/util v0.0.0-20230805171708-199bf3eec776/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84= -go.mau.fi/whatsmeow v0.0.0-20230817083005-1c185f033d88 h1:OyWiw4j6s8unaOhRuIYO1pTnJLFREIRnflDDp+GNJ2U= -go.mau.fi/whatsmeow v0.0.0-20230817083005-1c185f033d88/go.mod h1:Iv3G4uv6+HWtqL7XSLRa2dSy077Bnji14IvqUbG+bRo= +go.mau.fi/whatsmeow v0.0.0-20230821092221-636dac24aed9 h1:qmFclwp06TPlpmDcbMOeTDgOI7rV2bdGyejMcOOCwZ4= +go.mau.fi/whatsmeow v0.0.0-20230821092221-636dac24aed9/go.mod h1:Iv3G4uv6+HWtqL7XSLRa2dSy077Bnji14IvqUbG+bRo= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= From bd01c661ef45c7f78c124a22c4e7d734256156dc Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 21 Aug 2023 13:52:12 +0300 Subject: [PATCH 46/94] Move double puppeting login code to mautrix-go --- commands.go | 7 +- config/bridge.go | 10 +- config/config.go | 2 +- config/upgrade.go | 1 - custompuppet.go | 250 +++++--------------------------------------- example-config.yaml | 6 +- go.mod | 2 +- go.sum | 4 +- user.go | 24 ----- 9 files changed, 38 insertions(+), 268 deletions(-) diff --git a/commands.go b/commands.go index 7025466..1dcd8da 100644 --- a/commands.go +++ b/commands.go @@ -559,12 +559,7 @@ func fnLogout(ce *WrappedCommandEvent) { return } puppet := ce.Bridge.GetPuppetByJID(ce.User.JID) - if puppet.CustomMXID != "" { - err := puppet.SwitchCustomMXID("", "") - if err != nil { - ce.User.log.Warnln("Failed to logout-matrix while logging out of WhatsApp:", err) - } - } + puppet.ClearCustomMXID() err := ce.User.Client.Logout() if err != nil { ce.User.log.Warnln("Error while logging out:", err) diff --git a/config/bridge.go b/config/bridge.go index 2393cef..4e5a8b0 100644 --- a/config/bridge.go +++ b/config/bridge.go @@ -85,18 +85,14 @@ type BridgeConfig struct { UserAvatarSync bool `yaml:"user_avatar_sync"` BridgeMatrixLeave bool `yaml:"bridge_matrix_leave"` - SyncWithCustomPuppets bool `yaml:"sync_with_custom_puppets"` SyncDirectChatList bool `yaml:"sync_direct_chat_list"` SyncManualMarkedUnread bool `yaml:"sync_manual_marked_unread"` - DefaultBridgeReceipts bool `yaml:"default_bridge_receipts"` DefaultBridgePresence bool `yaml:"default_bridge_presence"` SendPresenceOnTyping bool `yaml:"send_presence_on_typing"` ForceActiveDeliveryReceipts bool `yaml:"force_active_delivery_receipts"` - DoublePuppetServerMap map[string]string `yaml:"double_puppet_server_map"` - DoublePuppetAllowDiscovery bool `yaml:"double_puppet_allow_discovery"` - LoginSharedSecretMap map[string]string `yaml:"login_shared_secret_map"` + DoublePuppetConfig bridgeconfig.DoublePuppetConfig `yaml:",inline"` PrivateChatPortalMeta string `yaml:"private_chat_portal_meta"` ParallelMemberSync bool `yaml:"parallel_member_sync"` @@ -151,6 +147,10 @@ type BridgeConfig struct { displaynameTemplate *template.Template `yaml:"-"` } +func (bc BridgeConfig) GetDoublePuppetConfig() bridgeconfig.DoublePuppetConfig { + return bc.DoublePuppetConfig +} + func (bc BridgeConfig) GetEncryptionConfig() bridgeconfig.EncryptionConfig { return bc.Encryption } diff --git a/config/config.go b/config/config.go index bb7ec25..2aaab9a 100644 --- a/config/config.go +++ b/config/config.go @@ -42,6 +42,6 @@ type Config struct { func (config *Config) CanAutoDoublePuppet(userID id.UserID) bool { _, homeserver, _ := userID.Parse() - _, hasSecret := config.Bridge.LoginSharedSecretMap[homeserver] + _, hasSecret := config.Bridge.DoublePuppetConfig.SharedSecretMap[homeserver] return hasSecret } diff --git a/config/upgrade.go b/config/upgrade.go index 02ae8fe..b405130 100644 --- a/config/upgrade.go +++ b/config/upgrade.go @@ -61,7 +61,6 @@ func DoUpgrade(helper *up.Helper) { helper.Copy(up.List, "bridge", "history_sync", "deferred") helper.Copy(up.Bool, "bridge", "user_avatar_sync") helper.Copy(up.Bool, "bridge", "bridge_matrix_leave") - helper.Copy(up.Bool, "bridge", "sync_with_custom_puppets") helper.Copy(up.Bool, "bridge", "sync_direct_chat_list") helper.Copy(up.Bool, "bridge", "default_bridge_receipts") helper.Copy(up.Bool, "bridge", "default_bridge_presence") diff --git a/custompuppet.go b/custompuppet.go index 8fe6099..af00a28 100644 --- a/custompuppet.go +++ b/custompuppet.go @@ -17,262 +17,66 @@ package main import ( - "crypto/hmac" - "crypto/sha512" - "encoding/hex" - "errors" - "fmt" - "time" - - "maunium.net/go/mautrix" - "maunium.net/go/mautrix/appservice" - "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" ) -var ( - ErrNoCustomMXID = errors.New("no custom mxid set") - ErrMismatchingMXID = errors.New("whoami result does not match custom mxid") -) - func (puppet *Puppet) SwitchCustomMXID(accessToken string, mxid id.UserID) error { - prevCustomMXID := puppet.CustomMXID - if puppet.customIntent != nil { - puppet.stopSyncing() - } puppet.CustomMXID = mxid puppet.AccessToken = accessToken - + puppet.EnablePresence = puppet.bridge.Config.Bridge.DefaultBridgePresence + puppet.Update() err := puppet.StartCustomMXID(false) if err != nil { return err } - - if len(prevCustomMXID) > 0 { - delete(puppet.bridge.puppetsByCustomMXID, prevCustomMXID) - } - if len(puppet.CustomMXID) > 0 { - puppet.bridge.puppetsByCustomMXID[puppet.CustomMXID] = puppet - } - puppet.EnablePresence = puppet.bridge.Config.Bridge.DefaultBridgePresence - puppet.EnableReceipts = puppet.bridge.Config.Bridge.DefaultBridgeReceipts - puppet.bridge.AS.StateStore.MarkRegistered(puppet.CustomMXID) - puppet.Update() // TODO leave rooms with default puppet return nil } -func (puppet *Puppet) loginWithSharedSecret(mxid id.UserID) (string, error) { - _, homeserver, _ := mxid.Parse() - puppet.log.Debugfln("Logging into %s with shared secret", mxid) - loginSecret := puppet.bridge.Config.Bridge.LoginSharedSecretMap[homeserver] - client, err := puppet.bridge.newDoublePuppetClient(mxid, "") - if err != nil { - return "", fmt.Errorf("failed to create mautrix client to log in: %v", err) - } - req := mautrix.ReqLogin{ - Identifier: mautrix.UserIdentifier{Type: mautrix.IdentifierTypeUser, User: string(mxid)}, - DeviceID: "WhatsApp Bridge", - InitialDeviceDisplayName: "WhatsApp Bridge", - } - if loginSecret == "appservice" { - client.AccessToken = puppet.bridge.AS.Registration.AppToken - req.Type = mautrix.AuthTypeAppservice - } else { - mac := hmac.New(sha512.New, []byte(loginSecret)) - mac.Write([]byte(mxid)) - req.Password = hex.EncodeToString(mac.Sum(nil)) - req.Type = mautrix.AuthTypePassword - } - resp, err := client.Login(&req) - if err != nil { - return "", err - } - return resp.AccessToken, nil -} - -func (br *WABridge) newDoublePuppetClient(mxid id.UserID, accessToken string) (*mautrix.Client, error) { - _, homeserver, err := mxid.Parse() - if err != nil { - return nil, err - } - homeserverURL, found := br.Config.Bridge.DoublePuppetServerMap[homeserver] - if !found { - if homeserver == br.AS.HomeserverDomain { - homeserverURL = "" - } else if br.Config.Bridge.DoublePuppetAllowDiscovery { - resp, err := mautrix.DiscoverClientAPI(homeserver) - if err != nil { - return nil, fmt.Errorf("failed to find homeserver URL for %s: %v", homeserver, err) - } - homeserverURL = resp.Homeserver.BaseURL - br.Log.Debugfln("Discovered URL %s for %s to enable double puppeting for %s", homeserverURL, homeserver, mxid) - } else { - return nil, fmt.Errorf("double puppeting from %s is not allowed", homeserver) - } - } - return br.AS.NewExternalMautrixClient(mxid, accessToken, homeserverURL) -} - -func (puppet *Puppet) newCustomIntent() (*appservice.IntentAPI, error) { - if len(puppet.CustomMXID) == 0 { - return nil, ErrNoCustomMXID - } - client, err := puppet.bridge.newDoublePuppetClient(puppet.CustomMXID, puppet.AccessToken) - if err != nil { - return nil, err - } - client.Syncer = puppet - client.Store = puppet - - ia := puppet.bridge.AS.NewIntentAPI("custom") - ia.Client = client - ia.Localpart, _, _ = puppet.CustomMXID.Parse() - ia.UserID = puppet.CustomMXID - ia.IsCustomPuppet = true - return ia, nil -} - -func (puppet *Puppet) clearCustomMXID() { +func (puppet *Puppet) ClearCustomMXID() { + save := puppet.CustomMXID != "" || puppet.AccessToken != "" puppet.CustomMXID = "" puppet.AccessToken = "" puppet.customIntent = nil puppet.customUser = nil + if save { + puppet.Update() + } } func (puppet *Puppet) StartCustomMXID(reloginOnFail bool) error { - if len(puppet.CustomMXID) == 0 { - puppet.clearCustomMXID() - return nil - } - intent, err := puppet.newCustomIntent() + newIntent, newAccessToken, err := puppet.bridge.DoublePuppet.Setup(puppet.CustomMXID, puppet.AccessToken, reloginOnFail) if err != nil { - puppet.clearCustomMXID() + puppet.ClearCustomMXID() return err } - resp, err := intent.Whoami() - if err != nil { - if !reloginOnFail || (errors.Is(err, mautrix.MUnknownToken) && !puppet.tryRelogin(err, "initializing double puppeting")) { - puppet.clearCustomMXID() - return err - } - intent.AccessToken = puppet.AccessToken - } else if resp.UserID != puppet.CustomMXID { - puppet.clearCustomMXID() - return ErrMismatchingMXID + if puppet.AccessToken != newAccessToken { + puppet.AccessToken = newAccessToken + puppet.Update() } - puppet.customIntent = intent + puppet.customIntent = newIntent puppet.customUser = puppet.bridge.GetUserByMXID(puppet.CustomMXID) - puppet.startSyncing() return nil } -func (puppet *Puppet) startSyncing() { - if !puppet.bridge.Config.Bridge.SyncWithCustomPuppets { +func (user *User) tryAutomaticDoublePuppeting() { + if !user.bridge.Config.CanAutoDoublePuppet(user.MXID) { return } - go func() { - puppet.log.Debugln("Starting syncing...") - puppet.customIntent.SyncPresence = "offline" - err := puppet.customIntent.Sync() - if err != nil { - puppet.log.Errorln("Fatal error syncing:", err) - } - }() -} - -func (puppet *Puppet) stopSyncing() { - if !puppet.bridge.Config.Bridge.SyncWithCustomPuppets { + user.zlog.Debug().Msg("Checking if double puppeting needs to be enabled") + puppet := user.bridge.GetPuppetByJID(user.JID) + if len(puppet.CustomMXID) > 0 { + user.zlog.Debug().Msg("User already has double-puppeting enabled") + // Custom puppet already enabled return } - puppet.customIntent.StopSync() -} - -func (puppet *Puppet) ProcessResponse(resp *mautrix.RespSync, _ string) error { - if !puppet.customUser.IsLoggedIn() { - puppet.log.Debugln("Skipping sync processing: custom user not connected to whatsapp") - return nil - } - for roomID, events := range resp.Rooms.Join { - for _, evt := range events.Ephemeral.Events { - evt.RoomID = roomID - err := evt.Content.ParseRaw(evt.Type) - if err != nil { - continue - } - switch evt.Type { - case event.EphemeralEventReceipt: - if puppet.EnableReceipts { - go puppet.bridge.MatrixHandler.HandleReceipt(evt) - } - case event.EphemeralEventTyping: - go puppet.bridge.MatrixHandler.HandleTyping(evt) - } - } - } - if puppet.EnablePresence { - for _, evt := range resp.Presence.Events { - if evt.Sender != puppet.CustomMXID { - continue - } - err := evt.Content.ParseRaw(evt.Type) - if err != nil { - continue - } - go puppet.bridge.HandlePresence(evt) - } - } - return nil -} - -func (puppet *Puppet) tryRelogin(cause error, action string) bool { - if !puppet.bridge.Config.CanAutoDoublePuppet(puppet.CustomMXID) { - return false - } - puppet.log.Debugfln("Trying to relogin after '%v' while %s", cause, action) - accessToken, err := puppet.loginWithSharedSecret(puppet.CustomMXID) + puppet.CustomMXID = user.MXID + puppet.EnablePresence = puppet.bridge.Config.Bridge.DefaultBridgePresence + err := puppet.StartCustomMXID(true) if err != nil { - puppet.log.Errorfln("Failed to relogin after '%v' while %s: %v", cause, action, err) - return false - } - puppet.log.Infofln("Successfully relogined after '%v' while %s", cause, action) - puppet.AccessToken = accessToken - return true -} - -func (puppet *Puppet) OnFailedSync(_ *mautrix.RespSync, err error) (time.Duration, error) { - puppet.log.Warnln("Sync error:", err) - if errors.Is(err, mautrix.MUnknownToken) { - if !puppet.tryRelogin(err, "syncing") { - return 0, err - } - puppet.customIntent.AccessToken = puppet.AccessToken - return 0, nil - } - return 10 * time.Second, nil -} - -func (puppet *Puppet) GetFilterJSON(_ id.UserID) *mautrix.Filter { - everything := []event.Type{{Type: "*"}} - return &mautrix.Filter{ - Presence: mautrix.FilterPart{ - Senders: []id.UserID{puppet.CustomMXID}, - Types: []event.Type{event.EphemeralEventPresence}, - }, - AccountData: mautrix.FilterPart{NotTypes: everything}, - Room: mautrix.RoomFilter{ - Ephemeral: mautrix.FilterPart{Types: []event.Type{event.EphemeralEventTyping, event.EphemeralEventReceipt}}, - IncludeLeave: false, - AccountData: mautrix.FilterPart{NotTypes: everything}, - State: mautrix.FilterPart{NotTypes: everything}, - Timeline: mautrix.FilterPart{NotTypes: everything}, - }, + user.zlog.Warn().Err(err).Msg("Failed to login with shared secret") + } else { + // TODO leave rooms with default puppet + user.zlog.Debug().Msg("Successfully automatically enabled custom puppet") } } - -func (puppet *Puppet) SaveFilterID(_ id.UserID, _ string) {} -func (puppet *Puppet) SaveNextBatch(_ id.UserID, nbt string) { puppet.NextBatch = nbt; puppet.Update() } -func (puppet *Puppet) SaveRoom(_ *mautrix.Room) {} -func (puppet *Puppet) LoadFilterID(_ id.UserID) string { return "" } -func (puppet *Puppet) LoadNextBatch(_ id.UserID) string { return puppet.NextBatch } -func (puppet *Puppet) LoadRoom(_ id.RoomID) *mautrix.Room { return nil } diff --git a/example-config.yaml b/example-config.yaml index f5a347a..be56be3 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -65,7 +65,6 @@ appservice: # Whether or not to receive ephemeral events via appservice transactions. # Requires MSC2409 support (i.e. Synapse 1.22+). - # You should disable bridge -> sync_with_custom_puppets when this is enabled. ephemeral_events: true # Should incoming events be handled asynchronously? @@ -211,8 +210,6 @@ bridge: user_avatar_sync: true # Should Matrix users leaving groups be bridged to WhatsApp? bridge_matrix_leave: true - # Should the bridge sync with double puppeting to receive EDUs that aren't normally sent to appservices. - sync_with_custom_puppets: false # Should the bridge update the m.direct account data event when double puppeting is enabled. # Note that updating the m.direct event is not atomic (except with mautrix-asmux) # and is therefore prone to race conditions. @@ -223,9 +220,8 @@ bridge: # com.famedly.marked_unread room account data. sync_manual_marked_unread: true # When double puppeting is enabled, users can use `!wa toggle` to change whether - # presence and read receipts are bridged. These settings set the default values. + # presence is bridged. This setting sets the default value. # Existing users won't be affected when these are changed. - default_bridge_receipts: true default_bridge_presence: true # Send the presence as "available" to whatsapp when users start typing on a portal. # This works as a workaround for homeservers that do not support presence, and allows diff --git a/go.mod b/go.mod index 2fb04ad..4e92fc7 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( golang.org/x/net v0.14.0 google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.16.0 + maunium.net/go/mautrix v0.16.1-0.20230821105106-ac5c2c22102c ) require ( diff --git a/go.sum b/go.sum index 45b12f9..9f5cbcb 100644 --- a/go.sum +++ b/go.sum @@ -133,5 +133,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.16.0 h1:iUqCzJE2yqBC1ddAK6eAn159My8rLb4X8g4SFtQh2Dk= -maunium.net/go/mautrix v0.16.0/go.mod h1:XAjE9pTSGcr6vXaiNgQGiip7tddJ8FQV1a29u2QdBG4= +maunium.net/go/mautrix v0.16.1-0.20230821105106-ac5c2c22102c h1:oRIaFbS4ds9biwJVguT+9Zu7n5zDbKQeuGklXHQxvCU= +maunium.net/go/mautrix v0.16.1-0.20230821105106-ac5c2c22102c/go.mod h1:XAjE9pTSGcr6vXaiNgQGiip7tddJ8FQV1a29u2QdBG4= diff --git a/user.go b/user.go index 8c110b4..02c8b57 100644 --- a/user.go +++ b/user.go @@ -612,30 +612,6 @@ func (user *User) IsLoggedIn() bool { return user.IsConnected() && user.Client.IsLoggedIn() } -func (user *User) tryAutomaticDoublePuppeting() { - if !user.bridge.Config.CanAutoDoublePuppet(user.MXID) { - return - } - user.log.Debugln("Checking if double puppeting needs to be enabled") - puppet := user.bridge.GetPuppetByJID(user.JID) - if len(puppet.CustomMXID) > 0 { - user.log.Debugln("User already has double-puppeting enabled") - // Custom puppet already enabled - return - } - accessToken, err := puppet.loginWithSharedSecret(user.MXID) - if err != nil { - user.log.Warnln("Failed to login with shared secret:", err) - return - } - err = puppet.SwitchCustomMXID(accessToken, user.MXID) - if err != nil { - puppet.log.Warnln("Failed to switch to auto-logined custom puppet:", err) - return - } - user.log.Infoln("Successfully automatically enabled custom puppet") -} - func (user *User) sendMarkdownBridgeAlert(formatString string, args ...interface{}) { if user.bridge.Config.Bridge.DisableBridgeAlerts { return From 5fc1e0cebcb9099bdff28d807e7f4fa99a722aa3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 22 Aug 2023 19:00:58 +0300 Subject: [PATCH 47/94] Fix updating puppetsByCustomMXID map --- custompuppet.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/custompuppet.go b/custompuppet.go index af00a28..47ae104 100644 --- a/custompuppet.go +++ b/custompuppet.go @@ -35,6 +35,11 @@ func (puppet *Puppet) SwitchCustomMXID(accessToken string, mxid id.UserID) error func (puppet *Puppet) ClearCustomMXID() { save := puppet.CustomMXID != "" || puppet.AccessToken != "" + puppet.bridge.puppetsLock.Lock() + if puppet.CustomMXID != "" && puppet.bridge.puppetsByCustomMXID[puppet.CustomMXID] == puppet { + delete(puppet.bridge.puppetsByCustomMXID, puppet.CustomMXID) + } + puppet.bridge.puppetsLock.Unlock() puppet.CustomMXID = "" puppet.AccessToken = "" puppet.customIntent = nil @@ -50,6 +55,9 @@ func (puppet *Puppet) StartCustomMXID(reloginOnFail bool) error { puppet.ClearCustomMXID() return err } + puppet.bridge.puppetsLock.Lock() + puppet.bridge.puppetsByCustomMXID[puppet.CustomMXID] = puppet + puppet.bridge.puppetsLock.Unlock() if puppet.AccessToken != newAccessToken { puppet.AccessToken = newAccessToken puppet.Update() From 82484ff079537bfcb8f296e4f63e479c2927d663 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 24 Aug 2023 18:17:15 +0300 Subject: [PATCH 48/94] Update whatsmeow --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4e92fc7..793d492 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.16.0 go.mau.fi/util v0.0.0-20230805171708-199bf3eec776 - go.mau.fi/whatsmeow v0.0.0-20230821092221-636dac24aed9 + go.mau.fi/whatsmeow v0.0.0-20230824151650-6da2abde6b7c golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb golang.org/x/image v0.11.0 golang.org/x/net v0.14.0 diff --git a/go.sum b/go.sum index 9f5cbcb..35fc99c 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= go.mau.fi/util v0.0.0-20230805171708-199bf3eec776 h1:VrxDCO/gLFHLQywGUsJzertrvt2mUEMrZPf4hEL/s18= go.mau.fi/util v0.0.0-20230805171708-199bf3eec776/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84= -go.mau.fi/whatsmeow v0.0.0-20230821092221-636dac24aed9 h1:qmFclwp06TPlpmDcbMOeTDgOI7rV2bdGyejMcOOCwZ4= -go.mau.fi/whatsmeow v0.0.0-20230821092221-636dac24aed9/go.mod h1:Iv3G4uv6+HWtqL7XSLRa2dSy077Bnji14IvqUbG+bRo= +go.mau.fi/whatsmeow v0.0.0-20230824151650-6da2abde6b7c h1:GH1QtL+bbkwyzgJBLyBQcr499oHR1y8svzB92KtG6OY= +go.mau.fi/whatsmeow v0.0.0-20230824151650-6da2abde6b7c/go.mod h1:Iv3G4uv6+HWtqL7XSLRa2dSy077Bnji14IvqUbG+bRo= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= From e3a93680b2de6abe886f72890373c59bbc5d6975 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 28 Aug 2023 14:52:14 +0300 Subject: [PATCH 49/94] Fix syncing community announcement groups --- CHANGELOG.md | 9 +++++++++ portal.go | 1 + 2 files changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6256ffc..274fa62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# unreleased + +* Added support for double puppeting with arbitrary `as_token`s. + See [docs](https://docs.mau.fi/bridges/general/double-puppeting.html#appservice-method-new) for more info. +* Fixed logging in with 8-letter code. +* Fixed syncing community announcement groups. +* Changed "Incoming call" message to explicitly say you have to open WhatsApp + on your phone to answer. + # v0.10.0 (2023-08-16) * Bumped minimum Go version to 1.20. diff --git a/portal.go b/portal.go index a8ea9dc..47b7e1b 100644 --- a/portal.go +++ b/portal.go @@ -1060,6 +1060,7 @@ func (portal *Portal) SyncParticipants(source *User, metadata *types.GroupInfo) userIDs := make([]id.UserID, 0, len(metadata.Participants)) for _, participant := range metadata.Participants { if participant.JID.IsEmpty() || participant.JID.Server != types.DefaultUserServer { + wg.Done() // TODO handle lids continue } From 015cc1f19f65e37e923675903e5fabf550983ef7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 1 Sep 2023 14:50:00 +0300 Subject: [PATCH 50/94] Add delivery status to MSS events --- messagetracking.go | 13 +++++++++---- portal.go | 29 +++++++++++++++++++++++++++++ user.go | 2 +- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/messagetracking.go b/messagetracking.go index d1c91e6..7066726 100644 --- a/messagetracking.go +++ b/messagetracking.go @@ -147,7 +147,7 @@ func (portal *Portal) sendErrorMessage(evt *event.Event, err error, msgType stri return resp.EventID } -func (portal *Portal) sendStatusEvent(evtID, lastRetry id.EventID, err error) { +func (portal *Portal) sendStatusEvent(evtID, lastRetry id.EventID, err error, deliveredTo *[]id.UserID) { if !portal.bridge.Config.Bridge.MessageStatusEvents { return } @@ -165,7 +165,8 @@ func (portal *Portal) sendStatusEvent(evtID, lastRetry id.EventID, err error) { Type: event.RelReference, EventID: evtID, }, - LastRetry: lastRetry, + DeliveredToUsers: deliveredTo, + LastRetry: lastRetry, } if err == nil { content.Status = event.MessageStatusSuccess @@ -224,12 +225,16 @@ func (portal *Portal) sendMessageMetrics(evt *event.Event, err error, part strin if sendNotice { ms.setNoticeID(portal.sendErrorMessage(evt, err, msgType, isCertain, ms.getNoticeID())) } - portal.sendStatusEvent(origEvtID, evt.ID, err) + portal.sendStatusEvent(origEvtID, evt.ID, err, nil) } else { portal.log.Debugfln("Handled Matrix %s %s", msgType, evtDescription) portal.sendDeliveryReceipt(evt.ID) portal.bridge.SendMessageSuccessCheckpoint(evt, status.MsgStepRemote, ms.getRetryNum()) - portal.sendStatusEvent(origEvtID, evt.ID, nil) + var deliveredTo *[]id.UserID + if portal.IsPrivateChat() { + deliveredTo = &[]id.UserID{} + } + portal.sendStatusEvent(origEvtID, evt.ID, nil, deliveredTo) if prevNotice := ms.popNoticeID(); prevNotice != "" { _, _ = portal.MainIntent().RedactEvent(portal.MXID, prevNotice, mautrix.ReqRedact{ Reason: "error resolved", diff --git a/portal.go b/portal.go index 47b7e1b..527a2ea 100644 --- a/portal.go +++ b/portal.go @@ -48,6 +48,7 @@ import ( "go.mau.fi/util/exerrors" "go.mau.fi/util/exmime" "go.mau.fi/util/ffmpeg" + "go.mau.fi/util/jsontime" "go.mau.fi/util/random" "go.mau.fi/util/variationselector" "go.mau.fi/whatsmeow" @@ -62,6 +63,7 @@ import ( "maunium.net/go/mautrix/appservice" "maunium.net/go/mautrix/bridge" "maunium.net/go/mautrix/bridge/bridgeconfig" + "maunium.net/go/mautrix/bridge/status" "maunium.net/go/mautrix/crypto/attachment" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/format" @@ -351,11 +353,38 @@ func (portal *Portal) handleMatrixMessageLoopItem(msg PortalMatrixMessage) { } } +func (portal *Portal) handleDeliveryReceipt(receipt *events.Receipt, source *User) { + if !portal.IsPrivateChat() { + return + } + for _, msgID := range receipt.MessageIDs { + msg := portal.bridge.DB.Message.GetByJID(portal.Key, msgID) + if msg == nil || msg.IsFakeMXID() { + continue + } + if msg.Sender == source.JID { + portal.bridge.SendRawMessageCheckpoint(&status.MessageCheckpoint{ + EventID: msg.MXID, + RoomID: portal.MXID, + Step: status.MsgStepRemote, + Timestamp: jsontime.UM(receipt.Timestamp), + Status: status.MsgStatusDelivered, + ReportedBy: status.MsgReportedByBridge, + }) + portal.sendStatusEvent(msg.MXID, "", nil, &[]id.UserID{portal.MainIntent().UserID}) + } + } +} + func (portal *Portal) handleReceipt(receipt *events.Receipt, source *User) { if receipt.Sender.Server != types.DefaultUserServer { // TODO handle lids return } + if receipt.Type == events.ReceiptTypeDelivered { + portal.handleDeliveryReceipt(receipt, source) + return + } // The order of the message ID array depends on the sender's platform, so we just have to find // the last message based on timestamp. Also, timestamps only have second precision, so if // there are many messages at the same second just mark them all as read, because we don't diff --git a/user.go b/user.go index 02c8b57..9adfde9 100644 --- a/user.go +++ b/user.go @@ -1224,7 +1224,7 @@ func (user *User) handleChatPresence(presence *events.ChatPresence) { } func (user *User) handleReceipt(receipt *events.Receipt) { - if receipt.Type != events.ReceiptTypeRead && receipt.Type != events.ReceiptTypeReadSelf { + if receipt.Type != events.ReceiptTypeRead && receipt.Type != events.ReceiptTypeReadSelf && receipt.Type != events.ReceiptTypeDelivered { return } portal := user.GetPortalByMessageSource(receipt.MessageSource) From c704191e2cc887e00fa95c11e54f5dca063914d1 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 3 Sep 2023 01:32:25 +0300 Subject: [PATCH 51/94] Catch panics in portal message handler --- portal.go | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/portal.go b/portal.go index 527a2ea..38779fb 100644 --- a/portal.go +++ b/portal.go @@ -298,7 +298,7 @@ var ( _ bridge.TypingPortal = (*Portal)(nil) ) -func (portal *Portal) handleMessageLoopItem(msg PortalMessage) { +func (portal *Portal) handleWhatsAppMessageLoopItem(msg PortalMessage) { if len(portal.MXID) == 0 { if msg.fake == nil && msg.undecryptable == nil && (msg.evt == nil || !containsSupportedMessage(msg.evt.Message)) { portal.log.Debugln("Not creating portal room for incoming message: message is not a chat message") @@ -423,14 +423,31 @@ func (portal *Portal) handleReceipt(receipt *events.Receipt, source *User) { func (portal *Portal) handleMessageLoop() { for { - select { - case msg := <-portal.messages: - portal.handleMessageLoopItem(msg) - case msg := <-portal.matrixMessages: - portal.handleMatrixMessageLoopItem(msg) - case retry := <-portal.mediaRetries: - portal.handleMediaRetry(retry.evt, retry.source) + portal.handleOneMessageLoopItem() + } +} + +func (portal *Portal) handleOneMessageLoopItem() { + defer func() { + if err := recover(); err != nil { + logEvt := portal.zlog.WithLevel(zerolog.FatalLevel). + Str(zerolog.ErrorStackFieldName, string(debug.Stack())) + actualErr, ok := err.(error) + if ok { + logEvt = logEvt.Err(actualErr) + } else { + logEvt = logEvt.Any(zerolog.ErrorFieldName, err) + } + logEvt.Msg("Portal message handler panicked") } + }() + select { + case msg := <-portal.messages: + portal.handleWhatsAppMessageLoopItem(msg) + case msg := <-portal.matrixMessages: + portal.handleMatrixMessageLoopItem(msg) + case retry := <-portal.mediaRetries: + portal.handleMediaRetry(retry.evt, retry.source) } } From 1ccd16626ab3e778e254c6612e744f5c073589f5 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 4 Sep 2023 12:20:49 +0300 Subject: [PATCH 52/94] Clarify that the login command doesn't require the phone number --- commands.go | 8 +++++--- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/commands.go b/commands.go index 1dcd8da..5aa0d5c 100644 --- a/commands.go +++ b/commands.go @@ -430,9 +430,11 @@ var cmdLogin = &commands.FullHandler{ Func: wrapCommand(fnLogin), Name: "login", Help: commands.HelpMeta{ - Section: commands.HelpSectionAuth, - Description: "Link the bridge to your WhatsApp account as a web client.", - Args: "[_phone number_]", + Section: commands.HelpSectionAuth, + Description: "Link the bridge to your WhatsApp account as a web client. " + + "The phone number parameter is optional: if provided, the bridge will create a 8-character login code " + + "that can be used instead of the QR code.", + Args: "[_phone number_]", }, } diff --git a/go.mod b/go.mod index 793d492..cbf461b 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( golang.org/x/net v0.14.0 google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.16.1-0.20230821105106-ac5c2c22102c + maunium.net/go/mautrix v0.16.1-0.20230904092006-3fffe3f31ce1 ) require ( diff --git a/go.sum b/go.sum index 35fc99c..d682cfc 100644 --- a/go.sum +++ b/go.sum @@ -133,5 +133,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.16.1-0.20230821105106-ac5c2c22102c h1:oRIaFbS4ds9biwJVguT+9Zu7n5zDbKQeuGklXHQxvCU= -maunium.net/go/mautrix v0.16.1-0.20230821105106-ac5c2c22102c/go.mod h1:XAjE9pTSGcr6vXaiNgQGiip7tddJ8FQV1a29u2QdBG4= +maunium.net/go/mautrix v0.16.1-0.20230904092006-3fffe3f31ce1 h1:o4SOY1Aim54E7DDCPh+Iyq+gBjnvOwKKBvziXftki9o= +maunium.net/go/mautrix v0.16.1-0.20230904092006-3fffe3f31ce1/go.mod h1:XAjE9pTSGcr6vXaiNgQGiip7tddJ8FQV1a29u2QdBG4= From feff16f3032f00a137d6afad675eccdfa44cf261 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 4 Sep 2023 15:34:48 +0300 Subject: [PATCH 53/94] Remove leftover debug print --- portal.go | 1 - 1 file changed, 1 deletion(-) diff --git a/portal.go b/portal.go index 38779fb..7b81ccc 100644 --- a/portal.go +++ b/portal.go @@ -3887,7 +3887,6 @@ func (portal *Portal) convertMatrixPollStart(_ context.Context, sender *User, ev if maxAnswers >= len(content.PollStart.Answers) || maxAnswers < 0 { maxAnswers = 0 } - fmt.Printf("%+v\n", content.PollStart) ctxInfo := portal.generateContextInfo(content.RelatesTo) var question string question, ctxInfo.MentionedJid = portal.msc1767ToWhatsApp(content.PollStart.Question, true) From 8b1308595f7f2f3b775a2b65249de622dc70a443 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 4 Sep 2023 19:28:15 +0300 Subject: [PATCH 54/94] Add support for collecting incoming galleries into single event --- config/bridge.go | 1 + config/upgrade.go | 1 + database/message.go | 30 ++++++++++--- example-config.yaml | 2 + historysync.go | 2 +- portal.go | 102 +++++++++++++++++++++++++++++++++++++++----- 6 files changed, 119 insertions(+), 19 deletions(-) diff --git a/config/bridge.go b/config/bridge.go index 4e5a8b0..689c263 100644 --- a/config/bridge.go +++ b/config/bridge.go @@ -111,6 +111,7 @@ type BridgeConfig struct { FederateRooms bool `yaml:"federate_rooms"` URLPreviews bool `yaml:"url_previews"` CaptionInMessage bool `yaml:"caption_in_message"` + BeeperGalleries bool `yaml:"beeper_galleries"` ExtEvPolls bool `yaml:"extev_polls"` CrossRoomReplies bool `yaml:"cross_room_replies"` DisableReplyFallbacks bool `yaml:"disable_reply_fallbacks"` diff --git a/config/upgrade.go b/config/upgrade.go index b405130..f307651 100644 --- a/config/upgrade.go +++ b/config/upgrade.go @@ -103,6 +103,7 @@ func DoUpgrade(helper *up.Helper) { helper.Copy(up.Bool, "bridge", "crash_on_stream_replaced") helper.Copy(up.Bool, "bridge", "url_previews") helper.Copy(up.Bool, "bridge", "caption_in_message") + helper.Copy(up.Bool, "bridge", "beeper_galleries") if intPolls, ok := helper.Get(up.Int, "bridge", "extev_polls"); ok { val := "false" if intPolls != "0" { diff --git a/database/message.go b/database/message.go index 8f35de8..dbd7e9c 100644 --- a/database/message.go +++ b/database/message.go @@ -19,6 +19,7 @@ package database import ( "database/sql" "errors" + "fmt" "strings" "time" @@ -133,12 +134,13 @@ const ( type MessageType string const ( - MsgUnknown MessageType = "" - MsgFake MessageType = "fake" - MsgNormal MessageType = "message" - MsgReaction MessageType = "reaction" - MsgEdit MessageType = "edit" - MsgMatrixPoll MessageType = "matrix-poll" + MsgUnknown MessageType = "" + MsgFake MessageType = "fake" + MsgNormal MessageType = "message" + MsgReaction MessageType = "reaction" + MsgEdit MessageType = "edit" + MsgMatrixPoll MessageType = "matrix-poll" + MsgBeeperGallery MessageType = "beeper-gallery" ) type Message struct { @@ -155,6 +157,8 @@ type Message struct { Type MessageType Error MessageErrorType + GalleryPart int + BroadcastListJID types.JID } @@ -166,6 +170,8 @@ func (msg *Message) IsFakeJID() bool { return strings.HasPrefix(msg.JID, "FAKE::") || msg.JID == string(msg.MXID) } +const fakeGalleryMXIDFormat = "com.beeper.gallery::%d:%s" + func (msg *Message) Scan(row dbutil.Scannable) *Message { var ts int64 err := row.Scan(&msg.Chat.JID, &msg.Chat.Receiver, &msg.JID, &msg.MXID, &msg.Sender, &msg.SenderMXID, &ts, &msg.Sent, &msg.Type, &msg.Error, &msg.BroadcastListJID) @@ -175,6 +181,12 @@ func (msg *Message) Scan(row dbutil.Scannable) *Message { } return nil } + if strings.HasPrefix(msg.MXID.String(), "com.beeper.gallery::") { + _, err = fmt.Sscanf(msg.MXID.String(), fakeGalleryMXIDFormat, &msg.GalleryPart, &msg.MXID) + if err != nil { + msg.log.Errorln("Parsing gallery MXID failed:", err) + } + } if ts != 0 { msg.Timestamp = time.Unix(ts, 0) } @@ -190,11 +202,15 @@ func (msg *Message) Insert(txn dbutil.Execable) { if msg.Sender.IsEmpty() { sender = "" } + mxid := msg.MXID.String() + if msg.GalleryPart != 0 { + mxid = fmt.Sprintf(fakeGalleryMXIDFormat, msg.GalleryPart, mxid) + } _, err := txn.Exec(` INSERT INTO message (chat_jid, chat_receiver, jid, mxid, sender, sender_mxid, timestamp, sent, type, error, broadcast_list_jid) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) - `, msg.Chat.JID, msg.Chat.Receiver, msg.JID, msg.MXID, sender, msg.SenderMXID, msg.Timestamp.Unix(), msg.Sent, msg.Type, msg.Error, msg.BroadcastListJID) + `, msg.Chat.JID, msg.Chat.Receiver, msg.JID, mxid, sender, msg.SenderMXID, msg.Timestamp.Unix(), msg.Sent, msg.Type, msg.Error, msg.BroadcastListJID) if err != nil { msg.log.Warnfln("Failed to insert %s@%s: %v", msg.Chat, msg.JID, err) } diff --git a/example-config.yaml b/example-config.yaml index be56be3..33e5a79 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -300,6 +300,8 @@ bridge: # Send captions in the same message as images. This will send data compatible with both MSC2530 and MSC3552. # This is currently not supported in most clients. caption_in_message: false + # Send galleries as a single event? This is not an MSC (yet). + beeper_galleries: false # Should polls be sent using MSC3381 event types? extev_polls: false # Should cross-chat replies from WhatsApp be bridged? Most servers and clients don't support this. diff --git a/historysync.go b/historysync.go index 175746e..9e7ef75 100644 --- a/historysync.go +++ b/historysync.go @@ -850,7 +850,7 @@ func (portal *Portal) finishBatch(txn dbutil.Transaction, eventIDs []id.EventID, } eventID := eventIDs[i] - portal.markHandled(txn, nil, info.MessageInfo, eventID, info.SenderMXID, true, false, info.Type, info.Error) + portal.markHandled(txn, nil, info.MessageInfo, eventID, info.SenderMXID, true, false, info.Type, 0, info.Error) if info.Type == database.MsgReaction { portal.upsertReaction(txn, nil, info.ReactionTarget, info.Sender, eventID, info.ID) } diff --git a/portal.go b/portal.go index 7b81ccc..4d9004e 100644 --- a/portal.go +++ b/portal.go @@ -284,12 +284,50 @@ type Portal struct { mediaErrorCache map[types.MessageID]*FailedMediaMeta + galleryCache []*event.MessageEventContent + galleryCacheRootEvent id.EventID + galleryCacheStart time.Time + galleryCacheReplyTo *ReplyInfo + galleryCacheSender types.JID + currentlySleepingToDelete sync.Map relayUser *User parentPortal *Portal } +const GalleryMaxTime = 10 * time.Minute + +func (portal *Portal) stopGallery() { + if portal.galleryCache != nil { + portal.galleryCache = nil + portal.galleryCacheSender = types.EmptyJID + portal.galleryCacheReplyTo = nil + portal.galleryCacheStart = time.Time{} + portal.galleryCacheRootEvent = "" + } +} + +func (portal *Portal) startGallery(evt *events.Message, msg *ConvertedMessage) { + portal.galleryCache = []*event.MessageEventContent{msg.Content} + portal.galleryCacheSender = evt.Info.Sender.ToNonAD() + portal.galleryCacheReplyTo = msg.ReplyTo + portal.galleryCacheStart = time.Now() +} + +func (portal *Portal) extendGallery(msg *ConvertedMessage) int { + portal.galleryCache = append(portal.galleryCache, msg.Content) + msg.Content = &event.MessageEventContent{ + MsgType: event.MsgBeeperGallery, + Body: "Sent a gallery", + BeeperGalleryImages: portal.galleryCache, + } + msg.Content.SetEdit(portal.galleryCacheRootEvent) + // Don't set the gallery images in the edit fallback + msg.Content.BeeperGalleryImages = nil + return len(portal.galleryCache) - 1 +} + var ( _ bridge.Portal = (*Portal)(nil) _ bridge.ReadReceiptHandlingPortal = (*Portal)(nil) @@ -319,8 +357,10 @@ func (portal *Portal) handleWhatsAppMessageLoopItem(msg PortalMessage) { case msg.receipt != nil: portal.handleReceipt(msg.receipt, msg.source) case msg.undecryptable != nil: + portal.stopGallery() portal.handleUndecryptableMessage(msg.source, msg.undecryptable) case msg.fake != nil: + portal.stopGallery() msg.fake.ID = "FAKE::" + msg.fake.ID portal.handleFakeMessage(*msg.fake) default: @@ -746,7 +786,7 @@ func (portal *Portal) handleUndecryptableMessage(source *User, evt *events.Undec portal.log.Errorfln("Failed to send decryption error of %s to Matrix: %v", evt.Info.ID, err) return } - portal.finishHandling(nil, &evt.Info, resp.EventID, intent.UserID, database.MsgUnknown, database.MsgErrDecryptionFailed) + portal.finishHandling(nil, &evt.Info, resp.EventID, intent.UserID, database.MsgUnknown, 0, database.MsgErrDecryptionFailed) } func (portal *Portal) handleFakeMessage(msg fakeMessage) { @@ -784,7 +824,7 @@ func (portal *Portal) handleFakeMessage(msg fakeMessage) { MessageSource: types.MessageSource{ Sender: msg.Sender, }, - }, resp.EventID, intent.UserID, database.MsgFake, database.MsgNoError) + }, resp.EventID, intent.UserID, database.MsgFake, 0, database.MsgNoError) } } @@ -841,6 +881,17 @@ func (portal *Portal) handleMessage(source *User, evt *events.Message, historica } converted := portal.convertMessage(intent, source, &evt.Info, evt.Message, false) if converted != nil { + isGalleriable := portal.bridge.Config.Bridge.BeeperGalleries && + (evt.Message.ImageMessage != nil || evt.Message.VideoMessage != nil) && + (portal.galleryCache == nil || + (evt.Info.Sender.ToNonAD() == portal.galleryCacheSender && + converted.ReplyTo.Equals(portal.galleryCacheReplyTo) && + time.Since(portal.galleryCacheStart) < GalleryMaxTime)) && + // Captions aren't allowed in galleries (this needs to be checked before the caption is merged) + converted.Caption == nil && + // Images can't be edited + editTargetMsg == nil + if !historical && portal.IsPrivateChat() && evt.Info.Sender.Device == 0 && converted.ExpiresIn > 0 && portal.ExpirationTime == 0 { portal.zlog.Info(). Str("timer", converted.ExpiresIn.String()). @@ -871,6 +922,20 @@ func (portal *Portal) handleMessage(source *User, evt *events.Message, historica dbMsgType = database.MsgEdit converted.Content.SetEdit(editTargetMsg.MXID) } + galleryStarted := false + var galleryPart int + if isGalleriable { + if portal.galleryCache == nil { + portal.startGallery(evt, converted) + galleryStarted = true + } else { + galleryPart = portal.extendGallery(converted) + dbMsgType = database.MsgBeeperGallery + } + } else if editTargetMsg == nil { + // Stop collecting a gallery (except if it's an edit) + portal.stopGallery() + } resp, err := portal.sendMessage(converted.Intent, converted.Type, converted.Content, converted.Extra, evt.Info.Timestamp.UnixMilli()) if err != nil { portal.log.Errorfln("Failed to send %s to Matrix: %v", msgID, err) @@ -880,6 +945,11 @@ func (portal *Portal) handleMessage(source *User, evt *events.Message, historica } eventID = resp.EventID lastEventID = eventID + if galleryStarted { + portal.galleryCacheRootEvent = eventID + } else if galleryPart != 0 { + eventID = portal.galleryCacheRootEvent + } } // TODO figure out how to handle captions with undecryptable messages turning decryptable if converted.Caption != nil && existingMsg == nil && editTargetMsg == nil { @@ -912,7 +982,7 @@ func (portal *Portal) handleMessage(source *User, evt *events.Message, historica } } if len(eventID) != 0 { - portal.finishHandling(existingMsg, &evt.Info, eventID, intent.UserID, dbMsgType, converted.Error) + portal.finishHandling(existingMsg, &evt.Info, eventID, intent.UserID, dbMsgType, galleryPart, converted.Error) } } else if msgType == "reaction" || msgType == "encrypted reaction" { if evt.Message.GetEncReactionMessage() != nil { @@ -957,12 +1027,13 @@ func (portal *Portal) isRecentlyHandled(id types.MessageID, error database.Messa return false } -func (portal *Portal) markHandled(txn dbutil.Transaction, msg *database.Message, info *types.MessageInfo, mxid id.EventID, senderMXID id.UserID, isSent, recent bool, msgType database.MessageType, errType database.MessageErrorType) *database.Message { +func (portal *Portal) markHandled(txn dbutil.Transaction, msg *database.Message, info *types.MessageInfo, mxid id.EventID, senderMXID id.UserID, isSent, recent bool, msgType database.MessageType, galleryPart int, errType database.MessageErrorType) *database.Message { if msg == nil { msg = portal.bridge.DB.Message.New() msg.Chat = portal.Key msg.JID = info.ID msg.MXID = mxid + msg.GalleryPart = galleryPart msg.Timestamp = info.Timestamp msg.Sender = info.Sender msg.SenderMXID = senderMXID @@ -1017,8 +1088,8 @@ func (portal *Portal) getMessageIntent(user *User, info *types.MessageInfo, msgT return intent } -func (portal *Portal) finishHandling(existing *database.Message, message *types.MessageInfo, mxid id.EventID, senderMXID id.UserID, msgType database.MessageType, errType database.MessageErrorType) { - portal.markHandled(nil, existing, message, mxid, senderMXID, true, true, msgType, errType) +func (portal *Portal) finishHandling(existing *database.Message, message *types.MessageInfo, mxid id.EventID, senderMXID id.UserID, msgType database.MessageType, galleryPart int, errType database.MessageErrorType) { + portal.markHandled(nil, existing, message, mxid, senderMXID, true, true, msgType, galleryPart, errType) portal.sendDeliveryReceipt(mxid) var suffix string if errType == database.MsgErrDecryptionFailed { @@ -2100,7 +2171,7 @@ func (portal *Portal) HandleMessageReaction(intent *appservice.IntentAPI, user * if err != nil { portal.log.Errorfln("Failed to redact reaction %s/%s from %s to %s: %v", existing.MXID, existing.JID, info.Sender, targetJID, err) } - portal.finishHandling(existingMsg, info, resp.EventID, intent.UserID, database.MsgReaction, database.MsgNoError) + portal.finishHandling(existingMsg, info, resp.EventID, intent.UserID, database.MsgReaction, 0, database.MsgNoError) existing.Delete() } else { target := portal.bridge.DB.Message.GetByJID(portal.Key, targetJID) @@ -2121,7 +2192,7 @@ func (portal *Portal) HandleMessageReaction(intent *appservice.IntentAPI, user * return } - portal.finishHandling(existingMsg, info, resp.EventID, intent.UserID, database.MsgReaction, database.MsgNoError) + portal.finishHandling(existingMsg, info, resp.EventID, intent.UserID, database.MsgReaction, 0, database.MsgNoError) portal.upsertReaction(nil, intent, target.JID, info.Sender, resp.EventID, info.ID) } } @@ -2209,6 +2280,15 @@ type ReplyInfo struct { Sender types.JID } +func (r *ReplyInfo) Equals(other *ReplyInfo) bool { + if r == nil { + return other == nil + } else if other == nil { + return false + } + return r.MessageID == other.MessageID && r.Chat == other.Chat && r.Sender == other.Sender +} + func (r ReplyInfo) MarshalZerologObject(e *zerolog.Event) { e.Str("message_id", r.MessageID) e.Str("chat_jid", r.Chat.String()) @@ -3927,7 +4007,7 @@ func (portal *Portal) generateContextInfo(relatesTo *event.RelatesTo) *waProto.C replyToID := relatesTo.GetReplyTo() if len(replyToID) > 0 { replyToMsg := portal.bridge.DB.Message.GetByMXID(replyToID) - if replyToMsg != nil && !replyToMsg.IsFakeJID() && (replyToMsg.Type == database.MsgNormal || replyToMsg.Type == database.MsgMatrixPoll) { + if replyToMsg != nil && !replyToMsg.IsFakeJID() && (replyToMsg.Type == database.MsgNormal || replyToMsg.Type == database.MsgMatrixPoll || replyToMsg.Type == database.MsgBeeperGallery) { ctxInfo.StanzaId = &replyToMsg.JID ctxInfo.Participant = proto.String(replyToMsg.Sender.ToNonAD().String()) // Using blank content here seems to work fine on all official WhatsApp apps. @@ -4283,7 +4363,7 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event, timing } info := portal.generateMessageInfo(sender) if dbMsg == nil { - dbMsg = portal.markHandled(nil, nil, info, evt.ID, evt.Sender, false, true, dbMsgType, database.MsgNoError) + dbMsg = portal.markHandled(nil, nil, info, evt.ID, evt.Sender, false, true, dbMsgType, 0, database.MsgNoError) } else { info.ID = dbMsg.JID } @@ -4338,7 +4418,7 @@ func (portal *Portal) handleMatrixReaction(sender *User, evt *event.Event) error return fmt.Errorf("unknown target event %s", content.RelatesTo.EventID) } info := portal.generateMessageInfo(sender) - dbMsg := portal.markHandled(nil, nil, info, evt.ID, evt.Sender, false, true, database.MsgReaction, database.MsgNoError) + dbMsg := portal.markHandled(nil, nil, info, evt.ID, evt.Sender, false, true, database.MsgReaction, 0, database.MsgNoError) portal.upsertReaction(nil, nil, target.JID, sender.JID, evt.ID, info.ID) portal.log.Debugln("Sending reaction", evt.ID, "to WhatsApp", info.ID) resp, err := portal.sendReactionToWhatsApp(sender, info.ID, target, content.RelatesTo.Key, evt.Timestamp) From 65e7cd37a44409fd19dd2fc8d473f43639b20248 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 17:58:00 +0300 Subject: [PATCH 55/94] Update dependencies --- go.mod | 16 ++++++++-------- go.sum | 32 ++++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index cbf461b..769fb0c 100644 --- a/go.mod +++ b/go.mod @@ -13,13 +13,13 @@ require ( github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.16.0 go.mau.fi/util v0.0.0-20230805171708-199bf3eec776 - go.mau.fi/whatsmeow v0.0.0-20230824151650-6da2abde6b7c - golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb - golang.org/x/image v0.11.0 - golang.org/x/net v0.14.0 + go.mau.fi/whatsmeow v0.0.0-20230906145423-1fc7b21635ef + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 + golang.org/x/image v0.12.0 + golang.org/x/net v0.15.0 google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.16.1-0.20230904092006-3fffe3f31ce1 + maunium.net/go/mautrix v0.16.1-0.20230905161804-aafd22eee6bb ) require ( @@ -42,9 +42,9 @@ require ( github.com/yuin/goldmark v1.5.5 // indirect go.mau.fi/libsignal v0.1.0 // indirect go.mau.fi/zeroconfig v0.1.2 // indirect - golang.org/x/crypto v0.12.0 // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect + golang.org/x/crypto v0.13.0 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/text v0.13.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect maunium.net/go/mauflag v1.0.0 // indirect diff --git a/go.sum b/go.sum index d682cfc..f1cc736 100644 --- a/go.sum +++ b/go.sum @@ -70,26 +70,26 @@ go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= go.mau.fi/util v0.0.0-20230805171708-199bf3eec776 h1:VrxDCO/gLFHLQywGUsJzertrvt2mUEMrZPf4hEL/s18= go.mau.fi/util v0.0.0-20230805171708-199bf3eec776/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84= -go.mau.fi/whatsmeow v0.0.0-20230824151650-6da2abde6b7c h1:GH1QtL+bbkwyzgJBLyBQcr499oHR1y8svzB92KtG6OY= -go.mau.fi/whatsmeow v0.0.0-20230824151650-6da2abde6b7c/go.mod h1:Iv3G4uv6+HWtqL7XSLRa2dSy077Bnji14IvqUbG+bRo= +go.mau.fi/whatsmeow v0.0.0-20230906145423-1fc7b21635ef h1:cKIiImbLs2v4CznuNT0sSlxqrme/P2W3oKgp/OvSfm0= +go.mau.fi/whatsmeow v0.0.0-20230906145423-1fc7b21635ef/go.mod h1:Iv3G4uv6+HWtqL7XSLRa2dSy077Bnji14IvqUbG+bRo= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= -golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb h1:mIKbk8weKhSeLH2GmUTrvx8CjkyJmnU1wFmg59CUjFA= -golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= -golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo= -golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/image v0.12.0 h1:w13vZbU4o5rKOFFR8y7M+c4A5jXDC0uXTdHYRP8X2DQ= +golang.org/x/image v0.12.0/go.mod h1:Lu90jvHG7GfemOIcldsh9A2hS01ocl6oNO7ype5mEnk= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -102,8 +102,8 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -111,8 +111,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -133,5 +133,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.16.1-0.20230904092006-3fffe3f31ce1 h1:o4SOY1Aim54E7DDCPh+Iyq+gBjnvOwKKBvziXftki9o= -maunium.net/go/mautrix v0.16.1-0.20230904092006-3fffe3f31ce1/go.mod h1:XAjE9pTSGcr6vXaiNgQGiip7tddJ8FQV1a29u2QdBG4= +maunium.net/go/mautrix v0.16.1-0.20230905161804-aafd22eee6bb h1:lC+0S+wX/VfR2bTiv6fleCZ1Mtr4e6jzLN1GLC69M4s= +maunium.net/go/mautrix v0.16.1-0.20230905161804-aafd22eee6bb/go.mod h1:XAjE9pTSGcr6vXaiNgQGiip7tddJ8FQV1a29u2QdBG4= From 1f3367e757e734073cdb068037472634c4f21703 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 19:10:16 +0300 Subject: [PATCH 56/94] Update dependencies again --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 769fb0c..87a090d 100644 --- a/go.mod +++ b/go.mod @@ -12,14 +12,14 @@ require ( github.com/rs/zerolog v1.30.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.16.0 - go.mau.fi/util v0.0.0-20230805171708-199bf3eec776 - go.mau.fi/whatsmeow v0.0.0-20230906145423-1fc7b21635ef + go.mau.fi/util v0.0.0-20230906155759-14bad39a8718 + go.mau.fi/whatsmeow v0.0.0-20230906160713-9dac130ab98c golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/image v0.12.0 golang.org/x/net v0.15.0 google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.16.1-0.20230905161804-aafd22eee6bb + maunium.net/go/mautrix v0.16.1-0.20230906160935-26b2e2e590a0 ) require ( @@ -39,7 +39,7 @@ require ( github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/sjson v1.2.5 // indirect - github.com/yuin/goldmark v1.5.5 // indirect + github.com/yuin/goldmark v1.5.6 // indirect go.mau.fi/libsignal v0.1.0 // indirect go.mau.fi/zeroconfig v0.1.2 // indirect golang.org/x/crypto v0.13.0 // indirect diff --git a/go.sum b/go.sum index f1cc736..4398f70 100644 --- a/go.sum +++ b/go.sum @@ -64,14 +64,14 @@ github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhso github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/goldmark v1.5.5 h1:IJznPe8wOzfIKETmMkd06F8nXkmlhaHqFRM9l1hAGsU= -github.com/yuin/goldmark v1.5.5/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.5.6 h1:COmQAWTCcGetChm3Ig7G/t8AFAN00t+o8Mt4cf7JpwA= +github.com/yuin/goldmark v1.5.6/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= -go.mau.fi/util v0.0.0-20230805171708-199bf3eec776 h1:VrxDCO/gLFHLQywGUsJzertrvt2mUEMrZPf4hEL/s18= -go.mau.fi/util v0.0.0-20230805171708-199bf3eec776/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84= -go.mau.fi/whatsmeow v0.0.0-20230906145423-1fc7b21635ef h1:cKIiImbLs2v4CznuNT0sSlxqrme/P2W3oKgp/OvSfm0= -go.mau.fi/whatsmeow v0.0.0-20230906145423-1fc7b21635ef/go.mod h1:Iv3G4uv6+HWtqL7XSLRa2dSy077Bnji14IvqUbG+bRo= +go.mau.fi/util v0.0.0-20230906155759-14bad39a8718 h1:hmm5bZqE0M8+Uvys0HJPCSbAIZIwYtTkBKYPjAWHuMM= +go.mau.fi/util v0.0.0-20230906155759-14bad39a8718/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84= +go.mau.fi/whatsmeow v0.0.0-20230906160713-9dac130ab98c h1:rMUhugt6iUWjFDIk0RDpxBQIiojLqzOSz0yU9aLy27A= +go.mau.fi/whatsmeow v0.0.0-20230906160713-9dac130ab98c/go.mod h1:liEVGnXRGs6UHDBOGXvJiWbibOnWuRGVFNVhkeXpMl4= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -133,5 +133,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.16.1-0.20230905161804-aafd22eee6bb h1:lC+0S+wX/VfR2bTiv6fleCZ1Mtr4e6jzLN1GLC69M4s= -maunium.net/go/mautrix v0.16.1-0.20230905161804-aafd22eee6bb/go.mod h1:XAjE9pTSGcr6vXaiNgQGiip7tddJ8FQV1a29u2QdBG4= +maunium.net/go/mautrix v0.16.1-0.20230906160935-26b2e2e590a0 h1:nB2hJYjXr2P/gv4w0QlrkQQaI8KY8MhP58WJIwt/WK0= +maunium.net/go/mautrix v0.16.1-0.20230906160935-26b2e2e590a0/go.mod h1:klle6WNVU8V14nPgxMNNztgdl2KKCJj7jS7wUf+ruu0= From 3b19cb0e732c2add624c4128061aa70084d61d6f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 9 Sep 2023 12:54:14 +0300 Subject: [PATCH 57/94] Update changelog [skip ci] --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 274fa62..1d954ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ * Added support for double puppeting with arbitrary `as_token`s. See [docs](https://docs.mau.fi/bridges/general/double-puppeting.html#appservice-method-new) for more info. +* Added retrying for media downloads when WhatsApp servers break and start + returning 429s and 503s. * Fixed logging in with 8-letter code. * Fixed syncing community announcement groups. * Changed "Incoming call" message to explicitly say you have to open WhatsApp From ca4b46257cf68be1450bc9ac4b36ddb3162c9282 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 16 Sep 2023 10:27:45 -0400 Subject: [PATCH 58/94] Bump version to v0.10.1 --- CHANGELOG.md | 6 +++--- go.mod | 6 +++--- go.sum | 12 ++++++------ main.go | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d954ac..5369dac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# unreleased +# v0.10.1 (2023-09-16) * Added support for double puppeting with arbitrary `as_token`s. See [docs](https://docs.mau.fi/bridges/general/double-puppeting.html#appservice-method-new) for more info. @@ -18,8 +18,8 @@ * Added support for logging in by entering a 8-letter code on the phone instead of scanning a QR code. * Note: due to a server-side change, code login may only work when `os_name` - and `browser_name` in the config are set in a specific way. A future release - will automatically change the values to always work with code login. + and `browser_name` in the config are set in a specific way. This is fixed + in v0.10.1. # v0.9.0 (2023-07-16) diff --git a/go.mod b/go.mod index 87a090d..8218c19 100644 --- a/go.mod +++ b/go.mod @@ -12,14 +12,14 @@ require ( github.com/rs/zerolog v1.30.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.16.0 - go.mau.fi/util v0.0.0-20230906155759-14bad39a8718 - go.mau.fi/whatsmeow v0.0.0-20230906160713-9dac130ab98c + go.mau.fi/util v0.1.0 + go.mau.fi/whatsmeow v0.0.0-20230916142552-a743fdc23bf1 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/image v0.12.0 golang.org/x/net v0.15.0 google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.16.1-0.20230906160935-26b2e2e590a0 + maunium.net/go/mautrix v0.16.1 ) require ( diff --git a/go.sum b/go.sum index 4398f70..2b98c5b 100644 --- a/go.sum +++ b/go.sum @@ -68,10 +68,10 @@ github.com/yuin/goldmark v1.5.6 h1:COmQAWTCcGetChm3Ig7G/t8AFAN00t+o8Mt4cf7JpwA= github.com/yuin/goldmark v1.5.6/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= -go.mau.fi/util v0.0.0-20230906155759-14bad39a8718 h1:hmm5bZqE0M8+Uvys0HJPCSbAIZIwYtTkBKYPjAWHuMM= -go.mau.fi/util v0.0.0-20230906155759-14bad39a8718/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84= -go.mau.fi/whatsmeow v0.0.0-20230906160713-9dac130ab98c h1:rMUhugt6iUWjFDIk0RDpxBQIiojLqzOSz0yU9aLy27A= -go.mau.fi/whatsmeow v0.0.0-20230906160713-9dac130ab98c/go.mod h1:liEVGnXRGs6UHDBOGXvJiWbibOnWuRGVFNVhkeXpMl4= +go.mau.fi/util v0.1.0 h1:BwIFWIOEeO7lsiI2eWKFkWTfc5yQmoe+0FYyOFVyaoE= +go.mau.fi/util v0.1.0/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84= +go.mau.fi/whatsmeow v0.0.0-20230916142552-a743fdc23bf1 h1:tfVqib0PAAgMJrZu/Ko25J436e91HKgZepwdhgPmeHM= +go.mau.fi/whatsmeow v0.0.0-20230916142552-a743fdc23bf1/go.mod h1:1xFS2b5zqsg53ApsYB4FDtko7xG7r+gVgBjh9k+9/GE= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -133,5 +133,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.16.1-0.20230906160935-26b2e2e590a0 h1:nB2hJYjXr2P/gv4w0QlrkQQaI8KY8MhP58WJIwt/WK0= -maunium.net/go/mautrix v0.16.1-0.20230906160935-26b2e2e590a0/go.mod h1:klle6WNVU8V14nPgxMNNztgdl2KKCJj7jS7wUf+ruu0= +maunium.net/go/mautrix v0.16.1 h1:Wb3CvOCe8A/NLsFeZYxKrgXKiqeZUQEBD1zqm7n/kWk= +maunium.net/go/mautrix v0.16.1/go.mod h1:2Jf15tulVtr6LxoiRL4smRXwpkGWUNfBFhwh/aXDBuk= diff --git a/main.go b/main.go index abd65df..7342245 100644 --- a/main.go +++ b/main.go @@ -262,7 +262,7 @@ func main() { Name: "mautrix-whatsapp", URL: "https://github.com/mautrix/whatsapp", Description: "A Matrix-WhatsApp puppeting bridge.", - Version: "0.10.0", + Version: "0.10.1", ProtocolName: "WhatsApp", BeeperServiceName: "whatsapp", BeeperNetworkName: "whatsapp", From 6c4411c666b7a11f5cff9c625d2cf2bda3740e80 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 18 Sep 2023 10:00:35 -0400 Subject: [PATCH 59/94] Use native webp for decoding Fixes #650 Fixes GHSA-w665-m639-6c38 --- portal.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/portal.go b/portal.go index 4d9004e..33b5756 100644 --- a/portal.go +++ b/portal.go @@ -41,7 +41,7 @@ import ( "sync" "time" - "github.com/chai2010/webp" + cwebp "github.com/chai2010/webp" "github.com/rs/zerolog" "github.com/tidwall/gjson" "go.mau.fi/util/dbutil" @@ -57,6 +57,7 @@ import ( "go.mau.fi/whatsmeow/types/events" "golang.org/x/exp/slices" "golang.org/x/image/draw" + "golang.org/x/image/webp" "google.golang.org/protobuf/proto" log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix" @@ -3621,7 +3622,7 @@ func (portal *Portal) convertToWebP(img []byte) ([]byte, error) { } var webpBuffer bytes.Buffer - if err = webp.Encode(&webpBuffer, decodedImg, nil); err != nil { + if err = cwebp.Encode(&webpBuffer, decodedImg, nil); err != nil { return img, fmt.Errorf("failed to encode webp image: %w", err) } From 77e30e85dd9bc9134ceb4d99fbc2c64bc9823769 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 19 Sep 2023 12:04:27 -0400 Subject: [PATCH 60/94] Bump version to v0.10.2 --- CHANGELOG.md | 4 ++++ main.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5369dac..8b797bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# v0.10.2 (security update) + +* Stopped using libwebp for decoding webps. + # v0.10.1 (2023-09-16) * Added support for double puppeting with arbitrary `as_token`s. diff --git a/main.go b/main.go index 7342245..cc86364 100644 --- a/main.go +++ b/main.go @@ -262,7 +262,7 @@ func main() { Name: "mautrix-whatsapp", URL: "https://github.com/mautrix/whatsapp", Description: "A Matrix-WhatsApp puppeting bridge.", - Version: "0.10.1", + Version: "0.10.2", ProtocolName: "WhatsApp", BeeperServiceName: "whatsapp", BeeperNetworkName: "whatsapp", From e1edb644c013a815f349d9a283d24e911d011f9c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 15 Sep 2023 15:06:01 -0400 Subject: [PATCH 61/94] Add support for sending Beeper galleries --- messagetracking.go | 3 +++ portal.go | 55 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/messagetracking.go b/messagetracking.go index 7066726..57cb4f8 100644 --- a/messagetracking.go +++ b/messagetracking.go @@ -55,6 +55,9 @@ var ( errPollMissingQuestion = errors.New("poll message is missing question") errPollDuplicateOption = errors.New("poll options must be unique") + errGalleryRelay = errors.New("can't send gallery through relay user") + errGalleryCaption = errors.New("can't send gallery with caption") + errEditUnknownTarget = errors.New("unknown edit target message") errEditUnknownTargetType = errors.New("unsupported edited message type") errEditDifferentSender = errors.New("can't edit message sent by another user") diff --git a/portal.go b/portal.go index 33b5756..ce91a77 100644 --- a/portal.go +++ b/portal.go @@ -4028,6 +4028,8 @@ func (portal *Portal) generateContextInfo(relatesTo *event.RelatesTo) *waProto.C type extraConvertMeta struct { PollOptions map[[32]byte]string EditRootMsg *database.Message + + GalleryExtraParts []*waProto.Message } func getEditError(rootMsg *database.Message, editer *User) error { @@ -4139,6 +4141,37 @@ func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, ev FileSha256: media.FileSHA256, FileLength: proto.Uint64(uint64(media.FileLength)), } + case event.MsgBeeperGallery: + if isRelay { + return nil, sender, extraMeta, errGalleryRelay + } else if content.BeeperGalleryCaption != "" { + return nil, sender, extraMeta, errGalleryCaption + } + for i, part := range content.BeeperGalleryImages { + // TODO support videos + media, err := portal.preprocessMatrixMedia(ctx, sender, false, part, evt.ID, whatsmeow.MediaImage) + if media == nil { + return nil, sender, extraMeta, fmt.Errorf("failed to handle image #%d: %w", i+1, err) + } + imageMsg := &waProto.ImageMessage{ + ContextInfo: ctxInfo, + JpegThumbnail: media.Thumbnail, + Url: &media.URL, + DirectPath: &media.DirectPath, + MediaKey: media.MediaKey, + Mimetype: &part.GetInfo().MimeType, + FileEncSha256: media.FileEncSHA256, + FileSha256: media.FileSHA256, + FileLength: proto.Uint64(uint64(media.FileLength)), + } + if i == 0 { + msg.ImageMessage = imageMsg + } else { + extraMeta.GalleryExtraParts = append(extraMeta.GalleryExtraParts, &waProto.Message{ + ImageMessage: imageMsg, + }) + } + } case event.MessageType(event.EventSticker.Type): media, err := portal.preprocessMatrixMedia(ctx, sender, relaybotFormatted, content, evt.ID, whatsmeow.MediaImage) if media == nil { @@ -4376,10 +4409,26 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event, timing resp, err := sender.Client.SendMessage(ctx, portal.Key.JID, msg, whatsmeow.SendRequestExtra{ID: info.ID}) timings.totalSend = time.Since(start) timings.whatsmeow = resp.DebugTimings - go ms.sendMessageMetrics(evt, err, "Error sending", true) - if err == nil { - dbMsg.MarkSent(resp.Timestamp) + if err != nil { + go ms.sendMessageMetrics(evt, err, "Error sending", true) + return } + dbMsg.MarkSent(resp.Timestamp) + if len(extraMeta.GalleryExtraParts) > 0 { + for i, part := range extraMeta.GalleryExtraParts { + partInfo := portal.generateMessageInfo(sender) + partDBMsg := portal.markHandled(nil, nil, partInfo, evt.ID, evt.Sender, false, true, database.MsgBeeperGallery, i+1, database.MsgNoError) + portal.log.Debugln("Sending gallery part", i+2, "of event", evt.ID, "to WhatsApp", partInfo.ID) + resp, err = sender.Client.SendMessage(ctx, portal.Key.JID, part, whatsmeow.SendRequestExtra{ID: partInfo.ID}) + if err != nil { + go ms.sendMessageMetrics(evt, err, "Error sending", true) + return + } + portal.log.Debugfln("Sent gallery part", i+2, "of event", evt.ID) + partDBMsg.MarkSent(resp.Timestamp) + } + } + go ms.sendMessageMetrics(evt, nil, "", true) } func (portal *Portal) HandleMatrixReaction(sender *User, evt *event.Event) { From d2110f6ee715fc316e945533f7ee494177c6e90d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 29 Sep 2023 14:19:48 +0300 Subject: [PATCH 62/94] Allow sending analytics to custom server --- segment.go => analytics.go | 25 ++++++++++++------------- config/config.go | 7 +++++-- config/upgrade.go | 7 ++++--- example-config.yaml | 12 ++++++++---- main.go | 20 +++++++++++++------- portal.go | 4 ++-- provisioning.go | 16 ++++++++-------- user.go | 4 ++-- 8 files changed, 54 insertions(+), 41 deletions(-) rename segment.go => analytics.go (76%) diff --git a/segment.go b/analytics.go similarity index 76% rename from segment.go rename to analytics.go index 16b6e53..60ebd0a 100644 --- a/segment.go +++ b/analytics.go @@ -26,27 +26,26 @@ import ( "maunium.net/go/mautrix/id" ) -const SegmentURL = "https://api.segment.io/v1/track" - -type SegmentClient struct { +type AnalyticsClient struct { + url string key string userID string log log.Logger client http.Client } -var Segment SegmentClient +var Analytics AnalyticsClient -func (sc *SegmentClient) trackSync(userID id.UserID, event string, properties map[string]interface{}) error { +func (sc *AnalyticsClient) trackSync(userID id.UserID, event string, properties map[string]interface{}) error { var buf bytes.Buffer - var segmentUserID string - if Segment.userID != "" { - segmentUserID = Segment.userID + var analyticsUserID string + if Analytics.userID != "" { + analyticsUserID = Analytics.userID } else { - segmentUserID = userID.String() + analyticsUserID = userID.String() } err := json.NewEncoder(&buf).Encode(map[string]interface{}{ - "userId": segmentUserID, + "userId": analyticsUserID, "event": event, "properties": properties, }) @@ -54,7 +53,7 @@ func (sc *SegmentClient) trackSync(userID id.UserID, event string, properties ma return err } - req, err := http.NewRequest("POST", SegmentURL, &buf) + req, err := http.NewRequest(http.MethodPost, sc.url, &buf) if err != nil { return err } @@ -70,11 +69,11 @@ func (sc *SegmentClient) trackSync(userID id.UserID, event string, properties ma return nil } -func (sc *SegmentClient) IsEnabled() bool { +func (sc *AnalyticsClient) IsEnabled() bool { return len(sc.key) > 0 } -func (sc *SegmentClient) Track(userID id.UserID, event string, properties ...map[string]interface{}) { +func (sc *AnalyticsClient) Track(userID id.UserID, event string, properties ...map[string]interface{}) { if !sc.IsEnabled() { return } else if len(properties) > 1 { diff --git a/config/config.go b/config/config.go index 2aaab9a..69dbd2c 100644 --- a/config/config.go +++ b/config/config.go @@ -24,8 +24,11 @@ import ( type Config struct { *bridgeconfig.BaseConfig `yaml:",inline"` - SegmentKey string `yaml:"segment_key"` - SegmentUserID string `yaml:"segment_user_id"` + Analytics struct { + Host string `yaml:"host"` + Token string `yaml:"token"` + UserID string `yaml:"user_id"` + } Metrics struct { Enabled bool `yaml:"enabled"` diff --git a/config/upgrade.go b/config/upgrade.go index f307651..13d7f79 100644 --- a/config/upgrade.go +++ b/config/upgrade.go @@ -27,8 +27,9 @@ import ( func DoUpgrade(helper *up.Helper) { bridgeconfig.Upgrader.DoUpgrade(helper) - helper.Copy(up.Str|up.Null, "segment_key") - helper.Copy(up.Str|up.Null, "segment_user_id") + helper.Copy(up.Str|up.Null, "analytics", "host") + helper.Copy(up.Str|up.Null, "analytics", "token") + helper.Copy(up.Str|up.Null, "analytics", "user_id") helper.Copy(up.Bool, "metrics", "enabled") helper.Copy(up.Str, "metrics", "listen") @@ -181,7 +182,7 @@ var SpacedBlocks = [][]string{ {"appservice", "database"}, {"appservice", "id"}, {"appservice", "as_token"}, - {"segment_key"}, + {"analytics"}, {"metrics"}, {"whatsapp"}, {"bridge"}, diff --git a/example-config.yaml b/example-config.yaml index 33e5a79..95ed83d 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -76,10 +76,14 @@ appservice: as_token: "This value is generated when generating the registration" hs_token: "This value is generated when generating the registration" -# Segment API key to track some events, like provisioning API login and encryption errors. -segment_key: null -# Optional user_id to use when sending Segment events. If null, defaults to using mxID. -segment_user_id: null +# Segment-compatible analytics endpoint for tracking some events, like provisioning API login and encryption errors. +analytics: + # Hostname of the tracking server. The path is hardcoded to /v1/track + host: api.segment.io + # API key to send with tracking requests. Tracking is disabled if this is null. + token: null + # Optional user ID for tracking events. If null, defaults to using Matrix user ID. + user_id: null # Prometheus config. metrics: diff --git a/main.go b/main.go index cc86364..0c7d7e4 100644 --- a/main.go +++ b/main.go @@ -19,6 +19,7 @@ package main import ( _ "embed" "net/http" + "net/url" "os" "strconv" "strings" @@ -90,13 +91,18 @@ func (br *WABridge) Init() { br.EventProcessor.On(TypeMSC3381PollResponse, br.MatrixHandler.HandleMessage) br.EventProcessor.On(TypeMSC3381V2PollResponse, br.MatrixHandler.HandleMessage) - Segment.log = br.Log.Sub("Segment") - Segment.key = br.Config.SegmentKey - Segment.userID = br.Config.SegmentUserID - if Segment.IsEnabled() { - Segment.log.Infoln("Segment metrics are enabled") - if Segment.userID != "" { - Segment.log.Infoln("Overriding Segment user_id with %v", Segment.userID) + Analytics.log = br.Log.Sub("Analytics") + Analytics.url = (&url.URL{ + Scheme: "https", + Host: br.Config.Analytics.Host, + Path: "/v1/track", + }).String() + Analytics.key = br.Config.Analytics.Token + Analytics.userID = br.Config.Analytics.UserID + if Analytics.IsEnabled() { + Analytics.log.Infoln("Analytics metrics are enabled") + if Analytics.userID != "" { + Analytics.log.Infoln("Overriding analytics user_id with %v", Analytics.userID) } } diff --git a/portal.go b/portal.go index ce91a77..cf1ad94 100644 --- a/portal.go +++ b/portal.go @@ -773,7 +773,7 @@ func (portal *Portal) handleUndecryptableMessage(source *User, evt *events.Undec if evt.IsUnavailable { metricType = "unavailable" } - Segment.Track(source.MXID, "WhatsApp undecryptable message", map[string]interface{}{ + Analytics.Track(source.MXID, "WhatsApp undecryptable message", map[string]interface{}{ "messageID": evt.Info.ID, "undecryptableType": metricType, }) @@ -849,7 +849,7 @@ func (portal *Portal) handleMessage(source *User, evt *events.Message, historica if evt.UnavailableRequestID != "" { resolveType = "phone" } - Segment.Track(source.MXID, "WhatsApp undecryptable message resolved", map[string]interface{}{ + Analytics.Track(source.MXID, "WhatsApp undecryptable message resolved", map[string]interface{}{ "messageID": evt.Info.ID, "resolveType": resolveType, }) diff --git a/provisioning.go b/provisioning.go index c9060a8..3f6bfc4 100644 --- a/provisioning.go +++ b/provisioning.go @@ -692,7 +692,7 @@ func (prov *ProvisioningAPI) Login(w http.ResponseWriter, r *http.Request) { } } user.log.Debugln("Started login via provisioning API") - Segment.Track(user.MXID, "$login_start") + Analytics.Track(user.MXID, "$login_start") for { select { @@ -701,7 +701,7 @@ func (prov *ProvisioningAPI) Login(w http.ResponseWriter, r *http.Request) { case whatsmeow.QRChannelSuccess.Event: jid := user.Client.Store.ID user.log.Debugln("Successful login as", jid, "via provisioning API") - Segment.Track(user.MXID, "$login_success") + Analytics.Track(user.MXID, "$login_success") _ = c.WriteJSON(map[string]interface{}{ "success": true, "jid": jid, @@ -711,7 +711,7 @@ func (prov *ProvisioningAPI) Login(w http.ResponseWriter, r *http.Request) { case whatsmeow.QRChannelTimeout.Event: user.log.Debugln("Login via provisioning API timed out") errCode := "login timed out" - Segment.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) + Analytics.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) _ = c.WriteJSON(Error{ Error: "QR code scan timed out. Please try again.", ErrCode: errCode, @@ -719,7 +719,7 @@ func (prov *ProvisioningAPI) Login(w http.ResponseWriter, r *http.Request) { case whatsmeow.QRChannelErrUnexpectedEvent.Event: user.log.Debugln("Login via provisioning API failed due to unexpected event") errCode := "unexpected event" - Segment.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) + Analytics.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) _ = c.WriteJSON(Error{ Error: "Got unexpected event while waiting for QRs, perhaps you're already logged in?", ErrCode: errCode, @@ -727,14 +727,14 @@ func (prov *ProvisioningAPI) Login(w http.ResponseWriter, r *http.Request) { case whatsmeow.QRChannelClientOutdated.Event: user.log.Debugln("Login via provisioning API failed due to outdated client") errCode := "bridge outdated" - Segment.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) + Analytics.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) _ = c.WriteJSON(Error{ Error: "Got client outdated error while waiting for QRs. The bridge must be updated to continue.", ErrCode: errCode, }) case whatsmeow.QRChannelScannedWithoutMultidevice.Event: errCode := "multidevice not enabled" - Segment.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) + Analytics.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) _ = c.WriteJSON(Error{ Error: "Please enable the WhatsApp multidevice beta and scan the QR code again.", ErrCode: errCode, @@ -742,13 +742,13 @@ func (prov *ProvisioningAPI) Login(w http.ResponseWriter, r *http.Request) { continue case "error": errCode := "fatal error" - Segment.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) + Analytics.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) _ = c.WriteJSON(Error{ Error: "Fatal error while logging in", ErrCode: errCode, }) case "code": - Segment.Track(user.MXID, "$qrcode_retrieved") + Analytics.Track(user.MXID, "$qrcode_retrieved") _ = c.WriteJSON(map[string]interface{}{ "code": evt.Code, "timeout": int(evt.Timeout.Seconds()), diff --git a/user.go b/user.go index 9adfde9..794d100 100644 --- a/user.go +++ b/user.go @@ -498,7 +498,7 @@ func (user *User) createClient(sess *store.Device) { user.Client.SetForceActiveDeliveryReceipts(user.bridge.Config.Bridge.ForceActiveDeliveryReceipts) user.Client.AutomaticMessageRerequestFromPhone = true user.Client.GetMessageForRetry = func(requester, to types.JID, id types.MessageID) *waProto.Message { - Segment.Track(user.MXID, "WhatsApp incoming retry (message not found)", map[string]interface{}{ + Analytics.Track(user.MXID, "WhatsApp incoming retry (message not found)", map[string]interface{}{ "requester": user.obfuscateJID(requester), "messageID": id, }) @@ -506,7 +506,7 @@ func (user *User) createClient(sess *store.Device) { return nil } user.Client.PreRetryCallback = func(receipt *events.Receipt, messageID types.MessageID, retryCount int, msg *waProto.Message) bool { - Segment.Track(user.MXID, "WhatsApp incoming retry (accepted)", map[string]interface{}{ + Analytics.Track(user.MXID, "WhatsApp incoming retry (accepted)", map[string]interface{}{ "requester": user.obfuscateJID(receipt.Sender), "messageID": messageID, "retryCount": retryCount, From 9cb0a2e43d9080f086304084d04c5d6aa7521933 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 9 Oct 2023 12:10:34 +0300 Subject: [PATCH 63/94] Add default mime type if it's missing --- portal.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/portal.go b/portal.go index cf1ad94..e513e81 100644 --- a/portal.go +++ b/portal.go @@ -3665,6 +3665,9 @@ func (portal *Portal) preprocessMatrixMedia(ctx context.Context, sender *User, r } } mimeType := content.GetInfo().MimeType + if mimeType == "" { + content.Info.MimeType = "application/octet-stream" + } var convertErr error // Allowed mime types from https://developers.facebook.com/docs/whatsapp/on-premises/reference/media switch { From 246587b616209d4587e6118e026e18d23b6492fe Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 9 Oct 2023 12:41:08 +0300 Subject: [PATCH 64/94] Switch to fork of chai2010/webp --- go.mod | 2 +- go.sum | 4 ++-- portal.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 8218c19..b31d27c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module maunium.net/go/mautrix-whatsapp go 1.20 require ( - github.com/chai2010/webp v1.1.1 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 github.com/lib/pq v1.10.9 @@ -13,6 +12,7 @@ require ( github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.16.0 go.mau.fi/util v0.1.0 + go.mau.fi/webp v0.1.0 go.mau.fi/whatsmeow v0.0.0-20230916142552-a743fdc23bf1 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/image v0.12.0 diff --git a/go.sum b/go.sum index 2b98c5b..f70de01 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chai2010/webp v1.1.1 h1:jTRmEccAJ4MGrhFOrPMpNGIJ/eybIgwKpcACsrTEapk= -github.com/chai2010/webp v1.1.1/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -70,6 +68,8 @@ go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= go.mau.fi/util v0.1.0 h1:BwIFWIOEeO7lsiI2eWKFkWTfc5yQmoe+0FYyOFVyaoE= go.mau.fi/util v0.1.0/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84= +go.mau.fi/webp v0.1.0 h1:BHObH/DcFntT9KYun5pDr0Ot4eUZO8k2C7eP7vF4ueA= +go.mau.fi/webp v0.1.0/go.mod h1:e42Z+VMFrUMS9cpEwGRIor+lQWO8oUAyPyMtcL+NMt8= go.mau.fi/whatsmeow v0.0.0-20230916142552-a743fdc23bf1 h1:tfVqib0PAAgMJrZu/Ko25J436e91HKgZepwdhgPmeHM= go.mau.fi/whatsmeow v0.0.0-20230916142552-a743fdc23bf1/go.mod h1:1xFS2b5zqsg53ApsYB4FDtko7xG7r+gVgBjh9k+9/GE= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= diff --git a/portal.go b/portal.go index e513e81..f3b8b2d 100644 --- a/portal.go +++ b/portal.go @@ -41,7 +41,6 @@ import ( "sync" "time" - cwebp "github.com/chai2010/webp" "github.com/rs/zerolog" "github.com/tidwall/gjson" "go.mau.fi/util/dbutil" @@ -51,6 +50,7 @@ import ( "go.mau.fi/util/jsontime" "go.mau.fi/util/random" "go.mau.fi/util/variationselector" + cwebp "go.mau.fi/webp" "go.mau.fi/whatsmeow" waProto "go.mau.fi/whatsmeow/binary/proto" "go.mau.fi/whatsmeow/types" From 6feebd827b49551e370a17092d5563953e051c58 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 13 Oct 2023 18:10:22 +0300 Subject: [PATCH 65/94] Add basic newsletter support. Fixes #655 --- commands.go | 4 +- database/portal.go | 2 +- go.mod | 4 +- go.sum | 8 +-- historysync.go | 4 +- portal.go | 154 ++++++++++++++++++++++++++++++++++++++++----- provisioning.go | 4 +- user.go | 35 +++++++++-- 8 files changed, 182 insertions(+), 33 deletions(-) diff --git a/commands.go b/commands.go index 5aa0d5c..f69e037 100644 --- a/commands.go +++ b/commands.go @@ -1011,10 +1011,10 @@ func fnOpen(ce *WrappedCommandEvent) { ce.Log.Debugln("Importing", jid, "for", ce.User.MXID) portal := ce.User.GetPortalByJID(info.JID) if len(portal.MXID) > 0 { - portal.UpdateMatrixRoom(ce.User, info) + portal.UpdateMatrixRoom(ce.User, info, nil) ce.Reply("Portal room synced.") } else { - err = portal.CreateMatrixRoom(ce.User, info, true, true) + err = portal.CreateMatrixRoom(ce.User, info, nil, true, true) if err != nil { ce.Reply("Failed to create room: %v", err) } else { diff --git a/database/portal.go b/database/portal.go index b9d4a66..1b3eb00 100644 --- a/database/portal.go +++ b/database/portal.go @@ -34,7 +34,7 @@ type PortalKey struct { } func NewPortalKey(jid, receiver types.JID) PortalKey { - if jid.Server == types.GroupServer { + if jid.Server == types.GroupServer || jid.Server == types.NewsletterServer { receiver = jid } else if jid.Server == types.LegacyUserServer { jid.Server = types.DefaultUserServer diff --git a/go.mod b/go.mod index b31d27c..b0fa94b 100644 --- a/go.mod +++ b/go.mod @@ -11,9 +11,9 @@ require ( github.com/rs/zerolog v1.30.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.16.0 - go.mau.fi/util v0.1.0 + go.mau.fi/util v0.1.1-0.20231013112707-e938021823cc go.mau.fi/webp v0.1.0 - go.mau.fi/whatsmeow v0.0.0-20230916142552-a743fdc23bf1 + go.mau.fi/whatsmeow v0.0.0-20231013150720-028a685d137c golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/image v0.12.0 golang.org/x/net v0.15.0 diff --git a/go.sum b/go.sum index f70de01..cb2a829 100644 --- a/go.sum +++ b/go.sum @@ -66,12 +66,12 @@ github.com/yuin/goldmark v1.5.6 h1:COmQAWTCcGetChm3Ig7G/t8AFAN00t+o8Mt4cf7JpwA= github.com/yuin/goldmark v1.5.6/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= -go.mau.fi/util v0.1.0 h1:BwIFWIOEeO7lsiI2eWKFkWTfc5yQmoe+0FYyOFVyaoE= -go.mau.fi/util v0.1.0/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84= +go.mau.fi/util v0.1.1-0.20231013112707-e938021823cc h1:/ZY5g+McWqVSA6fK8ROBOyJFb5hCBBQKMcm2oRFzc9c= +go.mau.fi/util v0.1.1-0.20231013112707-e938021823cc/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84= go.mau.fi/webp v0.1.0 h1:BHObH/DcFntT9KYun5pDr0Ot4eUZO8k2C7eP7vF4ueA= go.mau.fi/webp v0.1.0/go.mod h1:e42Z+VMFrUMS9cpEwGRIor+lQWO8oUAyPyMtcL+NMt8= -go.mau.fi/whatsmeow v0.0.0-20230916142552-a743fdc23bf1 h1:tfVqib0PAAgMJrZu/Ko25J436e91HKgZepwdhgPmeHM= -go.mau.fi/whatsmeow v0.0.0-20230916142552-a743fdc23bf1/go.mod h1:1xFS2b5zqsg53ApsYB4FDtko7xG7r+gVgBjh9k+9/GE= +go.mau.fi/whatsmeow v0.0.0-20231013150720-028a685d137c h1:fpNierRxUnaQfYUH0v45bBq8JK6rAt2EkTZtlIYdBDs= +go.mau.fi/whatsmeow v0.0.0-20231013150720-028a685d137c/go.mod h1:rczIT6OzqI4FQIujJe8X/rfxi0pQVFpjQIN2+9vD3Gg= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/historysync.go b/historysync.go index 9e7ef75..578629d 100644 --- a/historysync.go +++ b/historysync.go @@ -154,7 +154,7 @@ func (user *User) backfillAll() { Msg("Chat already has a room, deleting messages from database") user.bridge.DB.HistorySync.DeleteConversation(user.MXID, portal.Key.JID.String()) } else if limit < 0 || i < limit { - err = portal.CreateMatrixRoom(user, nil, true, true) + err = portal.CreateMatrixRoom(user, nil, nil, true, true) if err != nil { user.zlog.Err(err).Msg("Failed to create Matrix room for backfill") } @@ -316,7 +316,7 @@ func (user *User) backfillInChunks(req *database.Backfill, conv *database.Histor if len(portal.MXID) == 0 { user.log.Debugln("Creating portal for", portal.Key.JID, "as part of history sync handling") - err := portal.CreateMatrixRoom(user, nil, true, false) + err := portal.CreateMatrixRoom(user, nil, nil, true, false) if err != nil { user.log.Errorfln("Failed to create room for %s during backfill: %v", portal.Key.JID, err) return diff --git a/portal.go b/portal.go index f3b8b2d..69bde24 100644 --- a/portal.go +++ b/portal.go @@ -344,7 +344,7 @@ func (portal *Portal) handleWhatsAppMessageLoopItem(msg PortalMessage) { return } portal.log.Debugln("Creating Matrix room from incoming message") - err := portal.CreateMatrixRoom(msg.source, nil, false, true) + err := portal.CreateMatrixRoom(msg.source, nil, nil, false, true) if err != nil { portal.log.Errorln("Failed to create portal room:", err) return @@ -1077,6 +1077,9 @@ func (portal *Portal) getMessagePuppet(user *User, info *types.MessageInfo) (pup } func (portal *Portal) getMessageIntent(user *User, info *types.MessageInfo, msgType string) *appservice.IntentAPI { + if portal.IsNewsletter() && info.Sender == info.Chat { + return portal.MainIntent() + } puppet := portal.getMessagePuppet(user, info) if puppet == nil { return nil @@ -1161,6 +1164,9 @@ func (portal *Portal) syncParticipant(source *User, participant types.GroupParti } func (portal *Portal) SyncParticipants(source *User, metadata *types.GroupInfo) ([]id.UserID, *event.PowerLevelsEventContent) { + if portal.IsNewsletter() { + return nil, nil + } changed := false var levels *event.PowerLevelsEventContent var err error @@ -1239,6 +1245,18 @@ func reuploadAvatar(intent *appservice.IntentAPI, url string) (id.ContentURI, er return resp.ContentURI, nil } +func (user *User) reuploadAvatarDirectPath(intent *appservice.IntentAPI, directPath string) (id.ContentURI, error) { + data, err := user.Client.DownloadMediaWithPath(directPath, nil, nil, nil, 0, "", "") + if err != nil { + return id.ContentURI{}, fmt.Errorf("failed to download avatar: %w", err) + } + resp, err := intent.UploadBytes(data, http.DetectContentType(data)) + if err != nil { + return id.ContentURI{}, fmt.Errorf("failed to upload avatar to Matrix: %w", err) + } + return resp.ContentURI, nil +} + func (user *User) updateAvatar(jid types.JID, isCommunity bool, avatarID *string, avatarURL *id.ContentURI, avatarSet *bool, log log.Logger, intent *appservice.IntentAPI) bool { currentID := "" if *avatarSet && *avatarID != "remove" && *avatarID != "unauthorized" { @@ -1273,14 +1291,23 @@ func (user *User) updateAvatar(jid types.JID, isCommunity bool, avatarID *string } if avatar.ID == *avatarID && *avatarSet { return false - } else if len(avatar.URL) == 0 { + } else if len(avatar.URL) == 0 && len(avatar.DirectPath) == 0 { log.Warnln("Didn't get URL in response to avatar query") return false } else if avatar.ID != *avatarID || avatarURL.IsEmpty() { - url, err := reuploadAvatar(intent, avatar.URL) - if err != nil { - log.Warnln("Failed to reupload avatar:", err) - return false + var url id.ContentURI + if len(avatar.URL) > 0 { + url, err = reuploadAvatar(intent, avatar.URL) + if err != nil { + log.Warnln("Failed to reupload avatar:", err) + return false + } + } else { + url, err = user.reuploadAvatarDirectPath(intent, avatar.DirectPath) + if err != nil { + log.Warnln("Failed to reupload avatar:", err) + return false + } } *avatarURL = url } @@ -1290,10 +1317,61 @@ func (user *User) updateAvatar(jid types.JID, isCommunity bool, avatarID *string return true } +func (portal *Portal) UpdateNewsletterAvatar(user *User, meta *types.NewsletterMetadata) bool { + portal.avatarLock.Lock() + defer portal.avatarLock.Unlock() + var picID string + picture := meta.ThreadMeta.Picture + if picture == nil { + picID = meta.ThreadMeta.Preview.ID + } else { + picID = picture.ID + } + if picID == "" { + picID = "remove" + } + if portal.Avatar != picID || !portal.AvatarSet { + if picID == "remove" { + portal.AvatarURL = id.ContentURI{} + } else if portal.Avatar != picID || portal.AvatarURL.IsEmpty() { + var err error + if picture == nil { + meta, err = user.Client.GetNewsletterInfo(portal.Key.JID) + if err != nil { + portal.log.Warnln("Failed to fetch full res avatar info for newsletter:", err) + return false + } + picture = meta.ThreadMeta.Picture + if picture == nil { + portal.log.Warnln("Didn't get full res avatar info for newsletter") + return false + } + picID = picture.ID + } + portal.AvatarURL, err = user.reuploadAvatarDirectPath(portal.MainIntent(), picture.DirectPath) + if err != nil { + portal.log.Warnln("Failed to reupload newsletter avatar:", err) + return false + } + } + portal.Avatar = picID + portal.AvatarSet = false + return portal.setRoomAvatar(true, types.EmptyJID, true) + } + return false +} + func (portal *Portal) UpdateAvatar(user *User, setBy types.JID, updateInfo bool) bool { + if portal.IsNewsletter() { + return false + } portal.avatarLock.Lock() defer portal.avatarLock.Unlock() changed := user.updateAvatar(portal.Key.JID, portal.IsParent, &portal.Avatar, &portal.AvatarURL, &portal.AvatarSet, portal.log, portal.MainIntent()) + return portal.setRoomAvatar(changed, setBy, updateInfo) +} + +func (portal *Portal) setRoomAvatar(changed bool, setBy types.JID, updateInfo bool) bool { if !changed || portal.Avatar == "unauthorized" { if changed || updateInfo { portal.Update(nil) @@ -1391,6 +1469,21 @@ func (portal *Portal) UpdateTopic(topic string, setBy types.JID, updateInfo bool return false } +func newsletterToGroupInfo(meta *types.NewsletterMetadata) *types.GroupInfo { + var out types.GroupInfo + out.JID = meta.ID + out.Name = meta.ThreadMeta.Name.Text + out.NameSetAt = meta.ThreadMeta.Name.UpdateTime.Time + out.Topic = meta.ThreadMeta.Description.Text + out.TopicSetAt = meta.ThreadMeta.Description.UpdateTime.Time + out.TopicID = meta.ThreadMeta.Description.ID + out.GroupCreated = meta.ThreadMeta.CreationTime.Time + out.IsAnnounce = true + out.IsLocked = true + out.IsIncognito = true + return &out +} + func (portal *Portal) UpdateParentGroup(source *User, parent types.JID, updateInfo bool) bool { portal.parentGroupUpdateLock.Lock() defer portal.parentGroupUpdateLock.Unlock() @@ -1435,6 +1528,14 @@ func (portal *Portal) UpdateMetadata(user *User, groupInfo *types.GroupInfo) boo //update = portal.UpdateTopic(BroadcastTopic, "", nil, false) || update return update } + if groupInfo == nil && portal.IsNewsletter() { + newsletterInfo, err := user.Client.GetNewsletterInfo(portal.Key.JID) + if err != nil { + portal.zlog.Err(err).Msg("Failed to get newsletter info") + return false + } + groupInfo = newsletterToGroupInfo(newsletterInfo) + } if groupInfo == nil { var err error groupInfo, err = user.Client.GetGroupInfo(portal.Key.JID) @@ -1471,7 +1572,7 @@ func (portal *Portal) ensureUserInvited(user *User) bool { return user.ensureInvited(portal.MainIntent(), portal.MXID, portal.IsPrivateChat()) } -func (portal *Portal) UpdateMatrixRoom(user *User, groupInfo *types.GroupInfo) bool { +func (portal *Portal) UpdateMatrixRoom(user *User, groupInfo *types.GroupInfo, newsletterMetadata *types.NewsletterMetadata) bool { if len(portal.MXID) == 0 { return false } @@ -1480,10 +1581,16 @@ func (portal *Portal) UpdateMatrixRoom(user *User, groupInfo *types.GroupInfo) b portal.ensureUserInvited(user) go portal.addToPersonalSpace(user) + if groupInfo == nil && newsletterMetadata != nil { + groupInfo = newsletterToGroupInfo(newsletterMetadata) + } + update := false update = portal.UpdateMetadata(user, groupInfo) || update - if !portal.IsPrivateChat() && !portal.IsBroadcastList() { + if !portal.IsPrivateChat() && !portal.IsBroadcastList() && !portal.IsNewsletter() { update = portal.UpdateAvatar(user, types.EmptyJID, false) || update + } else if newsletterMetadata != nil { + update = portal.UpdateNewsletterAvatar(user, newsletterMetadata) || update } if update || portal.LastSync.Add(24*time.Hour).Before(time.Now()) { portal.LastSync = time.Now() @@ -1691,7 +1798,7 @@ func (portal *Portal) GetEncryptionEventContent() (evt *event.EncryptionEventCon return } -func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, isFullInfo, backfill bool) error { +func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, newsletterMetadata *types.NewsletterMetadata, isFullInfo, backfill bool) error { portal.roomCreateLock.Lock() defer portal.roomCreateLock.Unlock() if len(portal.MXID) > 0 { @@ -1738,7 +1845,18 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, i portal.log.Debugln("Broadcast list is not yet supported, not creating room after all") return fmt.Errorf("broadcast list bridging is currently not supported") } else { - if groupInfo == nil || !isFullInfo { + if portal.IsNewsletter() { + if newsletterMetadata == nil { + var err error + newsletterMetadata, err = user.Client.GetNewsletterInfo(portal.Key.JID) + if err != nil { + return err + } + } + if groupInfo == nil { + groupInfo = newsletterToGroupInfo(newsletterMetadata) + } + } else if groupInfo == nil || !isFullInfo { foundInfo, err := user.Client.GetGroupInfo(portal.Key.JID) // Ensure that the user is actually a participant in the conversation @@ -1764,7 +1882,11 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, i portal.ExpirationTime = groupInfo.DisappearingTimer } } - portal.UpdateAvatar(user, types.EmptyJID, false) + if portal.IsNewsletter() { + portal.UpdateNewsletterAvatar(user, newsletterMetadata) + } else { + portal.UpdateAvatar(user, types.EmptyJID, false) + } } powerLevels := portal.GetBasePowerLevels() @@ -1843,7 +1965,7 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, i autoJoinInvites := portal.bridge.SpecVersions.Supports(mautrix.BeeperFeatureAutojoinInvites) if autoJoinInvites { portal.log.Debugfln("Hungryserv mode: adding all group members in create request") - if groupInfo != nil { + if groupInfo != nil && !portal.IsNewsletter() { // TODO non-hungryserv could also include all members in invites, and then send joins manually? participants, powerLevels := portal.SyncParticipants(user, groupInfo) invite = append(invite, participants...) @@ -1911,7 +2033,7 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, i go portal.updateCommunitySpace(user, true, true) go portal.addToPersonalSpace(user) - if groupInfo != nil && !autoJoinInvites { + if !portal.IsNewsletter() && groupInfo != nil && !autoJoinInvites { portal.SyncParticipants(user, groupInfo) } //if broadcastMetadata != nil { @@ -1989,7 +2111,7 @@ func (portal *Portal) updateCommunitySpace(user *User, add, updateInfo bool) boo return false } portal.log.Debugfln("Creating portal for parent group %v", space.Key.JID) - err := space.CreateMatrixRoom(user, nil, false, false) + err := space.CreateMatrixRoom(user, nil, nil, false, false) if err != nil { portal.log.Debugfln("Failed to create portal for parent group: %v", err) return false @@ -2039,6 +2161,10 @@ func (portal *Portal) IsBroadcastList() bool { return portal.Key.JID.Server == types.BroadcastServer } +func (portal *Portal) IsNewsletter() bool { + return portal.Key.JID.Server == types.NewsletterServer +} + func (portal *Portal) IsStatusBroadcastList() bool { return portal.Key.JID == types.StatusBroadcastJID } diff --git a/provisioning.go b/provisioning.go index 3f6bfc4..65c8a2b 100644 --- a/provisioning.go +++ b/provisioning.go @@ -451,7 +451,7 @@ func (prov *ProvisioningAPI) OpenGroup(w http.ResponseWriter, r *http.Request) { portal := user.GetPortalByJID(info.JID) status := http.StatusOK if len(portal.MXID) == 0 { - err = portal.CreateMatrixRoom(user, info, true, true) + err = portal.CreateMatrixRoom(user, info, nil, true, true) if err != nil { jsonResponse(w, http.StatusInternalServerError, Error{ Error: fmt.Sprintf("Failed to create portal: %v", err), @@ -532,7 +532,7 @@ func (prov *ProvisioningAPI) JoinGroup(w http.ResponseWriter, r *http.Request) { status := http.StatusOK if len(portal.MXID) == 0 { time.Sleep(500 * time.Millisecond) // Wait for incoming group info to create the portal automatically - err = portal.CreateMatrixRoom(user, info, true, true) + err = portal.CreateMatrixRoom(user, info, nil, true, true) if err != nil { jsonResponse(w, http.StatusInternalServerError, Error{ Error: fmt.Sprintf("Failed to create portal: %v", err), diff --git a/user.go b/user.go index 794d100..5926fe7 100644 --- a/user.go +++ b/user.go @@ -327,7 +327,7 @@ func (user *User) doPuppetResync() { user.log.Warnfln("Failed to get group info for %s to do background sync: %v", portal.Key.JID, err) } else { user.log.Debugfln("Doing background sync for %s", portal.Key.JID) - portal.UpdateMatrixRoom(user, groupInfo) + portal.UpdateMatrixRoom(user, groupInfo, nil) } } if len(puppetJIDs) == 0 { @@ -850,6 +850,10 @@ func (user *User) HandleEvent(event interface{}) { case *events.JoinedGroup: user.groupListCache = nil go user.handleGroupCreate(v) + case *events.NewsletterJoin: + go user.handleNewsletterJoin(v) + case *events.NewsletterLeave: + go user.handleNewsletterLeave(v) case *events.Picture: go user.handlePictureUpdate(v) case *events.Receipt: @@ -1184,13 +1188,13 @@ func (user *User) ResyncGroups(createPortals bool) error { portal := user.GetPortalByJID(group.JID) if len(portal.MXID) == 0 { if createPortals { - err = portal.CreateMatrixRoom(user, group, true, true) + err = portal.CreateMatrixRoom(user, group, nil, true, true) if err != nil { return fmt.Errorf("failed to create room for %s: %w", group.JID, err) } } } else { - portal.UpdateMatrixRoom(user, group) + portal.UpdateMatrixRoom(user, group, nil) } } return nil @@ -1301,12 +1305,12 @@ func (user *User) handleGroupCreate(evt *events.JoinedGroup) { user.log.Debugfln("Ignoring group create event with key %s", evt.CreateKey) return } - err := portal.CreateMatrixRoom(user, &evt.GroupInfo, true, true) + err := portal.CreateMatrixRoom(user, &evt.GroupInfo, nil, true, true) if err != nil { user.log.Errorln("Failed to create Matrix room after join notification: %v", err) } } else { - portal.UpdateMatrixRoom(user, &evt.GroupInfo) + portal.UpdateMatrixRoom(user, &evt.GroupInfo, nil) } } @@ -1376,6 +1380,25 @@ func (user *User) handleGroupUpdate(evt *events.GroupInfo) { } } +func (user *User) handleNewsletterJoin(evt *events.NewsletterJoin) { + portal := user.GetPortalByJID(evt.ID) + if portal.MXID == "" { + err := portal.CreateMatrixRoom(user, nil, &evt.NewsletterMetadata, true, false) + if err != nil { + user.zlog.Err(err).Msg("Failed to create room on newsletter join event") + } + } else { + portal.UpdateMatrixRoom(user, nil, &evt.NewsletterMetadata) + } +} + +func (user *User) handleNewsletterLeave(evt *events.NewsletterLeave) { + portal := user.GetPortalByJID(evt.ID) + if portal.MXID != "" { + portal.HandleWhatsAppKick(user, user.JID, []types.JID{user.JID}) + } +} + func (user *User) handlePictureUpdate(evt *events.Picture) { if evt.JID.Server == types.DefaultUserServer { puppet := user.bridge.GetPuppetByJID(evt.JID) @@ -1405,7 +1428,7 @@ func (user *User) StartPM(jid types.JID, reason string) (*Portal, *Puppet, bool, return portal, puppet, false, nil } } - err := portal.CreateMatrixRoom(user, nil, false, true) + err := portal.CreateMatrixRoom(user, nil, nil, false, true) return portal, puppet, true, err } From d80f2fb2ce970258fb698d72da772976fad9f300 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 13 Oct 2023 18:20:31 +0300 Subject: [PATCH 66/94] Update changelog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b797bf..8bc2951 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# v0.10.3 (unreleased) + +* Added basic support for channels. +* Added default mime type for outgoing attachments when the origin Matrix + client forgets to specify the mime type. +* Updated libwebp version used for encoding. + # v0.10.2 (security update) * Stopped using libwebp for decoding webps. From 7e25e2c2da8dbcb1223ef812e2b95e3971c01fe0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 16 Oct 2023 13:07:06 +0300 Subject: [PATCH 67/94] Fix legacy backfill not checking if conversations have messages Fixes #631 --- CHANGELOG.md | 1 + database/historysync.go | 13 +++++++++++++ historysync.go | 19 ++++++++++--------- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bc2951..de5616f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ * Added basic support for channels. * Added default mime type for outgoing attachments when the origin Matrix client forgets to specify the mime type. +* Fixed legacy backfill creating portals for chats without messages. * Updated libwebp version used for encoding. # v0.10.2 (security update) diff --git a/database/historysync.go b/database/historysync.go index 2f88139..d896bbc 100644 --- a/database/historysync.go +++ b/database/historysync.go @@ -322,6 +322,19 @@ func (hsq *HistorySyncQuery) DeleteAllMessagesForPortal(userID id.UserID, portal } } +func (hsq *HistorySyncQuery) ConversationHasMessages(userID id.UserID, portalKey PortalKey) (exists bool) { + err := hsq.db.QueryRow(` + SELECT EXISTS( + SELECT 1 FROM history_sync_message + WHERE user_mxid=$1 AND conversation_id=$2 + ) + `, userID, portalKey.JID).Scan(&exists) + if err != nil { + hsq.log.Warnfln("Failed to check if any messages are stored for %s/%s: %v", userID, portalKey.JID, err) + } + return +} + func (hsq *HistorySyncQuery) DeleteConversation(userID id.UserID, jid string) { // This will also clear history_sync_message as there's a foreign key constraint _, err := hsq.db.Exec(` diff --git a/historysync.go b/historysync.go index 578629d..bd16077 100644 --- a/historysync.go +++ b/historysync.go @@ -20,6 +20,7 @@ import ( "crypto/sha256" "encoding/base64" "fmt" + "strings" "time" "github.com/rs/zerolog" @@ -138,8 +139,9 @@ func (user *User) backfillAll() { Int("conversation_count", len(conversations)). Msg("Probably received all history sync blobs, now backfilling conversations") limit := user.bridge.Config.Bridge.HistorySync.MaxInitialConversations + bridgedCount := 0 // Find the portals for all the conversations. - for i, conv := range conversations { + for _, conv := range conversations { jid, err := types.ParseJID(conv.ConversationID) if err != nil { user.zlog.Warn().Err(err). @@ -153,7 +155,12 @@ func (user *User) backfillAll() { Str("portal_jid", portal.Key.JID.String()). Msg("Chat already has a room, deleting messages from database") user.bridge.DB.HistorySync.DeleteConversation(user.MXID, portal.Key.JID.String()) - } else if limit < 0 || i < limit { + bridgedCount++ + } else if !user.bridge.DB.HistorySync.ConversationHasMessages(user.MXID, portal.Key) { + user.zlog.Debug().Str("portal_jid", portal.Key.JID.String()).Msg("Skipping chat with no messages in history sync") + user.bridge.DB.HistorySync.DeleteConversation(user.MXID, portal.Key.JID.String()) + } else if limit < 0 || bridgedCount < limit { + bridgedCount++ err = portal.CreateMatrixRoom(user, nil, nil, true, true) if err != nil { user.zlog.Err(err).Msg("Failed to create Matrix room for backfill") @@ -491,13 +498,7 @@ func (user *User) storeHistorySync(evt *waProto.HistorySync) { } msgType := getMessageType(msgEvt.Message) - if msgType == "unknown" || msgType == "ignore" || msgType == "unknown_protocol" { - unsupportedTypes++ - continue - } - - // Don't store unsupported messages. - if !containsSupportedMessage(msgEvt.Message) { + if msgType == "unknown" || msgType == "ignore" || strings.HasPrefix(msgType, "unknown_protocol_") || !containsSupportedMessage(msgEvt.Message) { unsupportedTypes++ continue } From 56ff4e1595db2a7fca9c76d7d4eae8349be0f418 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 16 Oct 2023 13:22:56 +0300 Subject: [PATCH 68/94] Update dependencies --- go.mod | 31 +++++++++---------- go.sum | 96 ++++++++++++++++++++-------------------------------------- 2 files changed, 48 insertions(+), 79 deletions(-) diff --git a/go.mod b/go.mod index b0fa94b..fbf2857 100644 --- a/go.mod +++ b/go.mod @@ -7,16 +7,16 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.17 - github.com/prometheus/client_golang v1.16.0 - github.com/rs/zerolog v1.30.0 + github.com/prometheus/client_golang v1.17.0 + github.com/rs/zerolog v1.31.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e - github.com/tidwall/gjson v1.16.0 - go.mau.fi/util v0.1.1-0.20231013112707-e938021823cc + github.com/tidwall/gjson v1.17.0 + go.mau.fi/util v0.2.0 go.mau.fi/webp v0.1.0 - go.mau.fi/whatsmeow v0.0.0-20231013150720-028a685d137c - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 - golang.org/x/image v0.12.0 - golang.org/x/net v0.15.0 + go.mau.fi/whatsmeow v0.0.0-20231016102050-3a30a129b626 + golang.org/x/exp v0.0.0-20231006140011-7918f672742d + golang.org/x/image v0.13.0 + golang.org/x/net v0.17.0 google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 maunium.net/go/mautrix v0.16.1 @@ -29,21 +29,20 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/kr/text v0.2.0 // indirect - github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.11.1 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/sjson v1.2.5 // indirect github.com/yuin/goldmark v1.5.6 // indirect go.mau.fi/libsignal v0.1.0 // indirect go.mau.fi/zeroconfig v0.1.2 // indirect - golang.org/x/crypto v0.13.0 // indirect - golang.org/x/sys v0.12.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index cb2a829..83712d4 100644 --- a/go.sum +++ b/go.sum @@ -11,7 +11,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= @@ -26,98 +25,69 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= -github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= -github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg= -github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= +github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.5.6 h1:COmQAWTCcGetChm3Ig7G/t8AFAN00t+o8Mt4cf7JpwA= github.com/yuin/goldmark v1.5.6/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= -go.mau.fi/util v0.1.1-0.20231013112707-e938021823cc h1:/ZY5g+McWqVSA6fK8ROBOyJFb5hCBBQKMcm2oRFzc9c= -go.mau.fi/util v0.1.1-0.20231013112707-e938021823cc/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84= +go.mau.fi/util v0.2.0 h1:AMGBEdg9Ya/smb/09dljo9wBwKr432EpfjDWF7aFQg0= +go.mau.fi/util v0.2.0/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84= go.mau.fi/webp v0.1.0 h1:BHObH/DcFntT9KYun5pDr0Ot4eUZO8k2C7eP7vF4ueA= go.mau.fi/webp v0.1.0/go.mod h1:e42Z+VMFrUMS9cpEwGRIor+lQWO8oUAyPyMtcL+NMt8= -go.mau.fi/whatsmeow v0.0.0-20231013150720-028a685d137c h1:fpNierRxUnaQfYUH0v45bBq8JK6rAt2EkTZtlIYdBDs= -go.mau.fi/whatsmeow v0.0.0-20231013150720-028a685d137c/go.mod h1:rczIT6OzqI4FQIujJe8X/rfxi0pQVFpjQIN2+9vD3Gg= +go.mau.fi/whatsmeow v0.0.0-20231016102050-3a30a129b626 h1:uqurXYTURlhbpEHldNqP8EC15EIP1zWuUABRL2Xqkbs= +go.mau.fi/whatsmeow v0.0.0-20231016102050-3a30a129b626/go.mod h1:u557d2vph8xcLrk3CKTBknUHoB6icUpqazA4w+binRU= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= -golang.org/x/image v0.12.0 h1:w13vZbU4o5rKOFFR8y7M+c4A5jXDC0uXTdHYRP8X2DQ= -golang.org/x/image v0.12.0/go.mod h1:Lu90jvHG7GfemOIcldsh9A2hS01ocl6oNO7ype5mEnk= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/image v0.13.0 h1:3cge/F/QTkNLauhf2QoE9zp+7sr+ZcL4HnoZmdwg9sg= +golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= From a112a7f1417d709d2d3d1049a6a013bc261362ce Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 16 Oct 2023 13:27:11 +0300 Subject: [PATCH 69/94] Bump version to v0.10.3 --- CHANGELOG.md | 2 +- main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de5616f..aad33c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# v0.10.3 (unreleased) +# v0.10.3 (2023-10-16) * Added basic support for channels. * Added default mime type for outgoing attachments when the origin Matrix diff --git a/main.go b/main.go index 0c7d7e4..a501919 100644 --- a/main.go +++ b/main.go @@ -268,7 +268,7 @@ func main() { Name: "mautrix-whatsapp", URL: "https://github.com/mautrix/whatsapp", Description: "A Matrix-WhatsApp puppeting bridge.", - Version: "0.10.2", + Version: "0.10.3", ProtocolName: "WhatsApp", BeeperServiceName: "whatsapp", BeeperNetworkName: "whatsapp", From c02e657d229d4b247280cbfff740edadb7bd6ea0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 16 Oct 2023 20:23:46 +0300 Subject: [PATCH 70/94] Remove old config copy line [skip cd] --- config/upgrade.go | 1 - 1 file changed, 1 deletion(-) diff --git a/config/upgrade.go b/config/upgrade.go index 13d7f79..570e201 100644 --- a/config/upgrade.go +++ b/config/upgrade.go @@ -63,7 +63,6 @@ func DoUpgrade(helper *up.Helper) { helper.Copy(up.Bool, "bridge", "user_avatar_sync") helper.Copy(up.Bool, "bridge", "bridge_matrix_leave") helper.Copy(up.Bool, "bridge", "sync_direct_chat_list") - helper.Copy(up.Bool, "bridge", "default_bridge_receipts") helper.Copy(up.Bool, "bridge", "default_bridge_presence") helper.Copy(up.Bool, "bridge", "send_presence_on_typing") helper.Copy(up.Bool, "bridge", "force_active_delivery_receipts") From e94f611002ffde64f02f3fea0a10700beff4a041 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 18 Oct 2023 13:59:43 +0300 Subject: [PATCH 71/94] Give room admin permission to newsletter admins --- portal.go | 57 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/portal.go b/portal.go index 69bde24..c23a3af 100644 --- a/portal.go +++ b/portal.go @@ -1505,7 +1505,7 @@ func (portal *Portal) UpdateParentGroup(source *User, parent types.JID, updateIn return false } -func (portal *Portal) UpdateMetadata(user *User, groupInfo *types.GroupInfo) bool { +func (portal *Portal) UpdateMetadata(user *User, groupInfo *types.GroupInfo, newsletterMetadata *types.NewsletterMetadata) bool { if portal.IsPrivateChat() { return false } else if portal.IsStatusBroadcastList() { @@ -1529,12 +1529,15 @@ func (portal *Portal) UpdateMetadata(user *User, groupInfo *types.GroupInfo) boo return update } if groupInfo == nil && portal.IsNewsletter() { - newsletterInfo, err := user.Client.GetNewsletterInfo(portal.Key.JID) - if err != nil { - portal.zlog.Err(err).Msg("Failed to get newsletter info") - return false + if newsletterMetadata == nil { + var err error + newsletterMetadata, err = user.Client.GetNewsletterInfo(portal.Key.JID) + if err != nil { + portal.zlog.Err(err).Msg("Failed to get newsletter info") + return false + } } - groupInfo = newsletterToGroupInfo(newsletterInfo) + groupInfo = newsletterToGroupInfo(newsletterMetadata) } if groupInfo == nil { var err error @@ -1564,6 +1567,9 @@ func (portal *Portal) UpdateMetadata(user *User, groupInfo *types.GroupInfo) boo portal.RestrictMessageSending(groupInfo.IsAnnounce) portal.RestrictMetadataChanges(groupInfo.IsLocked) + if newsletterMetadata != nil { + portal.PromoteNewsletterUser(user, newsletterMetadata.ViewerMeta.Role) + } return update } @@ -1586,7 +1592,7 @@ func (portal *Portal) UpdateMatrixRoom(user *User, groupInfo *types.GroupInfo, n } update := false - update = portal.UpdateMetadata(user, groupInfo) || update + update = portal.UpdateMetadata(user, groupInfo, newsletterMetadata) || update if !portal.IsPrivateChat() && !portal.IsBroadcastList() && !portal.IsNewsletter() { update = portal.UpdateAvatar(user, types.EmptyJID, false) || update } else if newsletterMetadata != nil { @@ -1697,6 +1703,35 @@ func (portal *Portal) RestrictMessageSending(restrict bool) id.EventID { } } +func (portal *Portal) PromoteNewsletterUser(user *User, role types.NewsletterRole) id.EventID { + levels, err := portal.MainIntent().PowerLevels(portal.MXID) + if err != nil { + levels = portal.GetBasePowerLevels() + } + + newLevel := 0 + switch role { + case types.NewsletterRoleAdmin: + newLevel = 50 + case types.NewsletterRoleOwner: + newLevel = 95 + } + + changed := portal.applyPowerLevelFixes(levels) + changed = levels.EnsureUserLevel(user.MXID, newLevel) || changed + if !changed { + return "" + } + + resp, err := portal.MainIntent().SetPowerLevels(portal.MXID, levels) + if err != nil { + portal.log.Errorln("Failed to change power levels:", err) + return "" + } else { + return resp.EventID + } +} + func (portal *Portal) RestrictMetadataChanges(restrict bool) id.EventID { levels, err := portal.MainIntent().PowerLevels(portal.MXID) if err != nil { @@ -1901,6 +1936,14 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, n powerLevels.EnsureEventLevel(event.StateTopic, 50) } } + if newsletterMetadata != nil { + switch newsletterMetadata.ViewerMeta.Role { + case types.NewsletterRoleAdmin: + powerLevels.EnsureUserLevel(user.MXID, 50) + case types.NewsletterRoleOwner: + powerLevels.EnsureUserLevel(user.MXID, 95) + } + } bridgeInfoStateKey, bridgeInfo := portal.getBridgeInfo() From 1d9d516b7801bcb7ea1253b3f7bbc8b819dfc2c2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 18 Oct 2023 14:17:41 +0300 Subject: [PATCH 72/94] Add support for newsletters in join and open commands --- commands.go | 62 ++++++++++++++++++++++++++++++++++++++--------------- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 48 insertions(+), 20 deletions(-) diff --git a/commands.go b/commands.go index f69e037..ef2edae 100644 --- a/commands.go +++ b/commands.go @@ -238,18 +238,32 @@ func fnJoin(ce *WrappedCommandEvent) { if len(ce.Args) == 0 { ce.Reply("**Usage:** `join `") return - } else if !strings.HasPrefix(ce.Args[0], whatsmeow.InviteLinkPrefix) { - ce.Reply("That doesn't look like a WhatsApp invite link") - return } - jid, err := ce.User.Client.JoinGroupWithLink(ce.Args[0]) - if err != nil { - ce.Reply("Failed to join group: %v", err) - return + if strings.HasPrefix(ce.Args[0], whatsmeow.InviteLinkPrefix) { + jid, err := ce.User.Client.JoinGroupWithLink(ce.Args[0]) + if err != nil { + ce.Reply("Failed to join group: %v", err) + return + } + ce.Log.Debugln("%s successfully joined group %s", ce.User.MXID, jid) + ce.Reply("Successfully joined group `%s`, the portal should be created momentarily", jid) + } else if strings.HasPrefix(ce.Args[0], whatsmeow.NewsletterLinkPrefix) { + info, err := ce.User.Client.GetNewsletterInfoWithInvite(ce.Args[0]) + if err != nil { + ce.Reply("Failed to get channel info: %v", err) + return + } + err = ce.User.Client.FollowNewsletter(info.ID) + if err != nil { + ce.Reply("Failed to follow channel: %v", err) + return + } + ce.Log.Debugln("%s successfully followed channel %s", ce.User.MXID, info.ID) + ce.Reply("Successfully followed channel `%s`, the portal should be created momentarily", info.ID) + } else { + ce.Reply("That doesn't look like a WhatsApp invite link") } - ce.Log.Debugln("%s successfully joined group %s", ce.User.MXID, jid) - ce.Reply("Successfully joined group `%s`, the portal should be created momentarily", jid) } func tryDecryptEvent(crypto bridge.Crypto, evt *event.Event) (json.RawMessage, error) { @@ -998,23 +1012,37 @@ func fnOpen(ce *WrappedCommandEvent) { } else { jid = types.NewJID(ce.Args[0], types.GroupServer) } - if jid.Server != types.GroupServer || (!strings.ContainsRune(jid.User, '-') && len(jid.User) < 15) { + if (jid.Server != types.GroupServer && jid.Server != types.NewsletterServer) || (!strings.ContainsRune(jid.User, '-') && len(jid.User) < 15) { ce.Reply("That does not look like a group JID") return } - info, err := ce.User.Client.GetGroupInfo(jid) - if err != nil { - ce.Reply("Failed to get group info: %v", err) - return + var err error + var groupInfo *types.GroupInfo + var newsletterMetadata *types.NewsletterMetadata + switch jid.Server { + case types.GroupServer: + groupInfo, err = ce.User.Client.GetGroupInfo(jid) + if err != nil { + ce.Reply("Failed to get group info: %v", err) + return + } + jid = groupInfo.JID + case types.NewsletterServer: + newsletterMetadata, err = ce.User.Client.GetNewsletterInfo(jid) + if err != nil { + ce.Reply("Failed to get channel info: %v", err) + return + } + jid = newsletterMetadata.ID } ce.Log.Debugln("Importing", jid, "for", ce.User.MXID) - portal := ce.User.GetPortalByJID(info.JID) + portal := ce.User.GetPortalByJID(jid) if len(portal.MXID) > 0 { - portal.UpdateMatrixRoom(ce.User, info, nil) + portal.UpdateMatrixRoom(ce.User, groupInfo, newsletterMetadata) ce.Reply("Portal room synced.") } else { - err = portal.CreateMatrixRoom(ce.User, info, nil, true, true) + err = portal.CreateMatrixRoom(ce.User, groupInfo, newsletterMetadata, true, true) if err != nil { ce.Reply("Failed to create room: %v", err) } else { diff --git a/go.mod b/go.mod index fbf2857..c026864 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/tidwall/gjson v1.17.0 go.mau.fi/util v0.2.0 go.mau.fi/webp v0.1.0 - go.mau.fi/whatsmeow v0.0.0-20231016102050-3a30a129b626 + go.mau.fi/whatsmeow v0.0.0-20231018112044-306bf6fef4d2 golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/image v0.13.0 golang.org/x/net v0.17.0 diff --git a/go.sum b/go.sum index 83712d4..7475bc4 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ go.mau.fi/util v0.2.0 h1:AMGBEdg9Ya/smb/09dljo9wBwKr432EpfjDWF7aFQg0= go.mau.fi/util v0.2.0/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84= go.mau.fi/webp v0.1.0 h1:BHObH/DcFntT9KYun5pDr0Ot4eUZO8k2C7eP7vF4ueA= go.mau.fi/webp v0.1.0/go.mod h1:e42Z+VMFrUMS9cpEwGRIor+lQWO8oUAyPyMtcL+NMt8= -go.mau.fi/whatsmeow v0.0.0-20231016102050-3a30a129b626 h1:uqurXYTURlhbpEHldNqP8EC15EIP1zWuUABRL2Xqkbs= -go.mau.fi/whatsmeow v0.0.0-20231016102050-3a30a129b626/go.mod h1:u557d2vph8xcLrk3CKTBknUHoB6icUpqazA4w+binRU= +go.mau.fi/whatsmeow v0.0.0-20231018112044-306bf6fef4d2 h1:27IohJQeoxrcJE44yEE2B2fOsiakqBeQpJDRHF483oM= +go.mau.fi/whatsmeow v0.0.0-20231018112044-306bf6fef4d2/go.mod h1:u557d2vph8xcLrk3CKTBknUHoB6icUpqazA4w+binRU= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= From 5ffb20b85b9441d2ccc5e0cd11a112c0d1c71d9f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Nov 2023 16:16:48 +0200 Subject: [PATCH 73/94] Update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 69edebe..4f2b27b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ *.log /mautrix-whatsapp +/start From 2767819ae0a3fbfef6c3ddca58f95f9f1b68240d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Nov 2023 16:17:13 +0200 Subject: [PATCH 74/94] Check that relaybot is logged in before relaying messages --- messagetracking.go | 4 +++- portal.go | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/messagetracking.go b/messagetracking.go index 57cb4f8..9e79f66 100644 --- a/messagetracking.go +++ b/messagetracking.go @@ -37,6 +37,7 @@ var ( errUserNotConnected = errors.New("you are not connected to WhatsApp") errDifferentUser = errors.New("user is not the recipient of this private chat portal") errUserNotLoggedIn = errors.New("user is not logged in and chat has no relay bot") + errRelaybotNotLoggedIn = errors.New("neither user nor relay bot of chat are logged in") errMNoticeDisabled = errors.New("bridging m.notice messages is disabled") errUnexpectedParsedContentType = errors.New("unexpected parsed content type") errInvalidGeoURI = errors.New("invalid `geo:` URI in message") @@ -111,7 +112,8 @@ func errorToStatusReason(err error) (reason event.MessageStatusReason, status ev errors.Is(err, errUserNotConnected): return event.MessageStatusGenericError, event.MessageStatusRetriable, true, true, "" case errors.Is(err, errUserNotLoggedIn), - errors.Is(err, errDifferentUser): + errors.Is(err, errDifferentUser), + errors.Is(err, errRelaybotNotLoggedIn): return event.MessageStatusGenericError, event.MessageStatusRetriable, true, false, "" case errors.Is(err, errMessageDisconnected), errors.Is(err, errMessageRetryDisconnected): diff --git a/portal.go b/portal.go index c23a3af..1165696 100644 --- a/portal.go +++ b/portal.go @@ -4237,6 +4237,9 @@ func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, ev return nil, sender, extraMeta, errUserNotLoggedIn } sender = portal.GetRelayUser() + if !sender.IsLoggedIn() { + return nil, sender, extraMeta, errRelaybotNotLoggedIn + } isRelay = true } var editRootMsg *database.Message From f1a38dddf325b1a0e906c77515aa6ef7692145f2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Nov 2023 16:20:54 +0200 Subject: [PATCH 75/94] Update dependencies --- go.mod | 24 ++++++++++++------------ go.sum | 48 ++++++++++++++++++++++++------------------------ portal.go | 4 ++-- 3 files changed, 38 insertions(+), 38 deletions(-) diff --git a/go.mod b/go.mod index c026864..322f59f 100644 --- a/go.mod +++ b/go.mod @@ -3,23 +3,23 @@ module maunium.net/go/mautrix-whatsapp go 1.20 require ( - github.com/gorilla/mux v1.8.0 - github.com/gorilla/websocket v1.5.0 + github.com/gorilla/mux v1.8.1 + github.com/gorilla/websocket v1.5.1 github.com/lib/pq v1.10.9 - github.com/mattn/go-sqlite3 v1.14.17 + github.com/mattn/go-sqlite3 v1.14.18 github.com/prometheus/client_golang v1.17.0 github.com/rs/zerolog v1.31.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.17.0 go.mau.fi/util v0.2.0 go.mau.fi/webp v0.1.0 - go.mau.fi/whatsmeow v0.0.0-20231018112044-306bf6fef4d2 - golang.org/x/exp v0.0.0-20231006140011-7918f672742d - golang.org/x/image v0.13.0 - golang.org/x/net v0.17.0 + go.mau.fi/whatsmeow v0.0.0-20231112141858-68b25b3f6fe9 + golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa + golang.org/x/image v0.14.0 + golang.org/x/net v0.18.0 google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.16.1 + maunium.net/go/mautrix v0.16.2-0.20231112104921-da5a51a279a7 ) require ( @@ -38,12 +38,12 @@ require ( github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/sjson v1.2.5 // indirect - github.com/yuin/goldmark v1.5.6 // indirect + github.com/yuin/goldmark v1.6.0 // indirect go.mau.fi/libsignal v0.1.0 // indirect go.mau.fi/zeroconfig v0.1.2 // indirect - golang.org/x/crypto v0.14.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/crypto v0.15.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect maunium.net/go/mauflag v1.0.0 // indirect diff --git a/go.sum b/go.sum index 7475bc4..1c6fddb 100644 --- a/go.sum +++ b/go.sum @@ -16,10 +16,10 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -30,8 +30,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= -github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI= +github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -60,34 +60,34 @@ github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -github.com/yuin/goldmark v1.5.6 h1:COmQAWTCcGetChm3Ig7G/t8AFAN00t+o8Mt4cf7JpwA= -github.com/yuin/goldmark v1.5.6/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.6.0 h1:boZcn2GTjpsynOsC0iJHnBWa4Bi0qzfJjthwauItG68= +github.com/yuin/goldmark v1.6.0/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= go.mau.fi/util v0.2.0 h1:AMGBEdg9Ya/smb/09dljo9wBwKr432EpfjDWF7aFQg0= go.mau.fi/util v0.2.0/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84= go.mau.fi/webp v0.1.0 h1:BHObH/DcFntT9KYun5pDr0Ot4eUZO8k2C7eP7vF4ueA= go.mau.fi/webp v0.1.0/go.mod h1:e42Z+VMFrUMS9cpEwGRIor+lQWO8oUAyPyMtcL+NMt8= -go.mau.fi/whatsmeow v0.0.0-20231018112044-306bf6fef4d2 h1:27IohJQeoxrcJE44yEE2B2fOsiakqBeQpJDRHF483oM= -go.mau.fi/whatsmeow v0.0.0-20231018112044-306bf6fef4d2/go.mod h1:u557d2vph8xcLrk3CKTBknUHoB6icUpqazA4w+binRU= +go.mau.fi/whatsmeow v0.0.0-20231112141858-68b25b3f6fe9 h1:KKiSWzaGPCaf9L0cmpuAkJB2ELccBVol2yHzSW5JE6I= +go.mau.fi/whatsmeow v0.0.0-20231112141858-68b25b3f6fe9/go.mod h1:u557d2vph8xcLrk3CKTBknUHoB6icUpqazA4w+binRU= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= -golang.org/x/image v0.13.0 h1:3cge/F/QTkNLauhf2QoE9zp+7sr+ZcL4HnoZmdwg9sg= -golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= +golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= +golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= @@ -103,5 +103,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.16.1 h1:Wb3CvOCe8A/NLsFeZYxKrgXKiqeZUQEBD1zqm7n/kWk= -maunium.net/go/mautrix v0.16.1/go.mod h1:2Jf15tulVtr6LxoiRL4smRXwpkGWUNfBFhwh/aXDBuk= +maunium.net/go/mautrix v0.16.2-0.20231112104921-da5a51a279a7 h1:et04LrG55EM3vZucOJPu/VzO9b03sKHbTZyHEHThA6A= +maunium.net/go/mautrix v0.16.2-0.20231112104921-da5a51a279a7/go.mod h1:fqEsPvpfZqvLvmwaZFm/6W8fD5SLzrMwZbTga+dVP/g= diff --git a/portal.go b/portal.go index 1165696..8bff916 100644 --- a/portal.go +++ b/portal.go @@ -1567,7 +1567,7 @@ func (portal *Portal) UpdateMetadata(user *User, groupInfo *types.GroupInfo, new portal.RestrictMessageSending(groupInfo.IsAnnounce) portal.RestrictMetadataChanges(groupInfo.IsLocked) - if newsletterMetadata != nil { + if newsletterMetadata != nil && newsletterMetadata.ViewerMeta != nil { portal.PromoteNewsletterUser(user, newsletterMetadata.ViewerMeta.Role) } @@ -1936,7 +1936,7 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, n powerLevels.EnsureEventLevel(event.StateTopic, 50) } } - if newsletterMetadata != nil { + if newsletterMetadata != nil && newsletterMetadata.ViewerMeta != nil { switch newsletterMetadata.ViewerMeta.Role { case types.NewsletterRoleAdmin: powerLevels.EnsureUserLevel(user.MXID, 50) From 0c18f065aaaac0b13214488f524a915e631b6e1c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Nov 2023 16:26:12 +0200 Subject: [PATCH 76/94] Update changelog [skip cd] --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aad33c8..8d5f6f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# v0.10.4 (unreleased) + +* Added support for channels in `join` and `open` commands. +* Added initial bridging of channel admin to room admin status. +* Fixed panic when trying to send message in a portal which has a relaybot set + if the relaybot user gets logged out of WhatsApp. + # v0.10.3 (2023-10-16) * Added basic support for channels. From 71ab396c7afb0ce1c118c9403a1a35d1e464ab25 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 14 Nov 2023 16:27:55 +0200 Subject: [PATCH 77/94] Add timeout for sending reactions and redactions --- go.mod | 6 +++--- go.sum | 12 ++++++------ portal.go | 8 ++++++-- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 322f59f..391d27f 100644 --- a/go.mod +++ b/go.mod @@ -11,15 +11,15 @@ require ( github.com/rs/zerolog v1.31.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.17.0 - go.mau.fi/util v0.2.0 + go.mau.fi/util v0.2.1-0.20231114144345-a692409c548f go.mau.fi/webp v0.1.0 - go.mau.fi/whatsmeow v0.0.0-20231112141858-68b25b3f6fe9 + go.mau.fi/whatsmeow v0.0.0-20231114143452-e8fa5b64d278 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa golang.org/x/image v0.14.0 golang.org/x/net v0.18.0 google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.16.2-0.20231112104921-da5a51a279a7 + maunium.net/go/mautrix v0.16.2-0.20231114144505-09daa655758e ) require ( diff --git a/go.sum b/go.sum index 1c6fddb..299034a 100644 --- a/go.sum +++ b/go.sum @@ -64,12 +64,12 @@ github.com/yuin/goldmark v1.6.0 h1:boZcn2GTjpsynOsC0iJHnBWa4Bi0qzfJjthwauItG68= github.com/yuin/goldmark v1.6.0/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= -go.mau.fi/util v0.2.0 h1:AMGBEdg9Ya/smb/09dljo9wBwKr432EpfjDWF7aFQg0= -go.mau.fi/util v0.2.0/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84= +go.mau.fi/util v0.2.1-0.20231114144345-a692409c548f h1:mJhRlbk3AStG2XfKe1MuO2rmikjUPLcO0pglA+GlWuA= +go.mau.fi/util v0.2.1-0.20231114144345-a692409c548f/go.mod h1:MjlzCQEMzJ+G8RsPawHzpLB8rwTo3aPIjG5FzBvQT/c= go.mau.fi/webp v0.1.0 h1:BHObH/DcFntT9KYun5pDr0Ot4eUZO8k2C7eP7vF4ueA= go.mau.fi/webp v0.1.0/go.mod h1:e42Z+VMFrUMS9cpEwGRIor+lQWO8oUAyPyMtcL+NMt8= -go.mau.fi/whatsmeow v0.0.0-20231112141858-68b25b3f6fe9 h1:KKiSWzaGPCaf9L0cmpuAkJB2ELccBVol2yHzSW5JE6I= -go.mau.fi/whatsmeow v0.0.0-20231112141858-68b25b3f6fe9/go.mod h1:u557d2vph8xcLrk3CKTBknUHoB6icUpqazA4w+binRU= +go.mau.fi/whatsmeow v0.0.0-20231114143452-e8fa5b64d278 h1:oauy+pDJoTbrzwk00+xqb01FUJ2etQGweIoiAvQK1JU= +go.mau.fi/whatsmeow v0.0.0-20231114143452-e8fa5b64d278/go.mod h1:u557d2vph8xcLrk3CKTBknUHoB6icUpqazA4w+binRU= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= @@ -103,5 +103,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.16.2-0.20231112104921-da5a51a279a7 h1:et04LrG55EM3vZucOJPu/VzO9b03sKHbTZyHEHThA6A= -maunium.net/go/mautrix v0.16.2-0.20231112104921-da5a51a279a7/go.mod h1:fqEsPvpfZqvLvmwaZFm/6W8fD5SLzrMwZbTga+dVP/g= +maunium.net/go/mautrix v0.16.2-0.20231114144505-09daa655758e h1:qoKoP99Aa+FvzZecO3R5efV87b7HmMsr1QPsw7lEIpw= +maunium.net/go/mautrix v0.16.2-0.20231114144505-09daa655758e/go.mod h1:Q7tbypaya4C9JVIjPXhEgeU8puotXRrX4hJvTLGNlLQ= diff --git a/portal.go b/portal.go index 8bff916..778f4ce 100644 --- a/portal.go +++ b/portal.go @@ -4659,7 +4659,9 @@ func (portal *Portal) sendReactionToWhatsApp(sender *User, id types.MessageID, t messageKeyParticipant = proto.String(target.Sender.ToNonAD().String()) } key = variationselector.Remove(key) - return sender.Client.SendMessage(context.TODO(), portal.Key.JID, &waProto.Message{ + ctx, cancel := context.WithTimeout(context.TODO(), 60*time.Second) + defer cancel() + return sender.Client.SendMessage(ctx, portal.Key.JID, &waProto.Message{ ReactionMessage: &waProto.ReactionMessage{ Key: &waProto.MessageKey{ RemoteJid: proto.String(portal.Key.JID.String()), @@ -4745,7 +4747,9 @@ func (portal *Portal) HandleMatrixRedaction(sender *User, evt *event.Event) { key.Participant = proto.String(msg.Sender.ToNonAD().String()) } portal.log.Debugfln("Sending redaction %s of %s/%s to WhatsApp", evt.ID, msg.MXID, msg.JID) - _, err := sender.Client.SendMessage(context.TODO(), portal.Key.JID, &waProto.Message{ + ctx, cancel := context.WithTimeout(context.TODO(), 60*time.Second) + defer cancel() + _, err := sender.Client.SendMessage(ctx, portal.Key.JID, &waProto.Message{ ProtocolMessage: &waProto.ProtocolMessage{ Type: waProto.ProtocolMessage_REVOKE.Enum(), Key: key, From 178010a7651ace024f27e9e8950f9d4dffba2e21 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 16 Nov 2023 23:33:52 +0200 Subject: [PATCH 78/94] Update dependencies --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 391d27f..40eebf7 100644 --- a/go.mod +++ b/go.mod @@ -3,23 +3,23 @@ module maunium.net/go/mautrix-whatsapp go 1.20 require ( - github.com/gorilla/mux v1.8.1 - github.com/gorilla/websocket v1.5.1 + github.com/gorilla/mux v1.8.0 + github.com/gorilla/websocket v1.5.0 github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.18 github.com/prometheus/client_golang v1.17.0 github.com/rs/zerolog v1.31.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.17.0 - go.mau.fi/util v0.2.1-0.20231114144345-a692409c548f + go.mau.fi/util v0.2.1 go.mau.fi/webp v0.1.0 - go.mau.fi/whatsmeow v0.0.0-20231114143452-e8fa5b64d278 + go.mau.fi/whatsmeow v0.0.0-20231116213319-217e0c985fd6 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa golang.org/x/image v0.14.0 golang.org/x/net v0.18.0 google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.16.2-0.20231114144505-09daa655758e + maunium.net/go/mautrix v0.16.2 ) require ( diff --git a/go.sum b/go.sum index 299034a..0852109 100644 --- a/go.sum +++ b/go.sum @@ -16,10 +16,10 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= -github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -64,12 +64,12 @@ github.com/yuin/goldmark v1.6.0 h1:boZcn2GTjpsynOsC0iJHnBWa4Bi0qzfJjthwauItG68= github.com/yuin/goldmark v1.6.0/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= -go.mau.fi/util v0.2.1-0.20231114144345-a692409c548f h1:mJhRlbk3AStG2XfKe1MuO2rmikjUPLcO0pglA+GlWuA= -go.mau.fi/util v0.2.1-0.20231114144345-a692409c548f/go.mod h1:MjlzCQEMzJ+G8RsPawHzpLB8rwTo3aPIjG5FzBvQT/c= +go.mau.fi/util v0.2.1 h1:eazulhFE/UmjOFtPrGg6zkF5YfAyiDzQb8ihLMbsPWw= +go.mau.fi/util v0.2.1/go.mod h1:MjlzCQEMzJ+G8RsPawHzpLB8rwTo3aPIjG5FzBvQT/c= go.mau.fi/webp v0.1.0 h1:BHObH/DcFntT9KYun5pDr0Ot4eUZO8k2C7eP7vF4ueA= go.mau.fi/webp v0.1.0/go.mod h1:e42Z+VMFrUMS9cpEwGRIor+lQWO8oUAyPyMtcL+NMt8= -go.mau.fi/whatsmeow v0.0.0-20231114143452-e8fa5b64d278 h1:oauy+pDJoTbrzwk00+xqb01FUJ2etQGweIoiAvQK1JU= -go.mau.fi/whatsmeow v0.0.0-20231114143452-e8fa5b64d278/go.mod h1:u557d2vph8xcLrk3CKTBknUHoB6icUpqazA4w+binRU= +go.mau.fi/whatsmeow v0.0.0-20231116213319-217e0c985fd6 h1:GLGU8Q7zoW0NjqZGXdPAjAwmJTSI09HIdhjL0DY21oc= +go.mau.fi/whatsmeow v0.0.0-20231116213319-217e0c985fd6/go.mod h1:5xTtHNaZpGni6z6aE1iEopjW7wNgsKcolZxZrOujK9M= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= @@ -103,5 +103,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.16.2-0.20231114144505-09daa655758e h1:qoKoP99Aa+FvzZecO3R5efV87b7HmMsr1QPsw7lEIpw= -maunium.net/go/mautrix v0.16.2-0.20231114144505-09daa655758e/go.mod h1:Q7tbypaya4C9JVIjPXhEgeU8puotXRrX4hJvTLGNlLQ= +maunium.net/go/mautrix v0.16.2 h1:a6GUJXNWsTEOO8VE4dROBfCIfPp50mqaqzv7KPzChvg= +maunium.net/go/mautrix v0.16.2/go.mod h1:YL4l4rZB46/vj/ifRMEjcibbvHjgxHftOF1SgmruLu4= From f6270d6bd28e67da029ccc455e9035c3080a89db Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 16 Nov 2023 23:34:04 +0200 Subject: [PATCH 79/94] Bump version to v0.10.4 --- CHANGELOG.md | 2 +- main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d5f6f2..7ab52bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# v0.10.4 (unreleased) +# v0.10.4 (2023-11-16) * Added support for channels in `join` and `open` commands. * Added initial bridging of channel admin to room admin status. diff --git a/main.go b/main.go index a501919..281c9d3 100644 --- a/main.go +++ b/main.go @@ -268,7 +268,7 @@ func main() { Name: "mautrix-whatsapp", URL: "https://github.com/mautrix/whatsapp", Description: "A Matrix-WhatsApp puppeting bridge.", - Version: "0.10.3", + Version: "0.10.4", ProtocolName: "WhatsApp", BeeperServiceName: "whatsapp", BeeperNetworkName: "whatsapp", From a8d1900203cfe3d9b7158edfdddc81f2dc7ce4e0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 19 Nov 2023 14:51:04 +0200 Subject: [PATCH 80/94] Add support for sending media to newsletters --- CHANGELOG.md | 4 ++++ go.mod | 2 +- go.sum | 4 ++-- portal.go | 27 ++++++++++++++++++++++++--- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ab52bc..289838b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# unreleaased + +* Added support for sending media to channels. + # v0.10.4 (2023-11-16) * Added support for channels in `join` and `open` commands. diff --git a/go.mod b/go.mod index 40eebf7..37d4152 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/tidwall/gjson v1.17.0 go.mau.fi/util v0.2.1 go.mau.fi/webp v0.1.0 - go.mau.fi/whatsmeow v0.0.0-20231116213319-217e0c985fd6 + go.mau.fi/whatsmeow v0.0.0-20231119124248-cbe5f2ddc37a golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa golang.org/x/image v0.14.0 golang.org/x/net v0.18.0 diff --git a/go.sum b/go.sum index 0852109..4624c07 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ go.mau.fi/util v0.2.1 h1:eazulhFE/UmjOFtPrGg6zkF5YfAyiDzQb8ihLMbsPWw= go.mau.fi/util v0.2.1/go.mod h1:MjlzCQEMzJ+G8RsPawHzpLB8rwTo3aPIjG5FzBvQT/c= go.mau.fi/webp v0.1.0 h1:BHObH/DcFntT9KYun5pDr0Ot4eUZO8k2C7eP7vF4ueA= go.mau.fi/webp v0.1.0/go.mod h1:e42Z+VMFrUMS9cpEwGRIor+lQWO8oUAyPyMtcL+NMt8= -go.mau.fi/whatsmeow v0.0.0-20231116213319-217e0c985fd6 h1:GLGU8Q7zoW0NjqZGXdPAjAwmJTSI09HIdhjL0DY21oc= -go.mau.fi/whatsmeow v0.0.0-20231116213319-217e0c985fd6/go.mod h1:5xTtHNaZpGni6z6aE1iEopjW7wNgsKcolZxZrOujK9M= +go.mau.fi/whatsmeow v0.0.0-20231119124248-cbe5f2ddc37a h1:u7W/BzMZla2yD5xf0txDRx6ZI7rJNZ1HjLQQKgOgdTw= +go.mau.fi/whatsmeow v0.0.0-20231119124248-cbe5f2ddc37a/go.mod h1:5xTtHNaZpGni6z6aE1iEopjW7wNgsKcolZxZrOujK9M= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= diff --git a/portal.go b/portal.go index 778f4ce..be07262 100644 --- a/portal.go +++ b/portal.go @@ -3894,7 +3894,12 @@ func (portal *Portal) preprocessMatrixMedia(ctx context.Context, sender *User, r portal.log.Warnfln("Failed to re-encode %s media: %v, continuing with original file", mimeType, convertErr) } } - uploadResp, err := sender.Client.Upload(ctx, data, mediaType) + var uploadResp whatsmeow.UploadResponse + if portal.Key.JID.Server == types.NewsletterServer { + uploadResp, err = sender.Client.UploadNewsletter(ctx, data, mediaType) + } else { + uploadResp, err = sender.Client.Upload(ctx, data, mediaType) + } if err != nil { return nil, exerrors.NewDualError(errMediaWhatsAppUploadFailed, err) } @@ -4202,6 +4207,8 @@ type extraConvertMeta struct { EditRootMsg *database.Message GalleryExtraParts []*waProto.Message + + MediaHandle string } func getEditError(rootMsg *database.Message, editer *User) error { @@ -4303,6 +4310,7 @@ func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, ev if media == nil { return nil, sender, extraMeta, err } + extraMeta.MediaHandle = media.Handle ctxInfo.MentionedJid = media.MentionedJIDs msg.ImageMessage = &waProto.ImageMessage{ ContextInfo: ctxInfo, @@ -4321,6 +4329,9 @@ func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, ev return nil, sender, extraMeta, errGalleryRelay } else if content.BeeperGalleryCaption != "" { return nil, sender, extraMeta, errGalleryCaption + } else if portal.Key.JID.Server == types.NewsletterServer { + // We don't handle the media handles properly for multiple messages + return nil, sender, extraMeta, fmt.Errorf("can't send gallery to newsletter") } for i, part := range content.BeeperGalleryImages { // TODO support videos @@ -4352,6 +4363,7 @@ func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, ev if media == nil { return nil, sender, extraMeta, err } + extraMeta.MediaHandle = media.Handle ctxInfo.MentionedJid = media.MentionedJIDs msg.StickerMessage = &waProto.StickerMessage{ ContextInfo: ctxInfo, @@ -4371,6 +4383,7 @@ func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, ev return nil, sender, extraMeta, err } duration := uint32(content.GetInfo().Duration / 1000) + extraMeta.MediaHandle = media.Handle ctxInfo.MentionedJid = media.MentionedJIDs msg.VideoMessage = &waProto.VideoMessage{ ContextInfo: ctxInfo, @@ -4391,6 +4404,7 @@ func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, ev if media == nil { return nil, sender, extraMeta, err } + extraMeta.MediaHandle = media.Handle duration := uint32(content.GetInfo().Duration / 1000) msg.AudioMessage = &waProto.AudioMessage{ ContextInfo: ctxInfo, @@ -4415,6 +4429,7 @@ func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, ev if media == nil { return nil, sender, extraMeta, err } + extraMeta.MediaHandle = media.Handle msg.DocumentMessage = &waProto.DocumentMessage{ ContextInfo: ctxInfo, Caption: &media.Caption, @@ -4562,6 +4577,9 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event, timing go ms.sendMessageMetrics(evt, err, "Error converting", true) return } + if extraMeta == nil { + extraMeta = &extraConvertMeta{} + } dbMsgType := database.MsgNormal if msg.PollCreationMessage != nil || msg.PollCreationMessageV2 != nil || msg.PollCreationMessageV3 != nil { dbMsgType = database.MsgMatrixPoll @@ -4576,12 +4594,15 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event, timing } else { info.ID = dbMsg.JID } - if dbMsgType == database.MsgMatrixPoll && extraMeta != nil && extraMeta.PollOptions != nil { + if dbMsgType == database.MsgMatrixPoll && extraMeta.PollOptions != nil { dbMsg.PutPollOptions(extraMeta.PollOptions) } portal.log.Debugln("Sending event", evt.ID, "to WhatsApp", info.ID) start = time.Now() - resp, err := sender.Client.SendMessage(ctx, portal.Key.JID, msg, whatsmeow.SendRequestExtra{ID: info.ID}) + resp, err := sender.Client.SendMessage(ctx, portal.Key.JID, msg, whatsmeow.SendRequestExtra{ + ID: info.ID, + MediaHandle: extraMeta.MediaHandle, + }) timings.totalSend = time.Since(start) timings.whatsmeow = resp.DebugTimings if err != nil { From 6c983176bd4aff5956e2a1b9375257b80c12dd69 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 27 Nov 2023 20:00:01 +0200 Subject: [PATCH 81/94] Update dependencies --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 37d4152..241a6e7 100644 --- a/go.mod +++ b/go.mod @@ -13,10 +13,10 @@ require ( github.com/tidwall/gjson v1.17.0 go.mau.fi/util v0.2.1 go.mau.fi/webp v0.1.0 - go.mau.fi/whatsmeow v0.0.0-20231119124248-cbe5f2ddc37a + go.mau.fi/whatsmeow v0.0.0-20231127175850-3c97433b4676 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa golang.org/x/image v0.14.0 - golang.org/x/net v0.18.0 + golang.org/x/net v0.19.0 google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 maunium.net/go/mautrix v0.16.2 @@ -41,8 +41,8 @@ require ( github.com/yuin/goldmark v1.6.0 // indirect go.mau.fi/libsignal v0.1.0 // indirect go.mau.fi/zeroconfig v0.1.2 // indirect - golang.org/x/crypto v0.15.0 // indirect - golang.org/x/sys v0.14.0 // indirect + golang.org/x/crypto v0.16.0 // indirect + golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 4624c07..7b20e9e 100644 --- a/go.sum +++ b/go.sum @@ -68,24 +68,24 @@ go.mau.fi/util v0.2.1 h1:eazulhFE/UmjOFtPrGg6zkF5YfAyiDzQb8ihLMbsPWw= go.mau.fi/util v0.2.1/go.mod h1:MjlzCQEMzJ+G8RsPawHzpLB8rwTo3aPIjG5FzBvQT/c= go.mau.fi/webp v0.1.0 h1:BHObH/DcFntT9KYun5pDr0Ot4eUZO8k2C7eP7vF4ueA= go.mau.fi/webp v0.1.0/go.mod h1:e42Z+VMFrUMS9cpEwGRIor+lQWO8oUAyPyMtcL+NMt8= -go.mau.fi/whatsmeow v0.0.0-20231119124248-cbe5f2ddc37a h1:u7W/BzMZla2yD5xf0txDRx6ZI7rJNZ1HjLQQKgOgdTw= -go.mau.fi/whatsmeow v0.0.0-20231119124248-cbe5f2ddc37a/go.mod h1:5xTtHNaZpGni6z6aE1iEopjW7wNgsKcolZxZrOujK9M= +go.mau.fi/whatsmeow v0.0.0-20231127175850-3c97433b4676 h1:YaFOVOdFJCSSByUT6WSgh65ChcVEDZN4QD7OTwaMw+k= +go.mau.fi/whatsmeow v0.0.0-20231127175850-3c97433b4676/go.mod h1:5xTtHNaZpGni6z6aE1iEopjW7wNgsKcolZxZrOujK9M= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= -golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= -golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 4e6e486e19772aaf8f76c2261b58c0c7611641f6 Mon Sep 17 00:00:00 2001 From: Toni Spets Date: Tue, 5 Dec 2023 11:08:42 +0200 Subject: [PATCH 82/94] Expose debug API with pprof Runs along the provisioning API with same authentication. --- config/bridge.go | 5 +++-- config/upgrade.go | 1 + example-config.yaml | 2 ++ provisioning.go | 8 ++++++++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/config/bridge.go b/config/bridge.go index 689c263..538908f 100644 --- a/config/bridge.go +++ b/config/bridge.go @@ -136,8 +136,9 @@ type BridgeConfig struct { Encryption bridgeconfig.EncryptionConfig `yaml:"encryption"` Provisioning struct { - Prefix string `yaml:"prefix"` - SharedSecret string `yaml:"shared_secret"` + Prefix string `yaml:"prefix"` + SharedSecret string `yaml:"shared_secret"` + DebugEndpoints bool `yaml:"debug_endpoints"` } `yaml:"provisioning"` Permissions bridgeconfig.PermissionConfig `yaml:"permissions"` diff --git a/config/upgrade.go b/config/upgrade.go index 570e201..b1999c8 100644 --- a/config/upgrade.go +++ b/config/upgrade.go @@ -160,6 +160,7 @@ func DoUpgrade(helper *up.Helper) { } else { helper.Copy(up.Str, "bridge", "provisioning", "prefix") } + helper.Copy(up.Bool, "bridge", "provisioning", "debug_endpoints") if secret, ok := helper.Get(up.Str, "appservice", "provisioning", "shared_secret"); ok && secret != "generate" { helper.Set(up.Str, secret, "bridge", "provisioning", "shared_secret") } else if secret, ok = helper.Get(up.Str, "bridge", "provisioning", "shared_secret"); !ok || secret == "generate" { diff --git a/example-config.yaml b/example-config.yaml index 95ed83d..058f3e1 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -424,6 +424,8 @@ bridge: # Shared secret for authentication. If set to "generate", a random secret will be generated, # or if set to "disable", the provisioning API will be disabled. shared_secret: generate + # Enable debug API at /debug with provisioning authentication. + debug_endpoints: false # Permissions for using the bridge. # Permitted values: diff --git a/provisioning.go b/provisioning.go index 65c8a2b..911338d 100644 --- a/provisioning.go +++ b/provisioning.go @@ -24,6 +24,7 @@ import ( "fmt" "net" "net/http" + _ "net/http/pprof" "strings" "time" @@ -71,6 +72,13 @@ func (prov *ProvisioningAPI) Init() { prov.bridge.AS.Router.HandleFunc("/_matrix/app/com.beeper.asmux/ping", prov.BridgeStatePing).Methods(http.MethodPost) prov.bridge.AS.Router.HandleFunc("/_matrix/app/com.beeper.bridge_state", prov.BridgeStatePing).Methods(http.MethodPost) + if prov.bridge.Config.Bridge.Provisioning.DebugEndpoints { + prov.log.Debugln("Enabling debug API at /debug") + r := prov.bridge.AS.Router.PathPrefix("/debug").Subrouter() + r.Use(prov.AuthMiddleware) + r.PathPrefix("/pprof").Handler(http.DefaultServeMux) + } + // Deprecated, just use /disconnect r.HandleFunc("/v1/delete_connection", prov.Disconnect).Methods(http.MethodPost) } From 2589049f141cf33190afc6a6f7ee8a293e8b34de Mon Sep 17 00:00:00 2001 From: Toni Spets Date: Wed, 6 Dec 2023 08:58:46 +0200 Subject: [PATCH 83/94] Compress portal event channels into one Use a single channel of pointers to pre-allocate less memory per portal. --- portal.go | 42 +++++++++++++++++++++++++++--------------- user.go | 52 ++++++++++++++++++++++++++++++++-------------------- 2 files changed, 59 insertions(+), 35 deletions(-) diff --git a/portal.go b/portal.go index be07262..ddf13a7 100644 --- a/portal.go +++ b/portal.go @@ -110,7 +110,13 @@ func (portal *Portal) MarkEncrypted() { func (portal *Portal) ReceiveMatrixEvent(user bridge.User, evt *event.Event) { if user.GetPermissionLevel() >= bridgeconfig.PermissionLevelUser || portal.HasRelaybot() { - portal.matrixMessages <- PortalMatrixMessage{user: user.(*User), evt: evt, receivedAt: time.Now()} + portal.events <- &PortalEvent{ + MatrixMessage: &PortalMatrixMessage{ + user: user.(*User), + evt: evt, + receivedAt: time.Now(), + }, + } } } @@ -199,9 +205,7 @@ func (br *WABridge) newBlankPortal(key database.PortalKey) *Portal { log: br.Log.Sub(fmt.Sprintf("Portal/%s", key)), zlog: br.ZLog.With().Str("portal_key", key.String()).Logger(), - messages: make(chan PortalMessage, br.Config.Bridge.PortalMessageBuffer), - matrixMessages: make(chan PortalMatrixMessage, br.Config.Bridge.PortalMessageBuffer), - mediaRetries: make(chan PortalMediaRetry, br.Config.Bridge.PortalMessageBuffer), + events: make(chan *PortalEvent, br.Config.Bridge.PortalMessageBuffer), mediaErrorCache: make(map[types.MessageID]*FailedMediaMeta), } @@ -232,6 +236,12 @@ type fakeMessage struct { Important bool } +type PortalEvent struct { + Message *PortalMessage + MatrixMessage *PortalMatrixMessage + MediaRetry *PortalMediaRetry +} + type PortalMessage struct { evt *events.Message undecryptable *events.UndecryptableMessage @@ -279,9 +289,7 @@ type Portal struct { currentlyTyping []id.UserID currentlyTypingLock sync.Mutex - messages chan PortalMessage - matrixMessages chan PortalMatrixMessage - mediaRetries chan PortalMediaRetry + events chan *PortalEvent mediaErrorCache map[types.MessageID]*FailedMediaMeta @@ -337,7 +345,7 @@ var ( _ bridge.TypingPortal = (*Portal)(nil) ) -func (portal *Portal) handleWhatsAppMessageLoopItem(msg PortalMessage) { +func (portal *Portal) handleWhatsAppMessageLoopItem(msg *PortalMessage) { if len(portal.MXID) == 0 { if msg.fake == nil && msg.undecryptable == nil && (msg.evt == nil || !containsSupportedMessage(msg.evt.Message)) { portal.log.Debugln("Not creating portal room for incoming message: message is not a chat message") @@ -369,7 +377,7 @@ func (portal *Portal) handleWhatsAppMessageLoopItem(msg PortalMessage) { } } -func (portal *Portal) handleMatrixMessageLoopItem(msg PortalMatrixMessage) { +func (portal *Portal) handleMatrixMessageLoopItem(msg *PortalMatrixMessage) { portal.latestEventBackfillLock.Lock() defer portal.latestEventBackfillLock.Unlock() evtTS := time.UnixMilli(msg.evt.Timestamp) @@ -483,12 +491,16 @@ func (portal *Portal) handleOneMessageLoopItem() { } }() select { - case msg := <-portal.messages: - portal.handleWhatsAppMessageLoopItem(msg) - case msg := <-portal.matrixMessages: - portal.handleMatrixMessageLoopItem(msg) - case retry := <-portal.mediaRetries: - portal.handleMediaRetry(retry.evt, retry.source) + case msg := <-portal.events: + if msg.Message != nil { + portal.handleWhatsAppMessageLoopItem(msg.Message) + } else if msg.MatrixMessage != nil { + portal.handleMatrixMessageLoopItem(msg.MatrixMessage) + } else if msg.MediaRetry != nil { + portal.handleMediaRetry(msg.MediaRetry.evt, msg.MediaRetry.source) + } else { + portal.log.Warn("Portal event loop returned an event without any data") + } } } diff --git a/user.go b/user.go index 5926fe7..00e276c 100644 --- a/user.go +++ b/user.go @@ -635,15 +635,17 @@ func (user *User) handleCallStart(sender types.JID, id, callType string, ts time if callType != "" { text = fmt.Sprintf("Incoming %s call. Use the WhatsApp app to answer.", callType) } - portal.messages <- PortalMessage{ - fake: &fakeMessage{ - Sender: sender, - Text: text, - ID: id, - Time: ts, - Important: true, + portal.events <- &PortalEvent{ + Message: &PortalMessage{ + fake: &fakeMessage{ + Sender: sender, + Text: text, + ID: id, + Time: ts, + Important: true, + }, + source: user, }, - source: user, } } @@ -865,11 +867,15 @@ func (user *User) HandleEvent(event interface{}) { go user.handleChatPresence(v) case *events.Message: portal := user.GetPortalByMessageSource(v.Info.MessageSource) - portal.messages <- PortalMessage{evt: v, source: user} + portal.events <- &PortalEvent{ + Message: &PortalMessage{evt: v, source: user}, + } case *events.MediaRetry: user.phoneSeen(v.Timestamp) portal := user.GetPortalByJID(v.ChatID) - portal.mediaRetries <- PortalMediaRetry{evt: v, source: user} + portal.events <- &PortalEvent{ + MediaRetry: &PortalMediaRetry{evt: v, source: user}, + } case *events.CallOffer: user.handleCallStart(v.CallCreator, v.CallID, "", v.Timestamp) case *events.CallOfferNotice: @@ -885,22 +891,26 @@ func (user *User) HandleEvent(event interface{}) { if v.Implicit { text = fmt.Sprintf("Your security code with %s (device #%d) changed.", puppet.Displayname, v.JID.Device) } - portal.messages <- PortalMessage{ - fake: &fakeMessage{ - Sender: v.JID, - Text: text, - ID: strconv.FormatInt(v.Timestamp.Unix(), 10), - Time: v.Timestamp, - Important: false, + portal.events <- &PortalEvent{ + Message: &PortalMessage{ + fake: &fakeMessage{ + Sender: v.JID, + Text: text, + ID: strconv.FormatInt(v.Timestamp.Unix(), 10), + Time: v.Timestamp, + Important: false, + }, + source: user, }, - source: user, } } case *events.CallTerminate, *events.CallRelayLatency, *events.CallAccept, *events.UnknownCallEvent: // ignore case *events.UndecryptableMessage: portal := user.GetPortalByMessageSource(v.Info.MessageSource) - portal.messages <- PortalMessage{undecryptable: v, source: user} + portal.events <- &PortalEvent{ + Message: &PortalMessage{undecryptable: v, source: user}, + } case *events.HistorySync: if user.bridge.Config.Bridge.HistorySync.Backfill { user.historySyncs <- v @@ -1235,7 +1245,9 @@ func (user *User) handleReceipt(receipt *events.Receipt) { if portal == nil || len(portal.MXID) == 0 { return } - portal.messages <- PortalMessage{receipt: receipt, source: user} + portal.events <- &PortalEvent{ + Message: &PortalMessage{receipt: receipt, source: user}, + } } func (user *User) makeReadMarkerContent(eventID id.EventID, doublePuppet bool) CustomReadMarkers { From 345288bf0a37cf36e0d3ce097926a0fc7896ada5 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 7 Dec 2023 13:48:19 +0200 Subject: [PATCH 84/94] Add icon for IDEA --- .idea/icon.svg | 1 + 1 file changed, 1 insertion(+) create mode 100644 .idea/icon.svg diff --git a/.idea/icon.svg b/.idea/icon.svg new file mode 100644 index 0000000..17c2d59 --- /dev/null +++ b/.idea/icon.svg @@ -0,0 +1 @@ + From e780eb2eb6869ca19387bfcf75e86b9b6b3a137d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 8 Dec 2023 15:02:40 +0200 Subject: [PATCH 85/94] Update Docker image to Alpine 3.19 --- Dockerfile | 4 ++-- Dockerfile.ci | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b2943ea..5931d5e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1-alpine3.18 AS builder +FROM golang:1-alpine3.19 AS builder RUN apk add --no-cache git ca-certificates build-base su-exec olm-dev @@ -6,7 +6,7 @@ COPY . /build WORKDIR /build RUN go build -o /usr/bin/mautrix-whatsapp -FROM alpine:3.18 +FROM alpine:3.19 ENV UID=1337 \ GID=1337 diff --git a/Dockerfile.ci b/Dockerfile.ci index 2860a1d..e08e6ca 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM alpine:3.18 +FROM alpine:3.19 ENV UID=1337 \ GID=1337 From 3e29c51c06561189636bb721490a812c58bb8ef6 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 13 Dec 2023 19:57:23 +0200 Subject: [PATCH 86/94] Add missing nil check --- portal.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portal.go b/portal.go index ddf13a7..85c61e3 100644 --- a/portal.go +++ b/portal.go @@ -4622,7 +4622,7 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event, timing return } dbMsg.MarkSent(resp.Timestamp) - if len(extraMeta.GalleryExtraParts) > 0 { + if extraMeta != nil && len(extraMeta.GalleryExtraParts) > 0 { for i, part := range extraMeta.GalleryExtraParts { partInfo := portal.generateMessageInfo(sender) partDBMsg := portal.markHandled(nil, nil, partInfo, evt.ID, evt.Sender, false, true, database.MsgBeeperGallery, i+1, database.MsgNoError) From b752bb9c15a2b83a512eab996ff6eca30226b384 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Dec 2023 14:47:41 +0200 Subject: [PATCH 87/94] Update whatsmeow --- go.mod | 2 +- go.sum | 4 ++-- portal.go | 10 +++------- user.go | 2 +- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 241a6e7..012b2b8 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/tidwall/gjson v1.17.0 go.mau.fi/util v0.2.1 go.mau.fi/webp v0.1.0 - go.mau.fi/whatsmeow v0.0.0-20231127175850-3c97433b4676 + go.mau.fi/whatsmeow v0.0.0-20231213182926-f17761e0fe8d golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa golang.org/x/image v0.14.0 golang.org/x/net v0.19.0 diff --git a/go.sum b/go.sum index 7b20e9e..fb22b62 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ go.mau.fi/util v0.2.1 h1:eazulhFE/UmjOFtPrGg6zkF5YfAyiDzQb8ihLMbsPWw= go.mau.fi/util v0.2.1/go.mod h1:MjlzCQEMzJ+G8RsPawHzpLB8rwTo3aPIjG5FzBvQT/c= go.mau.fi/webp v0.1.0 h1:BHObH/DcFntT9KYun5pDr0Ot4eUZO8k2C7eP7vF4ueA= go.mau.fi/webp v0.1.0/go.mod h1:e42Z+VMFrUMS9cpEwGRIor+lQWO8oUAyPyMtcL+NMt8= -go.mau.fi/whatsmeow v0.0.0-20231127175850-3c97433b4676 h1:YaFOVOdFJCSSByUT6WSgh65ChcVEDZN4QD7OTwaMw+k= -go.mau.fi/whatsmeow v0.0.0-20231127175850-3c97433b4676/go.mod h1:5xTtHNaZpGni6z6aE1iEopjW7wNgsKcolZxZrOujK9M= +go.mau.fi/whatsmeow v0.0.0-20231213182926-f17761e0fe8d h1:ERk1aBRvcEC1fLb2HXDCb6UFvFHtuOSyRzFee9BxII4= +go.mau.fi/whatsmeow v0.0.0-20231213182926-f17761e0fe8d/go.mod h1:5xTtHNaZpGni6z6aE1iEopjW7wNgsKcolZxZrOujK9M= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= diff --git a/portal.go b/portal.go index 85c61e3..f330bb4 100644 --- a/portal.go +++ b/portal.go @@ -430,7 +430,7 @@ func (portal *Portal) handleReceipt(receipt *events.Receipt, source *User) { // TODO handle lids return } - if receipt.Type == events.ReceiptTypeDelivered { + if receipt.Type == types.ReceiptTypeDelivered { portal.handleDeliveryReceipt(receipt, source) return } @@ -5039,9 +5039,7 @@ func (portal *Portal) HandleMatrixLeave(brSender bridge.User) { func (portal *Portal) HandleMatrixKick(brSender bridge.User, brTarget bridge.Ghost) { sender := brSender.(*User) target := brTarget.(*Puppet) - _, err := sender.Client.UpdateGroupParticipants(portal.Key.JID, map[types.JID]whatsmeow.ParticipantChange{ - target.JID: whatsmeow.ParticipantChangeRemove, - }) + _, err := sender.Client.UpdateGroupParticipants(portal.Key.JID, []types.JID{target.JID}, whatsmeow.ParticipantChangeRemove) if err != nil { portal.log.Errorfln("Failed to kick %s from group as %s: %v", target.JID, sender.MXID, err) return @@ -5052,9 +5050,7 @@ func (portal *Portal) HandleMatrixKick(brSender bridge.User, brTarget bridge.Gho func (portal *Portal) HandleMatrixInvite(brSender bridge.User, brTarget bridge.Ghost) { sender := brSender.(*User) target := brTarget.(*Puppet) - _, err := sender.Client.UpdateGroupParticipants(portal.Key.JID, map[types.JID]whatsmeow.ParticipantChange{ - target.JID: whatsmeow.ParticipantChangeAdd, - }) + _, err := sender.Client.UpdateGroupParticipants(portal.Key.JID, []types.JID{target.JID}, whatsmeow.ParticipantChangeAdd) if err != nil { portal.log.Errorfln("Failed to add %s to group as %s: %v", target.JID, sender.MXID, err) return diff --git a/user.go b/user.go index 00e276c..a6f9f68 100644 --- a/user.go +++ b/user.go @@ -1238,7 +1238,7 @@ func (user *User) handleChatPresence(presence *events.ChatPresence) { } func (user *User) handleReceipt(receipt *events.Receipt) { - if receipt.Type != events.ReceiptTypeRead && receipt.Type != events.ReceiptTypeReadSelf && receipt.Type != events.ReceiptTypeDelivered { + if receipt.Type != types.ReceiptTypeRead && receipt.Type != types.ReceiptTypeReadSelf && receipt.Type != types.ReceiptTypeDelivered { return } portal := user.GetPortalByMessageSource(receipt.MessageSource) From 19f6f8c896e19ad2e36ed1cc585d6b5facbc1a22 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 13 Dec 2023 20:41:16 +0200 Subject: [PATCH 88/94] Update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 289838b..5ed46d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ -# unreleaased +# v0.10.5 (unreleaased) * Added support for sending media to channels. +* Fixed voting in polls (seems to have broken due to a server-side change). +* Improved memory usage for bridges with lots of portals. # v0.10.4 (2023-11-16) From 6097735b3c4355a7007beecb23b183ab7eda0bb7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 15 Dec 2023 14:36:18 +0200 Subject: [PATCH 89/94] Add word to error message --- commands.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commands.go b/commands.go index ef2edae..339a195 100644 --- a/commands.go +++ b/commands.go @@ -114,7 +114,7 @@ func fnSetRelay(ce *WrappedCommandEvent) { if !ce.Bridge.Config.Bridge.Relay.Enabled { ce.Reply("Relay mode is not enabled on this instance of the bridge") } else if ce.Bridge.Config.Bridge.Relay.AdminOnly && !ce.User.Admin { - ce.Reply("Only admins are allowed to enable relay mode on this instance of the bridge") + ce.Reply("Only bridge admins are allowed to enable relay mode on this instance of the bridge") } else { ce.Portal.RelayUserID = ce.User.MXID ce.Portal.Update(nil) @@ -136,7 +136,7 @@ func fnUnsetRelay(ce *WrappedCommandEvent) { if !ce.Bridge.Config.Bridge.Relay.Enabled { ce.Reply("Relay mode is not enabled on this instance of the bridge") } else if ce.Bridge.Config.Bridge.Relay.AdminOnly && !ce.User.Admin { - ce.Reply("Only admins are allowed to enable relay mode on this instance of the bridge") + ce.Reply("Only bridge admins are allowed to enable relay mode on this instance of the bridge") } else { ce.Portal.RelayUserID = "" ce.Portal.Update(nil) From 687a90b7b2ec9ab0ef21a5b21e415920ee4d93f1 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 16 Dec 2023 23:37:12 +0200 Subject: [PATCH 90/94] Bump version to v0.10.5 --- CHANGELOG.md | 2 +- go.mod | 6 +++--- go.sum | 12 ++++++------ main.go | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ed46d1..949d7df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# v0.10.5 (unreleaased) +# v0.10.5 (2023-12-16) * Added support for sending media to channels. * Fixed voting in polls (seems to have broken due to a server-side change). diff --git a/go.mod b/go.mod index 012b2b8..6135da5 100644 --- a/go.mod +++ b/go.mod @@ -6,15 +6,15 @@ require ( github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 github.com/lib/pq v1.10.9 - github.com/mattn/go-sqlite3 v1.14.18 + github.com/mattn/go-sqlite3 v1.14.19 github.com/prometheus/client_golang v1.17.0 github.com/rs/zerolog v1.31.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.17.0 go.mau.fi/util v0.2.1 go.mau.fi/webp v0.1.0 - go.mau.fi/whatsmeow v0.0.0-20231213182926-f17761e0fe8d - golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa + go.mau.fi/whatsmeow v0.0.0-20231216213200-9d803dd92735 + golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 golang.org/x/image v0.14.0 golang.org/x/net v0.19.0 google.golang.org/protobuf v1.31.0 diff --git a/go.sum b/go.sum index fb22b62..f568af8 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI= -github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= +github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -68,14 +68,14 @@ go.mau.fi/util v0.2.1 h1:eazulhFE/UmjOFtPrGg6zkF5YfAyiDzQb8ihLMbsPWw= go.mau.fi/util v0.2.1/go.mod h1:MjlzCQEMzJ+G8RsPawHzpLB8rwTo3aPIjG5FzBvQT/c= go.mau.fi/webp v0.1.0 h1:BHObH/DcFntT9KYun5pDr0Ot4eUZO8k2C7eP7vF4ueA= go.mau.fi/webp v0.1.0/go.mod h1:e42Z+VMFrUMS9cpEwGRIor+lQWO8oUAyPyMtcL+NMt8= -go.mau.fi/whatsmeow v0.0.0-20231213182926-f17761e0fe8d h1:ERk1aBRvcEC1fLb2HXDCb6UFvFHtuOSyRzFee9BxII4= -go.mau.fi/whatsmeow v0.0.0-20231213182926-f17761e0fe8d/go.mod h1:5xTtHNaZpGni6z6aE1iEopjW7wNgsKcolZxZrOujK9M= +go.mau.fi/whatsmeow v0.0.0-20231216213200-9d803dd92735 h1:+teJYCOK6M4Kn2TYCj29levhHVwnJTmgCtEXLtgwQtM= +go.mau.fi/whatsmeow v0.0.0-20231216213200-9d803dd92735/go.mod h1:5xTtHNaZpGni6z6aE1iEopjW7wNgsKcolZxZrOujK9M= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= -golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= +golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4= +golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= diff --git a/main.go b/main.go index 281c9d3..c8958af 100644 --- a/main.go +++ b/main.go @@ -268,7 +268,7 @@ func main() { Name: "mautrix-whatsapp", URL: "https://github.com/mautrix/whatsapp", Description: "A Matrix-WhatsApp puppeting bridge.", - Version: "0.10.4", + Version: "0.10.5", ProtocolName: "WhatsApp", BeeperServiceName: "whatsapp", BeeperNetworkName: "whatsapp", From a1f1c91be1d0192d306f31f48b8fe71203d9ccc7 Mon Sep 17 00:00:00 2001 From: Toni Spets Date: Tue, 19 Dec 2023 13:28:34 +0200 Subject: [PATCH 91/94] Prevent creating portals for non-backfillable conversations Defer initializing portal on backfill until there are messages. --- historysync.go | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/historysync.go b/historysync.go index bd16077..332f922 100644 --- a/historysync.go +++ b/historysync.go @@ -21,6 +21,7 @@ import ( "encoding/base64" "fmt" "strings" + "sync" "time" "github.com/rs/zerolog" @@ -452,26 +453,29 @@ func (user *User) storeHistorySync(evt *waProto.HistorySync) { continue } totalMessageCount += len(conv.GetMessages()) - portal := user.GetPortalByJID(jid) log := log.With(). - Str("chat_jid", portal.Key.JID.String()). + Str("chat_jid", jid.String()). Int("msg_count", len(conv.GetMessages())). Logger() - historySyncConversation := user.bridge.DB.HistorySync.NewConversationWithValues( - user.MXID, - conv.GetId(), - &portal.Key, - getConversationTimestamp(conv), - conv.GetMuteEndTime(), - conv.GetArchived(), - conv.GetPinned(), - conv.GetDisappearingMode().GetInitiator(), - conv.GetEndOfHistoryTransferType(), - conv.EphemeralExpiration, - conv.GetMarkedAsUnread(), - conv.GetUnreadCount()) - historySyncConversation.Upsert() + initPortal := sync.OnceFunc(func() { + portal := user.GetPortalByJID(jid) + historySyncConversation := user.bridge.DB.HistorySync.NewConversationWithValues( + user.MXID, + conv.GetId(), + &portal.Key, + getConversationTimestamp(conv), + conv.GetMuteEndTime(), + conv.GetArchived(), + conv.GetPinned(), + conv.GetDisappearingMode().GetInitiator(), + conv.GetEndOfHistoryTransferType(), + conv.EphemeralExpiration, + conv.GetMarkedAsUnread(), + conv.GetUnreadCount()) + historySyncConversation.Upsert() + }) + var minTime, maxTime time.Time var minTimeIndex, maxTimeIndex int @@ -479,7 +483,7 @@ func (user *User) storeHistorySync(evt *waProto.HistorySync) { unsupportedTypes := 0 for i, rawMsg := range conv.GetMessages() { // Don't store messages that will just be skipped. - msgEvt, err := user.Client.ParseWebMessage(portal.Key.JID, rawMsg.GetMessage()) + msgEvt, err := user.Client.ParseWebMessage(jid, rawMsg.GetMessage()) if err != nil { log.Warn().Err(err). Int("msg_index", i). @@ -503,6 +507,8 @@ func (user *User) storeHistorySync(evt *waProto.HistorySync) { continue } + initPortal() + message, err := user.bridge.DB.HistorySync.NewMessageWithValues(user.MXID, conv.GetId(), msgEvt.Info.ID, rawMsg) if err != nil { log.Error().Err(err). From 7b3990dab9a380e2269089b28ae6817a5092d74d Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Sat, 23 Dec 2023 18:10:52 -0700 Subject: [PATCH 92/94] pre-commit: ban Msgf() from zerolog Signed-off-by: Sumner Evans --- .pre-commit-config.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1ef386e..64a6eae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: trailing-whitespace exclude_types: [markdown] @@ -13,3 +13,8 @@ repos: hooks: - id: go-imports-repo - id: go-vet-repo-mod + + - repo: https://github.com/beeper/pre-commit-go + rev: v0.2.2 + hooks: + - id: zerolog-ban-msgf From cb6afe2e82514ca0f16513f376c49f01af219434 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 28 Dec 2023 17:03:21 +0100 Subject: [PATCH 93/94] Update actions and run on both supported Go versions [skip cd] --- .github/workflows/go.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 965969e..b2e717d 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -5,13 +5,19 @@ on: [push, pull_request] jobs: lint: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + go-version: ["1.20", "1.21"] + steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: - go-version: "1.21" + go-version: ${{ matrix.go-version }} + cache: true - name: Install libolm run: sudo apt-get install libolm-dev libolm3 From 4efb2260459cfd74a863f7b9b71178873c1fc978 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 28 Dec 2023 17:13:16 +0100 Subject: [PATCH 94/94] Remove sync.OnceFunc usage. Fixes #679 --- historysync.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/historysync.go b/historysync.go index 332f922..4adb14e 100644 --- a/historysync.go +++ b/historysync.go @@ -21,7 +21,6 @@ import ( "encoding/base64" "fmt" "strings" - "sync" "time" "github.com/rs/zerolog" @@ -458,8 +457,12 @@ func (user *User) storeHistorySync(evt *waProto.HistorySync) { Int("msg_count", len(conv.GetMessages())). Logger() - initPortal := sync.OnceFunc(func() { - portal := user.GetPortalByJID(jid) + var portal *Portal + initPortal := func() { + if portal != nil { + return + } + portal = user.GetPortalByJID(jid) historySyncConversation := user.bridge.DB.HistorySync.NewConversationWithValues( user.MXID, conv.GetId(), @@ -474,7 +477,7 @@ func (user *User) storeHistorySync(evt *waProto.HistorySync) { conv.GetMarkedAsUnread(), conv.GetUnreadCount()) historySyncConversation.Upsert() - }) + } var minTime, maxTime time.Time var minTimeIndex, maxTimeIndex int