diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 3861af0..b2e717d 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -5,13 +5,19 @@ on: [push, pull_request] jobs: lint: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + go-version: ["1.20", "1.21"] + steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: - go-version: "1.20" + go-version: ${{ matrix.go-version }} + cache: true - name: Install libolm run: sudo apt-get install libolm-dev libolm3 diff --git a/.gitignore b/.gitignore index 69edebe..4f2b27b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ *.log /mautrix-whatsapp +/start diff --git a/.idea/icon.svg b/.idea/icon.svg new file mode 100644 index 0000000..17c2d59 --- /dev/null +++ b/.idea/icon.svg @@ -0,0 +1 @@ + diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1ef386e..64a6eae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: trailing-whitespace exclude_types: [markdown] @@ -13,3 +13,8 @@ repos: hooks: - id: go-imports-repo - id: go-vet-repo-mod + + - repo: https://github.com/beeper/pre-commit-go + rev: v0.2.2 + hooks: + - id: zerolog-ban-msgf diff --git a/CHANGELOG.md b/CHANGELOG.md index 257676d..949d7df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,72 @@ -# v0.8.6 (unreleased) +# v0.10.5 (2023-12-16) + +* Added support for sending media to channels. +* Fixed voting in polls (seems to have broken due to a server-side change). +* Improved memory usage for bridges with lots of portals. + +# v0.10.4 (2023-11-16) + +* Added support for channels in `join` and `open` commands. +* Added initial bridging of channel admin to room admin status. +* Fixed panic when trying to send message in a portal which has a relaybot set + if the relaybot user gets logged out of WhatsApp. + +# v0.10.3 (2023-10-16) + +* Added basic support for channels. +* Added default mime type for outgoing attachments when the origin Matrix + client forgets to specify the mime type. +* Fixed legacy backfill creating portals for chats without messages. +* Updated libwebp version used for encoding. + +# v0.10.2 (security update) + +* Stopped using libwebp for decoding webps. + +# v0.10.1 (2023-09-16) + +* Added support for double puppeting with arbitrary `as_token`s. + See [docs](https://docs.mau.fi/bridges/general/double-puppeting.html#appservice-method-new) for more info. +* Added retrying for media downloads when WhatsApp servers break and start + returning 429s and 503s. +* Fixed logging in with 8-letter code. +* Fixed syncing community announcement groups. +* Changed "Incoming call" message to explicitly say you have to open WhatsApp + on your phone to answer. + +# v0.10.0 (2023-08-16) + +* Bumped minimum Go version to 1.20. +* Added automatic re-requesting of undecryptable WhatsApp messages from primary + device. +* Added support for round video messages. +* Added support for logging in by entering a 8-letter code on the phone instead + of scanning a QR code. + * Note: due to a server-side change, code login may only work when `os_name` + and `browser_name` in the config are set in a specific way. This is fixed + in v0.10.1. + +# v0.9.0 (2023-07-16) + +* Removed MSC2716 support. +* Added legacy backfill support. +* Updated Docker image to Alpine 3.18. +* Changed all ogg audio messages from WhatsApp to be bridged as voice messages + to Matrix, as WhatsApp removes the voice message flag when forwarding for + some reason. +* Added Prometheus metric for WhatsApp connection failures + (thanks to [@Half-Shot] in [#620]). + +[#620]: https://github.com/mautrix/whatsapp/pull/620 + +# v0.8.6 (2023-06-16) * Implemented intentional mentions for outgoing messages. +* Added support for appservice websockets. +* Added additional index on message table to make bridging outgoing read + receipts and messages faster in chats with lots of messages. +* Fixed handling WhatsApp poll messages that only allow one choice. +* Fixed bridging new groups immediately when they're created. # v0.8.5 (2023-05-16) diff --git a/Dockerfile b/Dockerfile index 6278562..5931d5e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1-alpine3.17 AS builder +FROM golang:1-alpine3.19 AS builder RUN apk add --no-cache git ca-certificates build-base su-exec olm-dev @@ -6,7 +6,7 @@ COPY . /build WORKDIR /build RUN go build -o /usr/bin/mautrix-whatsapp -FROM alpine:3.17 +FROM alpine:3.19 ENV UID=1337 \ GID=1337 diff --git a/Dockerfile.ci b/Dockerfile.ci index b3f736c..e08e6ca 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM alpine:3.17 +FROM alpine:3.19 ENV UID=1337 \ GID=1337 diff --git a/Dockerfile.dev b/Dockerfile.dev index 5d4413d..4edc6ad 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM golang:1-alpine3.17 +FROM golang:1-alpine3.18 RUN apk add --no-cache git ca-certificates build-base su-exec olm-dev bash jq yq curl diff --git a/segment.go b/analytics.go similarity index 76% rename from segment.go rename to analytics.go index 16b6e53..60ebd0a 100644 --- a/segment.go +++ b/analytics.go @@ -26,27 +26,26 @@ import ( "maunium.net/go/mautrix/id" ) -const SegmentURL = "https://api.segment.io/v1/track" - -type SegmentClient struct { +type AnalyticsClient struct { + url string key string userID string log log.Logger client http.Client } -var Segment SegmentClient +var Analytics AnalyticsClient -func (sc *SegmentClient) trackSync(userID id.UserID, event string, properties map[string]interface{}) error { +func (sc *AnalyticsClient) trackSync(userID id.UserID, event string, properties map[string]interface{}) error { var buf bytes.Buffer - var segmentUserID string - if Segment.userID != "" { - segmentUserID = Segment.userID + var analyticsUserID string + if Analytics.userID != "" { + analyticsUserID = Analytics.userID } else { - segmentUserID = userID.String() + analyticsUserID = userID.String() } err := json.NewEncoder(&buf).Encode(map[string]interface{}{ - "userId": segmentUserID, + "userId": analyticsUserID, "event": event, "properties": properties, }) @@ -54,7 +53,7 @@ func (sc *SegmentClient) trackSync(userID id.UserID, event string, properties ma return err } - req, err := http.NewRequest("POST", SegmentURL, &buf) + req, err := http.NewRequest(http.MethodPost, sc.url, &buf) if err != nil { return err } @@ -70,11 +69,11 @@ func (sc *SegmentClient) trackSync(userID id.UserID, event string, properties ma return nil } -func (sc *SegmentClient) IsEnabled() bool { +func (sc *AnalyticsClient) IsEnabled() bool { return len(sc.key) > 0 } -func (sc *SegmentClient) Track(userID id.UserID, event string, properties ...map[string]interface{}) { +func (sc *AnalyticsClient) Track(userID id.UserID, event string, properties ...map[string]interface{}) { if !sc.IsEnabled() { return } else if len(properties) > 1 { diff --git a/backfillqueue.go b/backfillqueue.go index 51e50c2..ab79644 100644 --- a/backfillqueue.go +++ b/backfillqueue.go @@ -65,7 +65,7 @@ func (user *User) HandleBackfillRequestsLoop(backfillTypes []database.BackfillTy req := user.BackfillQueue.GetNextBackfill(user.MXID, backfillTypes, waitForBackfillTypes, reCheckChannel) user.log.Infofln("Handling backfill request %s", req) - conv := user.bridge.DB.HistorySync.GetConversation(user.MXID, req.Portal) + conv := user.bridge.DB.HistorySync.GetConversation(user.MXID, *req.Portal) if conv == nil { user.log.Debugfln("Could not find history sync conversation data for %s", req.Portal.String()) req.MarkDone() diff --git a/commands.go b/commands.go index b7bc65c..339a195 100644 --- a/commands.go +++ b/commands.go @@ -23,6 +23,7 @@ import ( "fmt" "html" "math" + "regexp" "sort" "strconv" "strings" @@ -41,8 +42,6 @@ import ( "maunium.net/go/mautrix/bridge/status" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" - - "maunium.net/go/mautrix-whatsapp/database" ) type WrappedCommandEvent struct { @@ -71,7 +70,6 @@ func (br *WABridge) RegisterCommands() { cmdPing, cmdDeletePortal, cmdDeleteAllPortals, - cmdBackfill, cmdList, cmdSearch, cmdOpen, @@ -116,7 +114,7 @@ func fnSetRelay(ce *WrappedCommandEvent) { if !ce.Bridge.Config.Bridge.Relay.Enabled { ce.Reply("Relay mode is not enabled on this instance of the bridge") } else if ce.Bridge.Config.Bridge.Relay.AdminOnly && !ce.User.Admin { - ce.Reply("Only admins are allowed to enable relay mode on this instance of the bridge") + ce.Reply("Only bridge admins are allowed to enable relay mode on this instance of the bridge") } else { ce.Portal.RelayUserID = ce.User.MXID ce.Portal.Update(nil) @@ -138,7 +136,7 @@ func fnUnsetRelay(ce *WrappedCommandEvent) { if !ce.Bridge.Config.Bridge.Relay.Enabled { ce.Reply("Relay mode is not enabled on this instance of the bridge") } else if ce.Bridge.Config.Bridge.Relay.AdminOnly && !ce.User.Admin { - ce.Reply("Only admins are allowed to enable relay mode on this instance of the bridge") + ce.Reply("Only bridge admins are allowed to enable relay mode on this instance of the bridge") } else { ce.Portal.RelayUserID = "" ce.Portal.Update(nil) @@ -240,18 +238,32 @@ func fnJoin(ce *WrappedCommandEvent) { if len(ce.Args) == 0 { ce.Reply("**Usage:** `join `") return - } else if !strings.HasPrefix(ce.Args[0], whatsmeow.InviteLinkPrefix) { - ce.Reply("That doesn't look like a WhatsApp invite link") - return } - jid, err := ce.User.Client.JoinGroupWithLink(ce.Args[0]) - if err != nil { - ce.Reply("Failed to join group: %v", err) - return + if strings.HasPrefix(ce.Args[0], whatsmeow.InviteLinkPrefix) { + jid, err := ce.User.Client.JoinGroupWithLink(ce.Args[0]) + if err != nil { + ce.Reply("Failed to join group: %v", err) + return + } + ce.Log.Debugln("%s successfully joined group %s", ce.User.MXID, jid) + ce.Reply("Successfully joined group `%s`, the portal should be created momentarily", jid) + } else if strings.HasPrefix(ce.Args[0], whatsmeow.NewsletterLinkPrefix) { + info, err := ce.User.Client.GetNewsletterInfoWithInvite(ce.Args[0]) + if err != nil { + ce.Reply("Failed to get channel info: %v", err) + return + } + err = ce.User.Client.FollowNewsletter(info.ID) + if err != nil { + ce.Reply("Failed to follow channel: %v", err) + return + } + ce.Log.Debugln("%s successfully followed channel %s", ce.User.MXID, info.ID) + ce.Reply("Successfully followed channel `%s`, the portal should be created momentarily", info.ID) + } else { + ce.Reply("That doesn't look like a WhatsApp invite link") } - ce.Log.Debugln("%s successfully joined group %s", ce.User.MXID, jid) - ce.Reply("Successfully joined group `%s`, the portal should be created momentarily", jid) } func tryDecryptEvent(crypto bridge.Crypto, evt *event.Event) (json.RawMessage, error) { @@ -382,7 +394,7 @@ func fnCreate(ce *WrappedCommandEvent) { } // TODO check m.space.parent to create rooms directly in communities - messageID := whatsmeow.GenerateMessageID() + messageID := ce.User.Client.GenerateMessageID() ce.Log.Infofln("Creating group for %s with name %s and participants %+v (create key: %s)", ce.RoomID, roomNameEvent.Name, participants, messageID) ce.User.createKeyDedup = messageID resp, err := ce.User.Client.CreateGroup(whatsmeow.ReqCreateGroup{ @@ -432,11 +444,16 @@ var cmdLogin = &commands.FullHandler{ Func: wrapCommand(fnLogin), Name: "login", Help: commands.HelpMeta{ - Section: commands.HelpSectionAuth, - Description: "Link the bridge to your WhatsApp account as a web client.", + Section: commands.HelpSectionAuth, + Description: "Link the bridge to your WhatsApp account as a web client. " + + "The phone number parameter is optional: if provided, the bridge will create a 8-character login code " + + "that can be used instead of the QR code.", + Args: "[_phone number_]", }, } +var looksLikeAPhoneRegex = regexp.MustCompile(`^\+[0-9]+$`) + func fnLogin(ce *WrappedCommandEvent) { if ce.User.Session != nil { if ce.User.IsConnected() { @@ -447,13 +464,33 @@ func fnLogin(ce *WrappedCommandEvent) { return } + var phoneNumber string + if len(ce.Args) > 0 { + phoneNumber = strings.TrimSpace(strings.Join(ce.Args, " ")) + if !looksLikeAPhoneRegex.MatchString(phoneNumber) { + ce.Reply("When specifying a phone number, it must be provided in international format without spaces or other extra characters") + return + } + } + qrChan, err := ce.User.Login(context.Background()) if err != nil { - ce.User.log.Errorf("Failed to log in:", err) + ce.ZLog.Err(err).Msg("Failed to start login") ce.Reply("Failed to log in: %v", err) return } + if phoneNumber != "" { + pairingCode, err := ce.User.Client.PairPhone(phoneNumber, true, whatsmeow.PairClientChrome, "Chrome (Linux)") + if err != nil { + ce.ZLog.Err(err).Msg("Failed to start phone code login") + ce.Reply("Failed to start phone code login: %v", err) + go ce.User.DeleteConnection() + return + } + ce.Reply("Scan the code below or enter the following code on your phone to log in: **%s**", pairingCode) + } + var qrEventID id.EventID for item := range qrChan { switch item.Event { @@ -461,7 +498,7 @@ func fnLogin(ce *WrappedCommandEvent) { jid := ce.User.Client.Store.ID ce.Reply("Successfully logged in as +%s (device #%d)", jid.User, jid.Device) case whatsmeow.QRChannelTimeout.Event: - ce.Reply("QR code timed out. Please restart the login.") + ce.Reply("Login timed out. Please restart the login.") case whatsmeow.QRChannelErrUnexpectedEvent.Event: ce.Reply("Failed to log in: unexpected connection event from server") case whatsmeow.QRChannelClientOutdated.Event: @@ -474,7 +511,9 @@ func fnLogin(ce *WrappedCommandEvent) { qrEventID = ce.User.sendQR(ce, item.Code, qrEventID) } } - _, _ = ce.Bot.RedactEvent(ce.RoomID, qrEventID) + if qrEventID != "" { + _, _ = ce.Bot.RedactEvent(ce.RoomID, qrEventID) + } } func (user *User) sendQR(ce *WrappedCommandEvent, code string, prevEvent id.EventID) id.EventID { @@ -536,12 +575,7 @@ func fnLogout(ce *WrappedCommandEvent) { return } puppet := ce.Bridge.GetPuppetByJID(ce.User.JID) - if puppet.CustomMXID != "" { - err := puppet.SwitchCustomMXID("", "") - if err != nil { - ce.User.log.Warnln("Failed to logout-matrix while logging out of WhatsApp:", err) - } - } + puppet.ClearCustomMXID() err := ce.User.Client.Logout() if err != nil { ce.User.log.Warnln("Error while logging out:", err) @@ -787,46 +821,6 @@ func fnDeleteAllPortals(ce *WrappedCommandEvent) { }() } -var cmdBackfill = &commands.FullHandler{ - Func: wrapCommand(fnBackfill), - Name: "backfill", - Help: commands.HelpMeta{ - Section: HelpSectionPortalManagement, - Description: "Backfill all messages the portal.", - Args: "[_batch size_] [_batch delay_]", - }, - RequiresPortal: true, -} - -func fnBackfill(ce *WrappedCommandEvent) { - if !ce.Bridge.Config.Bridge.HistorySync.Backfill { - ce.Reply("Backfill is not enabled for this bridge.") - return - } - batchSize := 100 - batchDelay := 5 - if len(ce.Args) >= 1 { - var err error - batchSize, err = strconv.Atoi(ce.Args[0]) - if err != nil || batchSize < 1 { - ce.Reply("\"%s\" isn't a valid batch size", ce.Args[0]) - return - } - } - if len(ce.Args) >= 2 { - var err error - batchDelay, err = strconv.Atoi(ce.Args[0]) - if err != nil || batchSize < 0 { - ce.Reply("\"%s\" isn't a valid batch delay", ce.Args[1]) - return - } - } - backfillMessages := ce.Portal.bridge.DB.Backfill.NewWithValues(ce.User.MXID, database.BackfillImmediate, 0, &ce.Portal.Key, nil, batchSize, -1, batchDelay) - backfillMessages.Insert() - - ce.User.BackfillQueue.ReCheck() -} - func matchesQuery(str string, query string) bool { if query == "" { return true @@ -1018,23 +1012,37 @@ func fnOpen(ce *WrappedCommandEvent) { } else { jid = types.NewJID(ce.Args[0], types.GroupServer) } - if jid.Server != types.GroupServer || (!strings.ContainsRune(jid.User, '-') && len(jid.User) < 15) { + if (jid.Server != types.GroupServer && jid.Server != types.NewsletterServer) || (!strings.ContainsRune(jid.User, '-') && len(jid.User) < 15) { ce.Reply("That does not look like a group JID") return } - info, err := ce.User.Client.GetGroupInfo(jid) - if err != nil { - ce.Reply("Failed to get group info: %v", err) - return + var err error + var groupInfo *types.GroupInfo + var newsletterMetadata *types.NewsletterMetadata + switch jid.Server { + case types.GroupServer: + groupInfo, err = ce.User.Client.GetGroupInfo(jid) + if err != nil { + ce.Reply("Failed to get group info: %v", err) + return + } + jid = groupInfo.JID + case types.NewsletterServer: + newsletterMetadata, err = ce.User.Client.GetNewsletterInfo(jid) + if err != nil { + ce.Reply("Failed to get channel info: %v", err) + return + } + jid = newsletterMetadata.ID } ce.Log.Debugln("Importing", jid, "for", ce.User.MXID) - portal := ce.User.GetPortalByJID(info.JID) + portal := ce.User.GetPortalByJID(jid) if len(portal.MXID) > 0 { - portal.UpdateMatrixRoom(ce.User, info) + portal.UpdateMatrixRoom(ce.User, groupInfo, newsletterMetadata) ce.Reply("Portal room synced.") } else { - err = portal.CreateMatrixRoom(ce.User, info, true, true) + err = portal.CreateMatrixRoom(ce.User, groupInfo, newsletterMetadata, true, true) if err != nil { ce.Reply("Failed to create room: %v", err) } else { diff --git a/config/bridge.go b/config/bridge.go index 6f3d75f..538908f 100644 --- a/config/bridge.go +++ b/config/bridge.go @@ -57,17 +57,16 @@ type BridgeConfig struct { IdentityChangeNotices bool `yaml:"identity_change_notices"` HistorySync struct { - CreatePortals bool `yaml:"create_portals"` - Backfill bool `yaml:"backfill"` + Backfill bool `yaml:"backfill"` - DoublePuppetBackfill bool `yaml:"double_puppet_backfill"` - RequestFullSync bool `yaml:"request_full_sync"` - FullSyncConfig struct { + RequestFullSync bool `yaml:"request_full_sync"` + FullSyncConfig struct { DaysLimit uint32 `yaml:"days_limit"` SizeLimit uint32 `yaml:"size_mb_limit"` StorageQuota uint32 `yaml:"storage_quota_mb"` } MaxInitialConversations int `yaml:"max_initial_conversations"` + MessageCount int `yaml:"message_count"` UnreadHoursThreshold int `yaml:"unread_hours_threshold"` Immediate struct { @@ -86,18 +85,14 @@ type BridgeConfig struct { UserAvatarSync bool `yaml:"user_avatar_sync"` BridgeMatrixLeave bool `yaml:"bridge_matrix_leave"` - SyncWithCustomPuppets bool `yaml:"sync_with_custom_puppets"` SyncDirectChatList bool `yaml:"sync_direct_chat_list"` SyncManualMarkedUnread bool `yaml:"sync_manual_marked_unread"` - DefaultBridgeReceipts bool `yaml:"default_bridge_receipts"` DefaultBridgePresence bool `yaml:"default_bridge_presence"` SendPresenceOnTyping bool `yaml:"send_presence_on_typing"` ForceActiveDeliveryReceipts bool `yaml:"force_active_delivery_receipts"` - DoublePuppetServerMap map[string]string `yaml:"double_puppet_server_map"` - DoublePuppetAllowDiscovery bool `yaml:"double_puppet_allow_discovery"` - LoginSharedSecretMap map[string]string `yaml:"login_shared_secret_map"` + DoublePuppetConfig bridgeconfig.DoublePuppetConfig `yaml:",inline"` PrivateChatPortalMeta string `yaml:"private_chat_portal_meta"` ParallelMemberSync bool `yaml:"parallel_member_sync"` @@ -116,6 +111,7 @@ type BridgeConfig struct { FederateRooms bool `yaml:"federate_rooms"` URLPreviews bool `yaml:"url_previews"` CaptionInMessage bool `yaml:"caption_in_message"` + BeeperGalleries bool `yaml:"beeper_galleries"` ExtEvPolls bool `yaml:"extev_polls"` CrossRoomReplies bool `yaml:"cross_room_replies"` DisableReplyFallbacks bool `yaml:"disable_reply_fallbacks"` @@ -140,8 +136,9 @@ type BridgeConfig struct { Encryption bridgeconfig.EncryptionConfig `yaml:"encryption"` Provisioning struct { - Prefix string `yaml:"prefix"` - SharedSecret string `yaml:"shared_secret"` + Prefix string `yaml:"prefix"` + SharedSecret string `yaml:"shared_secret"` + DebugEndpoints bool `yaml:"debug_endpoints"` } `yaml:"provisioning"` Permissions bridgeconfig.PermissionConfig `yaml:"permissions"` @@ -152,6 +149,10 @@ type BridgeConfig struct { displaynameTemplate *template.Template `yaml:"-"` } +func (bc BridgeConfig) GetDoublePuppetConfig() bridgeconfig.DoublePuppetConfig { + return bc.DoublePuppetConfig +} + func (bc BridgeConfig) GetEncryptionConfig() bridgeconfig.EncryptionConfig { return bc.Encryption } diff --git a/config/config.go b/config/config.go index 326f859..69dbd2c 100644 --- a/config/config.go +++ b/config/config.go @@ -24,8 +24,11 @@ import ( type Config struct { *bridgeconfig.BaseConfig `yaml:",inline"` - SegmentKey string `yaml:"segment_key"` - SegmentUserID string `yaml:"segment_user_id"` + Analytics struct { + Host string `yaml:"host"` + Token string `yaml:"token"` + UserID string `yaml:"user_id"` + } Metrics struct { Enabled bool `yaml:"enabled"` @@ -42,18 +45,6 @@ type Config struct { func (config *Config) CanAutoDoublePuppet(userID id.UserID) bool { _, homeserver, _ := userID.Parse() - _, hasSecret := config.Bridge.LoginSharedSecretMap[homeserver] + _, hasSecret := config.Bridge.DoublePuppetConfig.SharedSecretMap[homeserver] return hasSecret } - -func (config *Config) CanDoublePuppetBackfill(userID id.UserID) bool { - if !config.Bridge.HistorySync.DoublePuppetBackfill { - return false - } - _, homeserver, _ := userID.Parse() - // Batch sending can only use local users, so don't allow double puppets on other servers. - if homeserver != config.Homeserver.Domain && config.Homeserver.Software != bridgeconfig.SoftwareHungry { - return false - } - return true -} diff --git a/config/upgrade.go b/config/upgrade.go index d1f4e95..b1999c8 100644 --- a/config/upgrade.go +++ b/config/upgrade.go @@ -19,16 +19,17 @@ package config import ( "strings" + up "go.mau.fi/util/configupgrade" + "go.mau.fi/util/random" "maunium.net/go/mautrix/bridge/bridgeconfig" - "maunium.net/go/mautrix/util" - up "maunium.net/go/mautrix/util/configupgrade" ) func DoUpgrade(helper *up.Helper) { bridgeconfig.Upgrader.DoUpgrade(helper) - helper.Copy(up.Str|up.Null, "segment_key") - helper.Copy(up.Str|up.Null, "segment_user_id") + helper.Copy(up.Str|up.Null, "analytics", "host") + helper.Copy(up.Str|up.Null, "analytics", "token") + helper.Copy(up.Str|up.Null, "analytics", "user_id") helper.Copy(up.Bool, "metrics", "enabled") helper.Copy(up.Str, "metrics", "listen") @@ -45,9 +46,7 @@ func DoUpgrade(helper *up.Helper) { helper.Copy(up.Int, "bridge", "portal_message_buffer") helper.Copy(up.Bool, "bridge", "call_start_notices") helper.Copy(up.Bool, "bridge", "identity_change_notices") - helper.Copy(up.Bool, "bridge", "history_sync", "create_portals") helper.Copy(up.Bool, "bridge", "history_sync", "backfill") - helper.Copy(up.Bool, "bridge", "history_sync", "double_puppet_backfill") helper.Copy(up.Bool, "bridge", "history_sync", "request_full_sync") helper.Copy(up.Int|up.Null, "bridge", "history_sync", "full_sync_config", "days_limit") helper.Copy(up.Int|up.Null, "bridge", "history_sync", "full_sync_config", "size_mb_limit") @@ -56,15 +55,14 @@ func DoUpgrade(helper *up.Helper) { helper.Copy(up.Str, "bridge", "history_sync", "media_requests", "request_method") helper.Copy(up.Int, "bridge", "history_sync", "media_requests", "request_local_time") helper.Copy(up.Int, "bridge", "history_sync", "max_initial_conversations") + helper.Copy(up.Int, "bridge", "history_sync", "message_count") helper.Copy(up.Int, "bridge", "history_sync", "unread_hours_threshold") helper.Copy(up.Int, "bridge", "history_sync", "immediate", "worker_count") helper.Copy(up.Int, "bridge", "history_sync", "immediate", "max_events") helper.Copy(up.List, "bridge", "history_sync", "deferred") helper.Copy(up.Bool, "bridge", "user_avatar_sync") helper.Copy(up.Bool, "bridge", "bridge_matrix_leave") - helper.Copy(up.Bool, "bridge", "sync_with_custom_puppets") helper.Copy(up.Bool, "bridge", "sync_direct_chat_list") - helper.Copy(up.Bool, "bridge", "default_bridge_receipts") helper.Copy(up.Bool, "bridge", "default_bridge_presence") helper.Copy(up.Bool, "bridge", "send_presence_on_typing") helper.Copy(up.Bool, "bridge", "force_active_delivery_receipts") @@ -105,6 +103,7 @@ func DoUpgrade(helper *up.Helper) { helper.Copy(up.Bool, "bridge", "crash_on_stream_replaced") helper.Copy(up.Bool, "bridge", "url_previews") helper.Copy(up.Bool, "bridge", "caption_in_message") + helper.Copy(up.Bool, "bridge", "beeper_galleries") if intPolls, ok := helper.Get(up.Int, "bridge", "extev_polls"); ok { val := "false" if intPolls != "0" { @@ -135,6 +134,7 @@ func DoUpgrade(helper *up.Helper) { helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "delete_prev_on_new_session") helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "delete_on_device_delete") helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "periodically_delete_expired") + helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "delete_outdated_inbound") helper.Copy(up.Str, "bridge", "encryption", "verification_levels", "receive") helper.Copy(up.Str, "bridge", "encryption", "verification_levels", "send") helper.Copy(up.Str, "bridge", "encryption", "verification_levels", "share") @@ -160,10 +160,11 @@ func DoUpgrade(helper *up.Helper) { } else { helper.Copy(up.Str, "bridge", "provisioning", "prefix") } + helper.Copy(up.Bool, "bridge", "provisioning", "debug_endpoints") if secret, ok := helper.Get(up.Str, "appservice", "provisioning", "shared_secret"); ok && secret != "generate" { helper.Set(up.Str, secret, "bridge", "provisioning", "shared_secret") } else if secret, ok = helper.Get(up.Str, "bridge", "provisioning", "shared_secret"); !ok || secret == "generate" { - sharedSecret := util.RandomString(64) + sharedSecret := random.String(64) helper.Set(up.Str, sharedSecret, "bridge", "provisioning", "shared_secret") } else { helper.Copy(up.Str, "bridge", "provisioning", "shared_secret") @@ -181,7 +182,7 @@ var SpacedBlocks = [][]string{ {"appservice", "database"}, {"appservice", "id"}, {"appservice", "as_token"}, - {"segment_key"}, + {"analytics"}, {"metrics"}, {"whatsapp"}, {"bridge"}, diff --git a/custompuppet.go b/custompuppet.go index 8fe6099..47ae104 100644 --- a/custompuppet.go +++ b/custompuppet.go @@ -17,262 +17,74 @@ package main import ( - "crypto/hmac" - "crypto/sha512" - "encoding/hex" - "errors" - "fmt" - "time" - - "maunium.net/go/mautrix" - "maunium.net/go/mautrix/appservice" - "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" ) -var ( - ErrNoCustomMXID = errors.New("no custom mxid set") - ErrMismatchingMXID = errors.New("whoami result does not match custom mxid") -) - func (puppet *Puppet) SwitchCustomMXID(accessToken string, mxid id.UserID) error { - prevCustomMXID := puppet.CustomMXID - if puppet.customIntent != nil { - puppet.stopSyncing() - } puppet.CustomMXID = mxid puppet.AccessToken = accessToken - + puppet.EnablePresence = puppet.bridge.Config.Bridge.DefaultBridgePresence + puppet.Update() err := puppet.StartCustomMXID(false) if err != nil { return err } - - if len(prevCustomMXID) > 0 { - delete(puppet.bridge.puppetsByCustomMXID, prevCustomMXID) - } - if len(puppet.CustomMXID) > 0 { - puppet.bridge.puppetsByCustomMXID[puppet.CustomMXID] = puppet - } - puppet.EnablePresence = puppet.bridge.Config.Bridge.DefaultBridgePresence - puppet.EnableReceipts = puppet.bridge.Config.Bridge.DefaultBridgeReceipts - puppet.bridge.AS.StateStore.MarkRegistered(puppet.CustomMXID) - puppet.Update() // TODO leave rooms with default puppet return nil } -func (puppet *Puppet) loginWithSharedSecret(mxid id.UserID) (string, error) { - _, homeserver, _ := mxid.Parse() - puppet.log.Debugfln("Logging into %s with shared secret", mxid) - loginSecret := puppet.bridge.Config.Bridge.LoginSharedSecretMap[homeserver] - client, err := puppet.bridge.newDoublePuppetClient(mxid, "") - if err != nil { - return "", fmt.Errorf("failed to create mautrix client to log in: %v", err) +func (puppet *Puppet) ClearCustomMXID() { + save := puppet.CustomMXID != "" || puppet.AccessToken != "" + puppet.bridge.puppetsLock.Lock() + if puppet.CustomMXID != "" && puppet.bridge.puppetsByCustomMXID[puppet.CustomMXID] == puppet { + delete(puppet.bridge.puppetsByCustomMXID, puppet.CustomMXID) } - req := mautrix.ReqLogin{ - Identifier: mautrix.UserIdentifier{Type: mautrix.IdentifierTypeUser, User: string(mxid)}, - DeviceID: "WhatsApp Bridge", - InitialDeviceDisplayName: "WhatsApp Bridge", - } - if loginSecret == "appservice" { - client.AccessToken = puppet.bridge.AS.Registration.AppToken - req.Type = mautrix.AuthTypeAppservice - } else { - mac := hmac.New(sha512.New, []byte(loginSecret)) - mac.Write([]byte(mxid)) - req.Password = hex.EncodeToString(mac.Sum(nil)) - req.Type = mautrix.AuthTypePassword - } - resp, err := client.Login(&req) - if err != nil { - return "", err - } - return resp.AccessToken, nil -} - -func (br *WABridge) newDoublePuppetClient(mxid id.UserID, accessToken string) (*mautrix.Client, error) { - _, homeserver, err := mxid.Parse() - if err != nil { - return nil, err - } - homeserverURL, found := br.Config.Bridge.DoublePuppetServerMap[homeserver] - if !found { - if homeserver == br.AS.HomeserverDomain { - homeserverURL = "" - } else if br.Config.Bridge.DoublePuppetAllowDiscovery { - resp, err := mautrix.DiscoverClientAPI(homeserver) - if err != nil { - return nil, fmt.Errorf("failed to find homeserver URL for %s: %v", homeserver, err) - } - homeserverURL = resp.Homeserver.BaseURL - br.Log.Debugfln("Discovered URL %s for %s to enable double puppeting for %s", homeserverURL, homeserver, mxid) - } else { - return nil, fmt.Errorf("double puppeting from %s is not allowed", homeserver) - } - } - return br.AS.NewExternalMautrixClient(mxid, accessToken, homeserverURL) -} - -func (puppet *Puppet) newCustomIntent() (*appservice.IntentAPI, error) { - if len(puppet.CustomMXID) == 0 { - return nil, ErrNoCustomMXID - } - client, err := puppet.bridge.newDoublePuppetClient(puppet.CustomMXID, puppet.AccessToken) - if err != nil { - return nil, err - } - client.Syncer = puppet - client.Store = puppet - - ia := puppet.bridge.AS.NewIntentAPI("custom") - ia.Client = client - ia.Localpart, _, _ = puppet.CustomMXID.Parse() - ia.UserID = puppet.CustomMXID - ia.IsCustomPuppet = true - return ia, nil -} - -func (puppet *Puppet) clearCustomMXID() { + puppet.bridge.puppetsLock.Unlock() puppet.CustomMXID = "" puppet.AccessToken = "" puppet.customIntent = nil puppet.customUser = nil + if save { + puppet.Update() + } } func (puppet *Puppet) StartCustomMXID(reloginOnFail bool) error { - if len(puppet.CustomMXID) == 0 { - puppet.clearCustomMXID() - return nil - } - intent, err := puppet.newCustomIntent() + newIntent, newAccessToken, err := puppet.bridge.DoublePuppet.Setup(puppet.CustomMXID, puppet.AccessToken, reloginOnFail) if err != nil { - puppet.clearCustomMXID() + puppet.ClearCustomMXID() return err } - resp, err := intent.Whoami() - if err != nil { - if !reloginOnFail || (errors.Is(err, mautrix.MUnknownToken) && !puppet.tryRelogin(err, "initializing double puppeting")) { - puppet.clearCustomMXID() - return err - } - intent.AccessToken = puppet.AccessToken - } else if resp.UserID != puppet.CustomMXID { - puppet.clearCustomMXID() - return ErrMismatchingMXID + puppet.bridge.puppetsLock.Lock() + puppet.bridge.puppetsByCustomMXID[puppet.CustomMXID] = puppet + puppet.bridge.puppetsLock.Unlock() + if puppet.AccessToken != newAccessToken { + puppet.AccessToken = newAccessToken + puppet.Update() } - puppet.customIntent = intent + puppet.customIntent = newIntent puppet.customUser = puppet.bridge.GetUserByMXID(puppet.CustomMXID) - puppet.startSyncing() return nil } -func (puppet *Puppet) startSyncing() { - if !puppet.bridge.Config.Bridge.SyncWithCustomPuppets { +func (user *User) tryAutomaticDoublePuppeting() { + if !user.bridge.Config.CanAutoDoublePuppet(user.MXID) { return } - go func() { - puppet.log.Debugln("Starting syncing...") - puppet.customIntent.SyncPresence = "offline" - err := puppet.customIntent.Sync() - if err != nil { - puppet.log.Errorln("Fatal error syncing:", err) - } - }() -} - -func (puppet *Puppet) stopSyncing() { - if !puppet.bridge.Config.Bridge.SyncWithCustomPuppets { + user.zlog.Debug().Msg("Checking if double puppeting needs to be enabled") + puppet := user.bridge.GetPuppetByJID(user.JID) + if len(puppet.CustomMXID) > 0 { + user.zlog.Debug().Msg("User already has double-puppeting enabled") + // Custom puppet already enabled return } - puppet.customIntent.StopSync() -} - -func (puppet *Puppet) ProcessResponse(resp *mautrix.RespSync, _ string) error { - if !puppet.customUser.IsLoggedIn() { - puppet.log.Debugln("Skipping sync processing: custom user not connected to whatsapp") - return nil - } - for roomID, events := range resp.Rooms.Join { - for _, evt := range events.Ephemeral.Events { - evt.RoomID = roomID - err := evt.Content.ParseRaw(evt.Type) - if err != nil { - continue - } - switch evt.Type { - case event.EphemeralEventReceipt: - if puppet.EnableReceipts { - go puppet.bridge.MatrixHandler.HandleReceipt(evt) - } - case event.EphemeralEventTyping: - go puppet.bridge.MatrixHandler.HandleTyping(evt) - } - } - } - if puppet.EnablePresence { - for _, evt := range resp.Presence.Events { - if evt.Sender != puppet.CustomMXID { - continue - } - err := evt.Content.ParseRaw(evt.Type) - if err != nil { - continue - } - go puppet.bridge.HandlePresence(evt) - } - } - return nil -} - -func (puppet *Puppet) tryRelogin(cause error, action string) bool { - if !puppet.bridge.Config.CanAutoDoublePuppet(puppet.CustomMXID) { - return false - } - puppet.log.Debugfln("Trying to relogin after '%v' while %s", cause, action) - accessToken, err := puppet.loginWithSharedSecret(puppet.CustomMXID) + puppet.CustomMXID = user.MXID + puppet.EnablePresence = puppet.bridge.Config.Bridge.DefaultBridgePresence + err := puppet.StartCustomMXID(true) if err != nil { - puppet.log.Errorfln("Failed to relogin after '%v' while %s: %v", cause, action, err) - return false - } - puppet.log.Infofln("Successfully relogined after '%v' while %s", cause, action) - puppet.AccessToken = accessToken - return true -} - -func (puppet *Puppet) OnFailedSync(_ *mautrix.RespSync, err error) (time.Duration, error) { - puppet.log.Warnln("Sync error:", err) - if errors.Is(err, mautrix.MUnknownToken) { - if !puppet.tryRelogin(err, "syncing") { - return 0, err - } - puppet.customIntent.AccessToken = puppet.AccessToken - return 0, nil - } - return 10 * time.Second, nil -} - -func (puppet *Puppet) GetFilterJSON(_ id.UserID) *mautrix.Filter { - everything := []event.Type{{Type: "*"}} - return &mautrix.Filter{ - Presence: mautrix.FilterPart{ - Senders: []id.UserID{puppet.CustomMXID}, - Types: []event.Type{event.EphemeralEventPresence}, - }, - AccountData: mautrix.FilterPart{NotTypes: everything}, - Room: mautrix.RoomFilter{ - Ephemeral: mautrix.FilterPart{Types: []event.Type{event.EphemeralEventTyping, event.EphemeralEventReceipt}}, - IncludeLeave: false, - AccountData: mautrix.FilterPart{NotTypes: everything}, - State: mautrix.FilterPart{NotTypes: everything}, - Timeline: mautrix.FilterPart{NotTypes: everything}, - }, + user.zlog.Warn().Err(err).Msg("Failed to login with shared secret") + } else { + // TODO leave rooms with default puppet + user.zlog.Debug().Msg("Successfully automatically enabled custom puppet") } } - -func (puppet *Puppet) SaveFilterID(_ id.UserID, _ string) {} -func (puppet *Puppet) SaveNextBatch(_ id.UserID, nbt string) { puppet.NextBatch = nbt; puppet.Update() } -func (puppet *Puppet) SaveRoom(_ *mautrix.Room) {} -func (puppet *Puppet) LoadFilterID(_ id.UserID) string { return "" } -func (puppet *Puppet) LoadNextBatch(_ id.UserID) string { return puppet.NextBatch } -func (puppet *Puppet) LoadRoom(_ id.RoomID) *mautrix.Room { return nil } diff --git a/database/backfill.go b/database/backfill.go index 6925382..273370b 100644 --- a/database/backfill.go +++ b/database/backfill.go @@ -25,10 +25,10 @@ import ( "sync" "time" + "go.mau.fi/util/dbutil" log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" ) type BackfillType int diff --git a/database/database.go b/database/database.go index 7f519e7..4bc749a 100644 --- a/database/database.go +++ b/database/database.go @@ -23,10 +23,10 @@ import ( "github.com/lib/pq" _ "github.com/mattn/go-sqlite3" + "go.mau.fi/util/dbutil" "go.mau.fi/whatsmeow/store" "go.mau.fi/whatsmeow/store/sqlstore" "maunium.net/go/maulogger/v2" - "maunium.net/go/mautrix/util/dbutil" "maunium.net/go/mautrix-whatsapp/database/upgrades" ) diff --git a/database/disappearingmessage.go b/database/disappearingmessage.go index 7489381..bae11e0 100644 --- a/database/disappearingmessage.go +++ b/database/disappearingmessage.go @@ -21,10 +21,10 @@ import ( "errors" "time" + "go.mau.fi/util/dbutil" log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" ) type DisappearingMessageQuery struct { diff --git a/database/historysync.go b/database/historysync.go index 214440b..d896bbc 100644 --- a/database/historysync.go +++ b/database/historysync.go @@ -22,15 +22,13 @@ import ( "fmt" "time" - "google.golang.org/protobuf/proto" - - waProto "go.mau.fi/whatsmeow/binary/proto" - _ "github.com/mattn/go-sqlite3" + "go.mau.fi/util/dbutil" + waProto "go.mau.fi/whatsmeow/binary/proto" + "google.golang.org/protobuf/proto" log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" ) type HistorySyncQuery struct { @@ -166,7 +164,7 @@ func (hsc *HistorySyncConversation) Scan(row dbutil.Scannable) *HistorySyncConve return hsc } -func (hsq *HistorySyncQuery) GetNMostRecentConversations(userID id.UserID, n int) (conversations []*HistorySyncConversation) { +func (hsq *HistorySyncQuery) GetRecentConversations(userID id.UserID, n int) (conversations []*HistorySyncConversation) { nPtr := &n // Negative limit on SQLite means unlimited, but Postgres prefers a NULL limit. if n < 0 && hsq.db.Dialect == dbutil.Postgres { @@ -183,7 +181,7 @@ func (hsq *HistorySyncQuery) GetNMostRecentConversations(userID id.UserID, n int return } -func (hsq *HistorySyncQuery) GetConversation(userID id.UserID, portalKey *PortalKey) (conversation *HistorySyncConversation) { +func (hsq *HistorySyncQuery) GetConversation(userID id.UserID, portalKey PortalKey) (conversation *HistorySyncConversation) { rows, err := hsq.db.Query(getConversationByPortal, userID, portalKey.JID, portalKey.Receiver) defer rows.Close() if err != nil || rows == nil { @@ -323,3 +321,27 @@ func (hsq *HistorySyncQuery) DeleteAllMessagesForPortal(userID id.UserID, portal hsq.log.Warnfln("Failed to delete historical messages for %s/%s: %v", userID, portalKey.JID, err) } } + +func (hsq *HistorySyncQuery) ConversationHasMessages(userID id.UserID, portalKey PortalKey) (exists bool) { + err := hsq.db.QueryRow(` + SELECT EXISTS( + SELECT 1 FROM history_sync_message + WHERE user_mxid=$1 AND conversation_id=$2 + ) + `, userID, portalKey.JID).Scan(&exists) + if err != nil { + hsq.log.Warnfln("Failed to check if any messages are stored for %s/%s: %v", userID, portalKey.JID, err) + } + return +} + +func (hsq *HistorySyncQuery) DeleteConversation(userID id.UserID, jid string) { + // This will also clear history_sync_message as there's a foreign key constraint + _, err := hsq.db.Exec(` + DELETE FROM history_sync_conversation + WHERE user_mxid=$1 AND conversation_id=$2 + `, userID, jid) + if err != nil { + hsq.log.Warnfln("Failed to delete historical messages for %s/%s: %v", userID, jid, err) + } +} diff --git a/database/mediabackfillrequest.go b/database/mediabackfillrequest.go index c206796..e71b986 100644 --- a/database/mediabackfillrequest.go +++ b/database/mediabackfillrequest.go @@ -21,10 +21,10 @@ import ( "errors" _ "github.com/mattn/go-sqlite3" + "go.mau.fi/util/dbutil" log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" ) type MediaBackfillRequestStatus int diff --git a/database/message.go b/database/message.go index ffe33c1..dbd7e9c 100644 --- a/database/message.go +++ b/database/message.go @@ -19,15 +19,15 @@ package database import ( "database/sql" "errors" + "fmt" "strings" "time" + "go.mau.fi/util/dbutil" + "go.mau.fi/whatsmeow/types" log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" - - "go.mau.fi/whatsmeow/types" ) type MessageQuery struct { @@ -134,12 +134,13 @@ const ( type MessageType string const ( - MsgUnknown MessageType = "" - MsgFake MessageType = "fake" - MsgNormal MessageType = "message" - MsgReaction MessageType = "reaction" - MsgEdit MessageType = "edit" - MsgMatrixPoll MessageType = "matrix-poll" + MsgUnknown MessageType = "" + MsgFake MessageType = "fake" + MsgNormal MessageType = "message" + MsgReaction MessageType = "reaction" + MsgEdit MessageType = "edit" + MsgMatrixPoll MessageType = "matrix-poll" + MsgBeeperGallery MessageType = "beeper-gallery" ) type Message struct { @@ -156,6 +157,8 @@ type Message struct { Type MessageType Error MessageErrorType + GalleryPart int + BroadcastListJID types.JID } @@ -167,6 +170,8 @@ func (msg *Message) IsFakeJID() bool { return strings.HasPrefix(msg.JID, "FAKE::") || msg.JID == string(msg.MXID) } +const fakeGalleryMXIDFormat = "com.beeper.gallery::%d:%s" + func (msg *Message) Scan(row dbutil.Scannable) *Message { var ts int64 err := row.Scan(&msg.Chat.JID, &msg.Chat.Receiver, &msg.JID, &msg.MXID, &msg.Sender, &msg.SenderMXID, &ts, &msg.Sent, &msg.Type, &msg.Error, &msg.BroadcastListJID) @@ -176,6 +181,12 @@ func (msg *Message) Scan(row dbutil.Scannable) *Message { } return nil } + if strings.HasPrefix(msg.MXID.String(), "com.beeper.gallery::") { + _, err = fmt.Sscanf(msg.MXID.String(), fakeGalleryMXIDFormat, &msg.GalleryPart, &msg.MXID) + if err != nil { + msg.log.Errorln("Parsing gallery MXID failed:", err) + } + } if ts != 0 { msg.Timestamp = time.Unix(ts, 0) } @@ -191,11 +202,15 @@ func (msg *Message) Insert(txn dbutil.Execable) { if msg.Sender.IsEmpty() { sender = "" } + mxid := msg.MXID.String() + if msg.GalleryPart != 0 { + mxid = fmt.Sprintf(fakeGalleryMXIDFormat, msg.GalleryPart, mxid) + } _, err := txn.Exec(` INSERT INTO message (chat_jid, chat_receiver, jid, mxid, sender, sender_mxid, timestamp, sent, type, error, broadcast_list_jid) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) - `, msg.Chat.JID, msg.Chat.Receiver, msg.JID, msg.MXID, sender, msg.SenderMXID, msg.Timestamp.Unix(), msg.Sent, msg.Type, msg.Error, msg.BroadcastListJID) + `, msg.Chat.JID, msg.Chat.Receiver, msg.JID, mxid, sender, msg.SenderMXID, msg.Timestamp.Unix(), msg.Sent, msg.Type, msg.Error, msg.BroadcastListJID) if err != nil { msg.log.Warnfln("Failed to insert %s@%s: %v", msg.Chat, msg.JID, err) } diff --git a/database/polloption.go b/database/polloption.go index aedcd53..4af576f 100644 --- a/database/polloption.go +++ b/database/polloption.go @@ -21,7 +21,7 @@ import ( "strings" "github.com/lib/pq" - "maunium.net/go/mautrix/util/dbutil" + "go.mau.fi/util/dbutil" ) func scanPollOptionMapping(rows dbutil.Rows) (id string, hashArr [32]byte, err error) { diff --git a/database/portal.go b/database/portal.go index 304d535..1b3eb00 100644 --- a/database/portal.go +++ b/database/portal.go @@ -21,12 +21,11 @@ import ( "fmt" "time" + "go.mau.fi/util/dbutil" + "go.mau.fi/whatsmeow/types" log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" - - "go.mau.fi/whatsmeow/types" ) type PortalKey struct { @@ -35,7 +34,7 @@ type PortalKey struct { } func NewPortalKey(jid, receiver types.JID) PortalKey { - if jid.Server == types.GroupServer { + if jid.Server == types.GroupServer || jid.Server == types.NewsletterServer { receiver = jid } else if jid.Server == types.LegacyUserServer { jid.Server = types.DefaultUserServer diff --git a/database/puppet.go b/database/puppet.go index 17d7996..c490e7e 100644 --- a/database/puppet.go +++ b/database/puppet.go @@ -20,12 +20,11 @@ import ( "database/sql" "time" + "go.mau.fi/util/dbutil" + "go.mau.fi/whatsmeow/types" log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" - - "go.mau.fi/whatsmeow/types" ) type PuppetQuery struct { diff --git a/database/reaction.go b/database/reaction.go index f30f263..1769358 100644 --- a/database/reaction.go +++ b/database/reaction.go @@ -20,12 +20,11 @@ import ( "database/sql" "errors" + "go.mau.fi/util/dbutil" + "go.mau.fi/whatsmeow/types" log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" - - "go.mau.fi/whatsmeow/types" ) type ReactionQuery struct { diff --git a/database/upgrades/00-latest-revision.sql b/database/upgrades/00-latest-revision.sql index 4e1b5ea..dd799f1 100644 --- a/database/upgrades/00-latest-revision.sql +++ b/database/upgrades/00-latest-revision.sql @@ -1,4 +1,4 @@ --- v0 -> v56 (compatible with v45+): Latest revision +-- v0 -> v57 (compatible with v45+): Latest revision CREATE TABLE "user" ( mxid TEXT PRIMARY KEY, @@ -82,6 +82,8 @@ CREATE TABLE message ( FOREIGN KEY (chat_jid, chat_receiver) REFERENCES portal(jid, receiver) ON DELETE CASCADE ); +CREATE INDEX message_timestamp_idx ON message (chat_jid, chat_receiver, timestamp); + CREATE TABLE poll_option_id ( msg_mxid TEXT, opt_id TEXT, diff --git a/database/upgrades/57-message-timestamp-index.sql b/database/upgrades/57-message-timestamp-index.sql new file mode 100644 index 0000000..c6ebe13 --- /dev/null +++ b/database/upgrades/57-message-timestamp-index.sql @@ -0,0 +1,2 @@ +-- v57 (compatible with v45+): Add index for message timestamp to make read receipt handling faster +CREATE INDEX message_timestamp_idx ON message (chat_jid, chat_receiver, timestamp); diff --git a/database/upgrades/upgrades.go b/database/upgrades/upgrades.go index 00ac6bc..990f496 100644 --- a/database/upgrades/upgrades.go +++ b/database/upgrades/upgrades.go @@ -20,7 +20,7 @@ import ( "embed" "errors" - "maunium.net/go/mautrix/util/dbutil" + "go.mau.fi/util/dbutil" ) var Table dbutil.UpgradeTable diff --git a/database/user.go b/database/user.go index 8b77850..a4ebc35 100644 --- a/database/user.go +++ b/database/user.go @@ -21,12 +21,11 @@ import ( "sync" "time" + "go.mau.fi/util/dbutil" + "go.mau.fi/whatsmeow/types" log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" - - "go.mau.fi/whatsmeow/types" ) type UserQuery struct { @@ -123,14 +122,16 @@ func (user *User) usernamePtr() *string { func (user *User) agentPtr() *uint8 { if !user.JID.IsEmpty() { - return &user.JID.Agent + zero := uint8(0) + return &zero } return nil } func (user *User) devicePtr() *uint8 { if !user.JID.IsEmpty() { - return &user.JID.Device + device := uint8(user.JID.Device) + return &device } return nil } diff --git a/disappear.go b/disappear.go index c90fc05..c315bc2 100644 --- a/disappear.go +++ b/disappear.go @@ -20,9 +20,10 @@ import ( "fmt" "time" + "go.mau.fi/util/dbutil" + "maunium.net/go/mautrix" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" "maunium.net/go/mautrix-whatsapp/database" ) diff --git a/example-config.yaml b/example-config.yaml index a9fe4ac..058f3e1 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -65,7 +65,6 @@ appservice: # Whether or not to receive ephemeral events via appservice transactions. # Requires MSC2409 support (i.e. Synapse 1.22+). - # You should disable bridge -> sync_with_custom_puppets when this is enabled. ephemeral_events: true # Should incoming events be handled asynchronously? @@ -77,10 +76,14 @@ appservice: as_token: "This value is generated when generating the registration" hs_token: "This value is generated when generating the registration" -# Segment API key to track some events, like provisioning API login and encryption errors. -segment_key: null -# Optional user_id to use when sending Segment events. If null, defaults to using mxID. -segment_user_id: null +# Segment-compatible analytics endpoint for tracking some events, like provisioning API login and encryption errors. +analytics: + # Hostname of the tracking server. The path is hardcoded to /v1/track + host: api.segment.io + # API key to send with tracking requests. Tracking is disabled if this is null. + token: null + # Optional user ID for tracking events. If null, defaults to using Matrix user ID. + user_id: null # Prometheus config. metrics: @@ -95,7 +98,7 @@ whatsapp: os_name: Mautrix-WhatsApp bridge # Browser name that determines the logo shown in the mobile app. # Must be "unknown" for a generic icon or a valid browser name if you want a specific icon. - # List of valid browser names: https://github.com/tulir/whatsmeow/blob/8b34d886d543b72e5f4699cf5b2797f68d598f78/binary/proto/def.proto#L38-L51 + # List of valid browser names: https://github.com/tulir/whatsmeow/blob/efc632c008604016ddde63bfcfca8de4e5304da9/binary/proto/def.proto#L43-L64 browser_name: unknown # Bridge config @@ -127,20 +130,14 @@ bridge: portal_message_buffer: 128 # Settings for handling history sync payloads. history_sync: - # Enable backfilling history sync payloads from WhatsApp using batch sending? - # This requires a server with MSC2716 support, which is currently an experimental feature in synapse. - # It can be enabled by setting experimental_features -> msc2716_enabled to true in homeserver.yaml. - # Note that prior to Synapse 1.49, there were some bugs with the implementation, especially if using event persistence workers. - # There are also still some issues in Synapse's federation implementation. - backfill: false - # Should the bridge create portals for chats in the history sync payload? - # This has no effect unless backfill is enabled. - create_portals: true - # Use double puppets for backfilling? - # In order to use this, the double puppets must be in the appservice's user ID namespace - # (because the bridge can't use the double puppet access token with batch sending). - # This only affects double puppets on the local server, double puppets on other servers will never be used. - double_puppet_backfill: false + # Enable backfilling history sync payloads from WhatsApp? + backfill: true + # The maximum number of initial conversations that should be synced. + # Other conversations will be backfilled on demand when receiving a message or when initiating a direct chat. + max_initial_conversations: -1 + # Maximum number of messages to backfill in each conversation. + # Set to -1 to disable limit. + message_count: 50 # Should the bridge request a full sync from the phone when logging in? # This bumps the size of history syncs from 3 months to 1 year. request_full_sync: false @@ -154,51 +151,43 @@ bridge: size_mb_limit: null # This is presumably the local storage quota, which may affect what the phone includes in the history sync blob. storage_quota_mb: null - # Settings for media requests. If the media expired, then it will not - # be on the WA servers. - # Media can always be requested by reacting with the ♻️ (recycle) emoji. - # These settings determine if the media requests should be done - # automatically during or after backfill. - media_requests: - # Should expired media be automatically requested from the server as - # part of the backfill process? - auto_request_media: true - # Whether to request the media immediately after the media message - # is backfilled ("immediate") or at a specific time of the day - # ("local_time"). - request_method: immediate - # If request_method is "local_time", what time should the requests - # be sent (in minutes after midnight)? - request_local_time: 120 - # The maximum number of initial conversations that should be synced. - # Other conversations will be backfilled on demand when the start PM - # provisioning endpoint is used or when a message comes in from that - # chat. - max_initial_conversations: -1 - # If this value is greater than 0, then if the conversation's last - # message was more than this number of hours ago, then the conversation - # will automatically be marked it as read. - # Conversations that have a last message that is less than this number - # of hours ago will have their unread status synced from WhatsApp. + # If this value is greater than 0, then if the conversation's last message was more than + # this number of hours ago, then the conversation will automatically be marked it as read. + # Conversations that have a last message that is less than this number of hours ago will + # have their unread status synced from WhatsApp. unread_hours_threshold: 0 - # Settings for immediate backfills. These backfills should generally be - # small and their main purpose is to populate each of the initial chats - # (as configured by max_initial_conversations) with a few messages so - # that you can continue conversations without loosing context. + + ############################################################################### + # The settings below are only applicable for backfilling using batch sending, # + # which is no longer supported in Synapse. # + ############################################################################### + + # Settings for media requests. If the media expired, then it will not be on the WA servers. + # Media can always be requested by reacting with the ♻️ (recycle) emoji. + # These settings determine if the media requests should be done automatically during or after backfill. + media_requests: + # Should expired media be automatically requested from the server as part of the backfill process? + auto_request_media: true + # Whether to request the media immediately after the media message is backfilled ("immediate") + # or at a specific time of the day ("local_time"). + request_method: immediate + # If request_method is "local_time", what time should the requests be sent (in minutes after midnight)? + request_local_time: 120 + # Settings for immediate backfills. These backfills should generally be small and their main purpose is + # to populate each of the initial chats (as configured by max_initial_conversations) with a few messages + # so that you can continue conversations without losing context. immediate: - # The number of concurrent backfill workers to create for immediate - # backfills. Note that using more than one worker could cause the - # room list to jump around since there are no guarantees about the - # order in which the backfills will complete. + # The number of concurrent backfill workers to create for immediate backfills. + # Note that using more than one worker could cause the room list to jump around + # since there are no guarantees about the order in which the backfills will complete. worker_count: 1 # The maximum number of events to backfill initially. max_events: 10 - # Settings for deferred backfills. The purpose of these backfills are - # to fill in the rest of the chat history that was not covered by the - # immediate backfills. These backfills generally should happen at a - # slower pace so as not to overload the homeserver. - # Each deferred backfill config should define a "stage" of backfill - # (i.e. the last week of messages). The fields are as follows: + # Settings for deferred backfills. The purpose of these backfills are to fill in the rest of + # the chat history that was not covered by the immediate backfills. + # These backfills generally should happen at a slower pace so as not to overload the homeserver. + # Each deferred backfill config should define a "stage" of backfill (i.e. the last week of messages). + # The fields are as follows: # - start_days_ago: the number of days ago to start backfilling from. # To indicate the start of time, use -1. For example, for a week ago, use 7. # - max_batch_events: the number of events to send per batch. @@ -220,12 +209,11 @@ bridge: - start_days_ago: -1 max_batch_events: 500 batch_delay: 10 + # Should puppet avatars be fetched from the server even if an avatar is already set? user_avatar_sync: true # Should Matrix users leaving groups be bridged to WhatsApp? bridge_matrix_leave: true - # Should the bridge sync with double puppeting to receive EDUs that aren't normally sent to appservices. - sync_with_custom_puppets: false # Should the bridge update the m.direct account data event when double puppeting is enabled. # Note that updating the m.direct event is not atomic (except with mautrix-asmux) # and is therefore prone to race conditions. @@ -236,9 +224,8 @@ bridge: # com.famedly.marked_unread room account data. sync_manual_marked_unread: true # When double puppeting is enabled, users can use `!wa toggle` to change whether - # presence and read receipts are bridged. These settings set the default values. + # presence is bridged. This setting sets the default value. # Existing users won't be affected when these are changed. - default_bridge_receipts: true default_bridge_presence: true # Send the presence as "available" to whatsapp when users start typing on a portal. # This works as a workaround for homeservers that do not support presence, and allows @@ -317,6 +304,8 @@ bridge: # Send captions in the same message as images. This will send data compatible with both MSC2530 and MSC3552. # This is currently not supported in most clients. caption_in_message: false + # Send galleries as a single event? This is not an MSC (yet). + beeper_galleries: false # Should polls be sent using MSC3381 event types? extev_polls: false # Should cross-chat replies from WhatsApp be bridged? Most servers and clients don't support this. @@ -385,6 +374,10 @@ bridge: delete_on_device_delete: false # Periodically delete megolm sessions when 2x max_age has passed since receiving the session. periodically_delete_expired: false + # Delete inbound megolm sessions that don't have the received_at field used for + # automatic ratcheting and expired session deletion. This is meant as a migration + # to delete old keys prior to the bridge update. + delete_outdated_inbound: false # What level of device verification should be required from users? # # Valid levels: @@ -431,6 +424,8 @@ bridge: # Shared secret for authentication. If set to "generate", a random secret will be generated, # or if set to "disable", the provisioning API will be disabled. shared_secret: generate + # Enable debug API at /debug with provisioning authentication. + debug_endpoints: false # Permissions for using the bridge. # Permitted values: diff --git a/formatting.go b/formatting.go index 3a06575..a60eb99 100644 --- a/formatting.go +++ b/formatting.go @@ -141,6 +141,9 @@ func (formatter *Formatter) ParseWhatsApp(roomID id.RoomID, content *event.Messa continue } else if jid.Server == types.LegacyUserServer { jid.Server = types.DefaultUserServer + } else if jid.Server != types.DefaultUserServer { + // TODO lid support? + continue } mxid, displayname := formatter.getMatrixInfoByJID(roomID, jid) number := "@" + jid.User diff --git a/go.mod b/go.mod index c2fef41..6135da5 100644 --- a/go.mod +++ b/go.mod @@ -1,24 +1,25 @@ module maunium.net/go/mautrix-whatsapp -go 1.19 +go 1.20 require ( - github.com/chai2010/webp v1.1.1 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 github.com/lib/pq v1.10.9 - github.com/mattn/go-sqlite3 v1.14.17 - github.com/prometheus/client_golang v1.15.1 - github.com/rs/zerolog v1.29.1 + github.com/mattn/go-sqlite3 v1.14.19 + github.com/prometheus/client_golang v1.17.0 + github.com/rs/zerolog v1.31.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e - github.com/tidwall/gjson v1.14.4 - go.mau.fi/whatsmeow v0.0.0-20230608204524-7aedaa1de108 - golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea - golang.org/x/image v0.7.0 - golang.org/x/net v0.10.0 - google.golang.org/protobuf v1.30.0 + github.com/tidwall/gjson v1.17.0 + go.mau.fi/util v0.2.1 + go.mau.fi/webp v0.1.0 + go.mau.fi/whatsmeow v0.0.0-20231216213200-9d803dd92735 + golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 + golang.org/x/image v0.14.0 + golang.org/x/net v0.19.0 + google.golang.org/protobuf v1.31.0 maunium.net/go/maulogger/v2 v2.4.1 - maunium.net/go/mautrix v0.15.3-0.20230609124302-54a73ab22ef9 + maunium.net/go/mautrix v0.16.2 ) require ( @@ -28,30 +29,22 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/kr/text v0.2.0 // indirect - github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.11.1 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/sjson v1.2.5 // indirect - github.com/yuin/goldmark v1.5.4 // indirect + github.com/yuin/goldmark v1.6.0 // indirect go.mau.fi/libsignal v0.1.0 // indirect go.mau.fi/zeroconfig v0.1.2 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/crypto v0.16.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect maunium.net/go/mauflag v1.0.0 // indirect ) - -// Exclude some things that cause go.sum to explode -exclude ( - cloud.google.com/go v0.65.0 - github.com/prometheus/client_golang v1.12.1 - google.golang.org/appengine v1.6.6 -) diff --git a/go.sum b/go.sum index cc7a65b..f568af8 100644 --- a/go.sum +++ b/go.sum @@ -5,15 +5,12 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chai2010/webp v1.1.1 h1:jTRmEccAJ4MGrhFOrPMpNGIJ/eybIgwKpcACsrTEapk= -github.com/chai2010/webp v1.1.1/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= @@ -28,99 +25,74 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= -github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= +github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= -github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= -github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= -github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= +github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= -github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.6.0 h1:boZcn2GTjpsynOsC0iJHnBWa4Bi0qzfJjthwauItG68= +github.com/yuin/goldmark v1.6.0/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c= go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I= -go.mau.fi/whatsmeow v0.0.0-20230608204524-7aedaa1de108 h1:kDOgPHj0urv2vsXyE8dt+KgyC18jxzQW48HGXa53pzc= -go.mau.fi/whatsmeow v0.0.0-20230608204524-7aedaa1de108/go.mod h1:+ObGpFE6cbbY4hKc1FmQH9MVfqaemmlXGXSnwDvCOyE= +go.mau.fi/util v0.2.1 h1:eazulhFE/UmjOFtPrGg6zkF5YfAyiDzQb8ihLMbsPWw= +go.mau.fi/util v0.2.1/go.mod h1:MjlzCQEMzJ+G8RsPawHzpLB8rwTo3aPIjG5FzBvQT/c= +go.mau.fi/webp v0.1.0 h1:BHObH/DcFntT9KYun5pDr0Ot4eUZO8k2C7eP7vF4ueA= +go.mau.fi/webp v0.1.0/go.mod h1:e42Z+VMFrUMS9cpEwGRIor+lQWO8oUAyPyMtcL+NMt8= +go.mau.fi/whatsmeow v0.0.0-20231216213200-9d803dd92735 h1:+teJYCOK6M4Kn2TYCj29levhHVwnJTmgCtEXLtgwQtM= +go.mau.fi/whatsmeow v0.0.0-20231216213200-9d803dd92735/go.mod h1:5xTtHNaZpGni6z6aE1iEopjW7wNgsKcolZxZrOujK9M= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea h1:vLCWI/yYrdEHyN2JzIzPO3aaQJHQdp89IZBA/+azVC4= -golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= -golang.org/x/image v0.7.0 h1:gzS29xtG1J5ybQlv0PuyfE3nmc6R4qB73m6LUUmvFuw= -golang.org/x/image v0.7.0/go.mod h1:nd/q4ef1AKKYl/4kft7g+6UyGbdiqWqTP1ZAbRoV7Rg= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4= +golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= +golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= @@ -131,5 +103,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= -maunium.net/go/mautrix v0.15.3-0.20230609124302-54a73ab22ef9 h1:MixixPn9FMv99V/6wdwDB18HR/1m69JDAc4VcEar8Wc= -maunium.net/go/mautrix v0.15.3-0.20230609124302-54a73ab22ef9/go.mod h1:h4NwfKqE4YxGTLSgn/gawKzXAb2sF4qx8agL6QEFtGg= +maunium.net/go/mautrix v0.16.2 h1:a6GUJXNWsTEOO8VE4dROBfCIfPp50mqaqzv7KPzChvg= +maunium.net/go/mautrix v0.16.2/go.mod h1:YL4l4rZB46/vj/ifRMEjcibbvHjgxHftOF1SgmruLu4= diff --git a/historysync.go b/historysync.go index a6dcf98..4adb14e 100644 --- a/historysync.go +++ b/historysync.go @@ -20,19 +20,19 @@ import ( "crypto/sha256" "encoding/base64" "fmt" + "strings" "time" - "maunium.net/go/mautrix/util/variationselector" - + "github.com/rs/zerolog" + "go.mau.fi/util/dbutil" + "go.mau.fi/util/variationselector" waProto "go.mau.fi/whatsmeow/binary/proto" "go.mau.fi/whatsmeow/types" "maunium.net/go/mautrix" "maunium.net/go/mautrix/appservice" - "maunium.net/go/mautrix/bridge/bridgeconfig" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/dbutil" "maunium.net/go/mautrix-whatsapp/config" "maunium.net/go/mautrix-whatsapp/database" @@ -60,25 +60,28 @@ func (user *User) handleHistorySyncsLoop() { return } - // Start the backfill queue. - user.BackfillQueue = &BackfillQueue{ - BackfillQuery: user.bridge.DB.Backfill, - reCheckChannels: []chan bool{}, - log: user.log.Sub("BackfillQueue"), + batchSend := user.bridge.SpecVersions.Supports(mautrix.BeeperFeatureBatchSending) + if batchSend { + // Start the backfill queue. + user.BackfillQueue = &BackfillQueue{ + BackfillQuery: user.bridge.DB.Backfill, + reCheckChannels: []chan bool{}, + log: user.log.Sub("BackfillQueue"), + } + + forwardAndImmediate := []database.BackfillType{database.BackfillImmediate, database.BackfillForward} + + // Immediate backfills can be done in parallel + for i := 0; i < user.bridge.Config.Bridge.HistorySync.Immediate.WorkerCount; i++ { + go user.HandleBackfillRequestsLoop(forwardAndImmediate, []database.BackfillType{}) + } + + // Deferred backfills should be handled synchronously so as not to + // overload the homeserver. Users can configure their backfill stages + // to be more or less aggressive with backfilling at this stage. + go user.HandleBackfillRequestsLoop([]database.BackfillType{database.BackfillDeferred}, forwardAndImmediate) } - forwardAndImmediate := []database.BackfillType{database.BackfillImmediate, database.BackfillForward} - - // Immediate backfills can be done in parallel - for i := 0; i < user.bridge.Config.Bridge.HistorySync.Immediate.WorkerCount; i++ { - go user.HandleBackfillRequestsLoop(forwardAndImmediate, []database.BackfillType{}) - } - - // Deferred backfills should be handled synchronously so as not to - // overload the homeserver. Users can configure their backfill stages - // to be more or less aggressive with backfilling at this stage. - go user.HandleBackfillRequestsLoop([]database.BackfillType{database.BackfillDeferred}, forwardAndImmediate) - if user.bridge.Config.Bridge.HistorySync.MediaRequests.AutoRequestMedia && user.bridge.Config.Bridge.HistorySync.MediaRequests.RequestMethod == config.MediaRequestMethodLocalTime { go user.dailyMediaRequestLoop() @@ -92,9 +95,13 @@ func (user *User) handleHistorySyncsLoop() { if evt == nil { return } - user.handleHistorySync(user.BackfillQueue, evt.Data) + user.storeHistorySync(evt.Data) case <-user.enqueueBackfillsTimer.C: - user.enqueueAllBackfills() + if batchSend { + user.enqueueAllBackfills() + } else { + user.backfillAll() + } } } } @@ -102,7 +109,7 @@ func (user *User) handleHistorySyncsLoop() { const EnqueueBackfillsDelay = 30 * time.Second func (user *User) enqueueAllBackfills() { - nMostRecent := user.bridge.DB.HistorySync.GetNMostRecentConversations(user.MXID, user.bridge.Config.Bridge.HistorySync.MaxInitialConversations) + nMostRecent := user.bridge.DB.HistorySync.GetRecentConversations(user.MXID, user.bridge.Config.Bridge.HistorySync.MaxInitialConversations) if len(nMostRecent) > 0 { user.log.Infofln("%v has passed since the last history sync blob, enqueueing backfills for %d chats", EnqueueBackfillsDelay, len(nMostRecent)) // Find the portals for all the conversations. @@ -125,6 +132,82 @@ func (user *User) enqueueAllBackfills() { } } +func (user *User) backfillAll() { + conversations := user.bridge.DB.HistorySync.GetRecentConversations(user.MXID, -1) + if len(conversations) > 0 { + user.zlog.Info(). + Int("conversation_count", len(conversations)). + Msg("Probably received all history sync blobs, now backfilling conversations") + limit := user.bridge.Config.Bridge.HistorySync.MaxInitialConversations + bridgedCount := 0 + // Find the portals for all the conversations. + for _, conv := range conversations { + jid, err := types.ParseJID(conv.ConversationID) + if err != nil { + user.zlog.Warn().Err(err). + Str("conversation_id", conv.ConversationID). + Msg("Failed to parse chat JID in history sync") + continue + } + portal := user.GetPortalByJID(jid) + if portal.MXID != "" { + user.zlog.Debug(). + Str("portal_jid", portal.Key.JID.String()). + Msg("Chat already has a room, deleting messages from database") + user.bridge.DB.HistorySync.DeleteConversation(user.MXID, portal.Key.JID.String()) + bridgedCount++ + } else if !user.bridge.DB.HistorySync.ConversationHasMessages(user.MXID, portal.Key) { + user.zlog.Debug().Str("portal_jid", portal.Key.JID.String()).Msg("Skipping chat with no messages in history sync") + user.bridge.DB.HistorySync.DeleteConversation(user.MXID, portal.Key.JID.String()) + } else if limit < 0 || bridgedCount < limit { + bridgedCount++ + err = portal.CreateMatrixRoom(user, nil, nil, true, true) + if err != nil { + user.zlog.Err(err).Msg("Failed to create Matrix room for backfill") + } + } + } + } +} + +func (portal *Portal) legacyBackfill(user *User) { + defer portal.latestEventBackfillLock.Unlock() + // This should only be called from CreateMatrixRoom which locks latestEventBackfillLock before creating the room. + if portal.latestEventBackfillLock.TryLock() { + panic("legacyBackfill() called without locking latestEventBackfillLock") + } + // TODO use portal.zlog instead of user.zlog + log := user.zlog.With(). + Str("portal_jid", portal.Key.JID.String()). + Str("action", "legacy backfill"). + Logger() + conv := user.bridge.DB.HistorySync.GetConversation(user.MXID, portal.Key) + messages := user.bridge.DB.HistorySync.GetMessagesBetween(user.MXID, portal.Key.JID.String(), nil, nil, portal.bridge.Config.Bridge.HistorySync.MessageCount) + log.Debug().Int("message_count", len(messages)).Msg("Got messages to backfill from database") + for i := len(messages) - 1; i >= 0; i-- { + msgEvt, err := user.Client.ParseWebMessage(portal.Key.JID, messages[i]) + if err != nil { + log.Warn().Err(err). + Int("msg_index", i). + Str("msg_id", messages[i].GetKey().GetId()). + Uint64("msg_time_seconds", messages[i].GetMessageTimestamp()). + Msg("Dropping historical message due to parse error") + continue + } + portal.handleMessage(user, msgEvt, true) + } + if conv != nil { + isUnread := conv.MarkedAsUnread || conv.UnreadCount > 0 + isTooOld := user.bridge.Config.Bridge.HistorySync.UnreadHoursThreshold > 0 && conv.LastMessageTimestamp.Before(time.Now().Add(time.Duration(-user.bridge.Config.Bridge.HistorySync.UnreadHoursThreshold)*time.Hour)) + shouldMarkAsRead := !isUnread || isTooOld + if shouldMarkAsRead { + user.markSelfReadFull(portal) + } + } + log.Debug().Msg("Backfill complete, deleting leftover messages from database") + user.bridge.DB.HistorySync.DeleteConversation(user.MXID, portal.Key.JID.String()) +} + func (user *User) dailyMediaRequestLoop() { // Calculate when to do the first set of media retry requests now := time.Now() @@ -175,8 +258,8 @@ func (user *User) backfillInChunks(req *database.Backfill, conv *database.Histor portal.backfillLock.Lock() defer portal.backfillLock.Unlock() - if !user.shouldCreatePortalForHistorySync(conv, portal) { - return + if len(portal.MXID) > 0 && !user.bridge.AS.StateStore.IsInRoom(portal.MXID, user.MXID) { + portal.ensureUserInvited(user) } backfillState := user.bridge.DB.Backfill.GetBackfillState(user.MXID, &portal.Key) @@ -186,19 +269,17 @@ func (user *User) backfillInChunks(req *database.Backfill, conv *database.Histor backfillState.SetProcessingBatch(true) defer backfillState.SetProcessingBatch(false) - var forwardPrevID id.EventID var timeEnd *time.Time - var isLatestEvents, shouldMarkAsRead, shouldAtomicallyMarkAsRead bool + var forward, shouldMarkAsRead bool portal.latestEventBackfillLock.Lock() if req.BackfillType == database.BackfillForward { // TODO this overrides the TimeStart set when enqueuing the backfill // maybe the enqueue should instead include the prev event ID lastMessage := portal.bridge.DB.Message.GetLastInChat(portal.Key) - forwardPrevID = lastMessage.MXID start := lastMessage.Timestamp.Add(1 * time.Second) req.TimeStart = &start // Sending events at the end of the room (= latest events) - isLatestEvents = true + forward = true } else { firstMessage := portal.bridge.DB.Message.GetFirstInChat(portal.Key) if firstMessage != nil { @@ -207,10 +288,10 @@ func (user *User) backfillInChunks(req *database.Backfill, conv *database.Histor user.log.Debugfln("Limiting backfill to end at %v", end) } else { // Portal is empty -> events are latest - isLatestEvents = true + forward = true } } - if !isLatestEvents { + if !forward { // We'll use normal batch sending, so no need to keep blocking new message processing portal.latestEventBackfillLock.Unlock() } else { @@ -221,7 +302,6 @@ func (user *User) backfillInChunks(req *database.Backfill, conv *database.Histor isUnread := conv.MarkedAsUnread || conv.UnreadCount > 0 isTooOld := user.bridge.Config.Bridge.HistorySync.UnreadHoursThreshold > 0 && conv.LastMessageTimestamp.Before(time.Now().Add(time.Duration(-user.bridge.Config.Bridge.HistorySync.UnreadHoursThreshold)*time.Hour)) shouldMarkAsRead = !isUnread || isTooOld - shouldAtomicallyMarkAsRead = shouldMarkAsRead && user.bridge.Config.Homeserver.Software == bridgeconfig.SoftwareHungry } allMsgs := user.bridge.DB.HistorySync.GetMessagesBetween(user.MXID, conv.ConversationID, req.TimeStart, timeEnd, req.MaxTotalEvents) @@ -243,7 +323,7 @@ func (user *User) backfillInChunks(req *database.Backfill, conv *database.Histor if len(portal.MXID) == 0 { user.log.Debugln("Creating portal for", portal.Key.JID, "as part of history sync handling") - err := portal.CreateMatrixRoom(user, nil, true, false) + err := portal.CreateMatrixRoom(user, nil, nil, true, false) if err != nil { user.log.Errorfln("Failed to create room for %s during backfill: %v", portal.Key.JID, err) return @@ -280,7 +360,6 @@ func (user *User) backfillInChunks(req *database.Backfill, conv *database.Histor user.log.Infofln("Backfilling %d messages in %s, %d messages at a time (queue ID: %d)", len(allMsgs), portal.Key.JID, req.MaxBatchEvents, req.QueueID) toBackfill := allMsgs[0:] - var insertionEventIds []id.EventID for len(toBackfill) > 0 { var msgs []*waProto.WebMessageInfo if len(toBackfill) <= req.MaxBatchEvents || req.MaxBatchEvents < 0 { @@ -294,19 +373,10 @@ func (user *User) backfillInChunks(req *database.Backfill, conv *database.Histor if len(msgs) > 0 { time.Sleep(time.Duration(req.BatchDelay) * time.Second) user.log.Debugfln("Backfilling %d messages in %s (queue ID: %d)", len(msgs), portal.Key.JID, req.QueueID) - resp := portal.backfill(user, msgs, req.BackfillType == database.BackfillForward, isLatestEvents, shouldAtomicallyMarkAsRead, forwardPrevID) - if resp != nil && (resp.BaseInsertionEventID != "" || !isLatestEvents) { - insertionEventIds = append(insertionEventIds, resp.BaseInsertionEventID) - } + portal.backfill(user, msgs, forward, shouldMarkAsRead) } } user.log.Debugfln("Finished backfilling %d messages in %s (queue ID: %d)", len(allMsgs), portal.Key.JID, req.QueueID) - if len(insertionEventIds) > 0 { - portal.sendPostBackfillDummy( - time.Unix(int64(allMsgs[0].GetMessageTimestamp()), 0), - insertionEventIds[0]) - } - user.log.Debugfln("Deleting %d history sync messages after backfilling (queue ID: %d)", len(allMsgs), req.QueueID) err := user.bridge.DB.HistorySync.DeleteMessages(user.MXID, conv.ConversationID, allMsgs) if err != nil { user.log.Warnfln("Failed to delete %d history sync messages after backfilling (queue ID: %d): %v", len(allMsgs), req.QueueID, err) @@ -332,38 +402,14 @@ func (user *User) backfillInChunks(req *database.Backfill, conv *database.Histor backfillState.Upsert() portal.updateBackfillStatus(backfillState) } - - if isLatestEvents && !shouldAtomicallyMarkAsRead { - if shouldMarkAsRead { - user.markSelfReadFull(portal) - } else if conv.MarkedAsUnread && user.bridge.Config.Bridge.SyncManualMarkedUnread { - user.markUnread(portal, true) - } - } } -func (user *User) shouldCreatePortalForHistorySync(conv *database.HistorySyncConversation, portal *Portal) bool { - if len(portal.MXID) > 0 { - if !user.bridge.AS.StateStore.IsInRoom(portal.MXID, user.MXID) { - portal.ensureUserInvited(user) - } - // Portal exists, let backfill continue - return true - } else if !user.bridge.Config.Bridge.HistorySync.CreatePortals { - user.log.Debugfln("Not creating portal for %s: creating rooms from history sync is disabled", portal.Key.JID) - return false - } else { - // Portal doesn't exist, but should be created - return true - } -} - -func (user *User) handleHistorySync(backfillQueue *BackfillQueue, evt *waProto.HistorySync) { +func (user *User) storeHistorySync(evt *waProto.HistorySync) { if evt == nil || evt.SyncType == nil { return } log := user.bridge.ZLog.With(). - Str("method", "User.handleHistorySync"). + Str("method", "User.storeHistorySync"). Str("user_id", user.MXID.String()). Str("sync_type", evt.GetSyncType().String()). Uint32("chunk_order", evt.GetChunkOrder()). @@ -401,28 +447,38 @@ func (user *User) handleHistorySync(backfillQueue *BackfillQueue, evt *waProto.H } else if jid.Server == types.BroadcastServer { log.Debug().Str("chat_jid", jid.String()).Msg("Skipping broadcast list in history sync") continue + } else if jid.Server == types.HiddenUserServer { + log.Debug().Str("chat_jid", jid.String()).Msg("Skipping hidden user JID chat in history sync") + continue } totalMessageCount += len(conv.GetMessages()) - portal := user.GetPortalByJID(jid) log := log.With(). - Str("chat_jid", portal.Key.JID.String()). + Str("chat_jid", jid.String()). Int("msg_count", len(conv.GetMessages())). Logger() - historySyncConversation := user.bridge.DB.HistorySync.NewConversationWithValues( - user.MXID, - conv.GetId(), - &portal.Key, - getConversationTimestamp(conv), - conv.GetMuteEndTime(), - conv.GetArchived(), - conv.GetPinned(), - conv.GetDisappearingMode().GetInitiator(), - conv.GetEndOfHistoryTransferType(), - conv.EphemeralExpiration, - conv.GetMarkedAsUnread(), - conv.GetUnreadCount()) - historySyncConversation.Upsert() + var portal *Portal + initPortal := func() { + if portal != nil { + return + } + portal = user.GetPortalByJID(jid) + historySyncConversation := user.bridge.DB.HistorySync.NewConversationWithValues( + user.MXID, + conv.GetId(), + &portal.Key, + getConversationTimestamp(conv), + conv.GetMuteEndTime(), + conv.GetArchived(), + conv.GetPinned(), + conv.GetDisappearingMode().GetInitiator(), + conv.GetEndOfHistoryTransferType(), + conv.EphemeralExpiration, + conv.GetMarkedAsUnread(), + conv.GetUnreadCount()) + historySyncConversation.Upsert() + } + var minTime, maxTime time.Time var minTimeIndex, maxTimeIndex int @@ -430,7 +486,7 @@ func (user *User) handleHistorySync(backfillQueue *BackfillQueue, evt *waProto.H unsupportedTypes := 0 for i, rawMsg := range conv.GetMessages() { // Don't store messages that will just be skipped. - msgEvt, err := user.Client.ParseWebMessage(portal.Key.JID, rawMsg.GetMessage()) + msgEvt, err := user.Client.ParseWebMessage(jid, rawMsg.GetMessage()) if err != nil { log.Warn().Err(err). Int("msg_index", i). @@ -449,16 +505,12 @@ func (user *User) handleHistorySync(backfillQueue *BackfillQueue, evt *waProto.H } msgType := getMessageType(msgEvt.Message) - if msgType == "unknown" || msgType == "ignore" || msgType == "unknown_protocol" { + if msgType == "unknown" || msgType == "ignore" || strings.HasPrefix(msgType, "unknown_protocol_") || !containsSupportedMessage(msgEvt.Message) { unsupportedTypes++ continue } - // Don't store unsupported messages. - if !containsSupportedMessage(msgEvt.Message) { - unsupportedTypes++ - continue - } + initPortal() message, err := user.bridge.DB.HistorySync.NewMessageWithValues(user.MXID, conv.GetId(), msgEvt.Info.ID, rawMsg) if err != nil { @@ -487,6 +539,14 @@ func (user *User) handleHistorySync(backfillQueue *BackfillQueue, evt *waProto.H Int("lowest_time_index", minTimeIndex). Time("highest_time", maxTime). Int("highest_time_index", maxTimeIndex). + Dict("metadata", zerolog.Dict(). + Uint32("ephemeral_expiration", conv.GetEphemeralExpiration()). + Bool("marked_unread", conv.GetMarkedAsUnread()). + Bool("archived", conv.GetArchived()). + Uint32("pinned", conv.GetPinned()). + Uint64("mute_end", conv.GetMuteEndTime()). + Uint32("unread_count", conv.GetUnreadCount()), + ). Msg("Saved messages from history sync conversation") } log.Info(). @@ -560,69 +620,20 @@ func (portal *Portal) deterministicEventID(sender types.JID, messageID types.Mes var ( PortalCreationDummyEvent = event.Type{Type: "fi.mau.dummy.portal_created", Class: event.MessageEventType} - PreBackfillDummyEvent = event.Type{Type: "fi.mau.dummy.pre_backfill", Class: event.MessageEventType} - - HistorySyncMarker = event.Type{Type: "org.matrix.msc2716.marker", Class: event.MessageEventType} BackfillStatusEvent = event.Type{Type: "com.beeper.backfill_status", Class: event.StateEventType} ) -func (portal *Portal) backfill(source *User, messages []*waProto.WebMessageInfo, isForward, isLatest, atomicMarkAsRead bool, prevEventID id.EventID) *mautrix.RespBatchSend { - var req mautrix.ReqBatchSend +func (portal *Portal) backfill(source *User, messages []*waProto.WebMessageInfo, isForward, atomicMarkAsRead bool) *mautrix.RespBeeperBatchSend { + var req mautrix.ReqBeeperBatchSend var infos []*wrappedInfo - if !isForward { - if portal.FirstEventID != "" || portal.NextBatchID != "" { - req.PrevEventID = portal.FirstEventID - req.BatchID = portal.NextBatchID - } else { - portal.log.Warnfln("Can't backfill %d messages through %s to chat: first event ID not known", len(messages), source.MXID) - return nil - } - } else { - req.PrevEventID = prevEventID - } - req.BeeperNewMessages = isLatest && req.BatchID == "" + req.Forward = isForward if atomicMarkAsRead { - req.BeeperMarkReadBy = source.MXID + req.MarkReadBy = source.MXID } - beforeFirstMessageTimestampMillis := (int64(messages[len(messages)-1].GetMessageTimestamp()) * 1000) - 1 - req.StateEventsAtStart = make([]*event.Event, 0) - - addedMembers := make(map[id.UserID]struct{}) - addMember := func(puppet *Puppet) { - if portal.bridge.Config.Homeserver.Software == bridgeconfig.SoftwareHungry { - // Hungryserv doesn't need state_events_at_start, it can figure out memberships automatically - return - } else if _, alreadyAdded := addedMembers[puppet.MXID]; alreadyAdded { - return - } - mxid := puppet.MXID.String() - content := event.MemberEventContent{ - Membership: event.MembershipJoin, - Displayname: puppet.Displayname, - AvatarURL: puppet.AvatarURL.CUString(), - } - inviteContent := content - inviteContent.Membership = event.MembershipInvite - req.StateEventsAtStart = append(req.StateEventsAtStart, &event.Event{ - Type: event.StateMember, - Sender: portal.MainIntent().UserID, - StateKey: &mxid, - Timestamp: beforeFirstMessageTimestampMillis, - Content: event.Content{Parsed: &inviteContent}, - }, &event.Event{ - Type: event.StateMember, - Sender: puppet.MXID, - StateKey: &mxid, - Timestamp: beforeFirstMessageTimestampMillis, - Content: event.Content{Parsed: &content}, - }) - addedMembers[puppet.MXID] = struct{}{} - } - - portal.log.Infofln("Processing history sync with %d messages (forward: %t, latest: %t, prev: %s, batch: %s)", len(messages), isForward, isLatest, req.PrevEventID, req.BatchID) + portal.log.Infofln("Processing history sync with %d messages (forward: %t)", len(messages), isForward) // The messages are ordered newest to oldest, so iterate them in reverse order. for i := len(messages) - 1; i >= 0; i-- { webMsg := messages[i] @@ -653,21 +664,14 @@ func (portal *Portal) backfill(source *User, messages []*waProto.WebMessageInfo, if puppet == nil { continue } - intent := puppet.IntentFor(portal) - if intent.IsCustomPuppet && !portal.bridge.Config.CanDoublePuppetBackfill(puppet.CustomMXID) { - intent = puppet.DefaultIntent() - } - converted := portal.convertMessage(intent, source, &msgEvt.Info, msgEvt.Message, true) + converted := portal.convertMessage(puppet.IntentFor(portal), source, &msgEvt.Info, msgEvt.Message, true) if converted == nil { portal.log.Debugfln("Skipping unsupported message %s in backfill", msgEvt.Info.ID) continue } - if !intent.IsCustomPuppet && !portal.bridge.StateStore.IsInRoom(portal.MXID, puppet.MXID) { - addMember(puppet) - } if converted.ReplyTo != nil { - portal.SetReply(converted.Content, converted.ReplyTo, true) + portal.SetReply(msgEvt.Info.ID, converted.Content, converted.ReplyTo, true) } err = portal.appendBatchEvents(source, converted, &msgEvt.Info, webMsg, &req.Events, &infos) if err != nil { @@ -680,15 +684,7 @@ func (portal *Portal) backfill(source *User, messages []*waProto.WebMessageInfo, return nil } - if len(req.BatchID) == 0 || isForward { - portal.log.Debugln("Sending a dummy event to avoid forward extremity errors with backfill") - _, err := portal.MainIntent().SendMessageEvent(portal.MXID, PreBackfillDummyEvent, struct{}{}) - if err != nil { - portal.log.Warnln("Error sending pre-backfill dummy event:", err) - } - } - - resp, err := portal.MainIntent().BatchSend(portal.MXID, &req) + resp, err := portal.MainIntent().BeeperBatchSend(portal.MXID, &req) if err != nil { portal.log.Errorln("Error batch sending messages:", err) return nil @@ -699,12 +695,7 @@ func (portal *Portal) backfill(source *User, messages []*waProto.WebMessageInfo, return nil } - // Do the following block in the transaction - { - portal.finishBatch(txn, resp.EventIDs, infos) - portal.NextBatchID = resp.NextBatchID - portal.Update(txn) - } + portal.finishBatch(txn, resp.EventIDs, infos) err = txn.Commit() if err != nil { @@ -779,19 +770,16 @@ func (portal *Portal) appendBatchEvents(source *User, converted *ConvertedMessag *infoArray = append(*infoArray, nil) } } - // Sending reactions in the same batch requires deterministic event IDs, so only do it on hungryserv - if portal.bridge.Config.Homeserver.Software == bridgeconfig.SoftwareHungry { - for _, reaction := range raw.GetReactions() { - reactionEvent, reactionInfo := portal.wrapBatchReaction(source, reaction, mainEvt.ID, info.Timestamp) - if reactionEvent != nil { - *eventsArray = append(*eventsArray, reactionEvent) - *infoArray = append(*infoArray, &wrappedInfo{ - MessageInfo: reactionInfo, - SenderMXID: reactionEvent.Sender, - ReactionTarget: info.ID, - Type: database.MsgReaction, - }) - } + for _, reaction := range raw.GetReactions() { + reactionEvent, reactionInfo := portal.wrapBatchReaction(source, reaction, mainEvt.ID, info.Timestamp) + if reactionEvent != nil { + *eventsArray = append(*eventsArray, reactionEvent) + *infoArray = append(*infoArray, &wrappedInfo{ + MessageInfo: reactionInfo, + SenderMXID: reactionEvent.Sender, + ReactionTarget: info.ID, + Type: database.MsgReaction, + }) } } return nil @@ -856,13 +844,8 @@ func (portal *Portal) wrapBatchEvent(info *types.MessageInfo, intent *appservice return nil, err } intent.AddDoublePuppetValue(&wrappedContent) - var eventID id.EventID - if portal.bridge.Config.Homeserver.Software == bridgeconfig.SoftwareHungry { - eventID = portal.deterministicEventID(info.Sender, info.ID, partName) - } - return &event.Event{ - ID: eventID, + ID: portal.deterministicEventID(info.Sender, info.ID, partName), Sender: intent.UserID, Type: newEventType, Timestamp: info.Timestamp.UnixMilli(), @@ -877,7 +860,7 @@ func (portal *Portal) finishBatch(txn dbutil.Transaction, eventIDs []id.EventID, } eventID := eventIDs[i] - portal.markHandled(txn, nil, info.MessageInfo, eventID, info.SenderMXID, true, false, info.Type, info.Error) + portal.markHandled(txn, nil, info.MessageInfo, eventID, info.SenderMXID, true, false, info.Type, 0, info.Error) if info.Type == database.MsgReaction { portal.upsertReaction(txn, nil, info.ReactionTarget, info.Sender, eventID, info.ID) } @@ -889,33 +872,13 @@ func (portal *Portal) finishBatch(txn dbutil.Transaction, eventIDs []id.EventID, portal.log.Infofln("Successfully sent %d events", len(eventIDs)) } -func (portal *Portal) sendPostBackfillDummy(lastTimestamp time.Time, insertionEventId id.EventID) { - resp, err := portal.MainIntent().SendMessageEvent(portal.MXID, HistorySyncMarker, map[string]interface{}{ - "org.matrix.msc2716.marker.insertion": insertionEventId, - //"m.marker.insertion": insertionEventId, - }) - if err != nil { - portal.log.Errorln("Error sending post-backfill dummy event:", err) - return - } - msg := portal.bridge.DB.Message.New() - msg.Chat = portal.Key - msg.MXID = resp.EventID - msg.SenderMXID = portal.MainIntent().UserID - msg.JID = types.MessageID(resp.EventID) - msg.Timestamp = lastTimestamp.Add(1 * time.Second) - msg.Sent = true - msg.Type = database.MsgFake - msg.Insert(nil) -} - func (portal *Portal) updateBackfillStatus(backfillState *database.BackfillState) { backfillStatus := "backfilling" if backfillState.BackfillComplete { backfillStatus = "complete" } - _, err := portal.MainIntent().SendStateEvent(portal.MXID, BackfillStatusEvent, "", map[string]interface{}{ + _, err := portal.bridge.Bot.SendStateEvent(portal.MXID, BackfillStatusEvent, "", map[string]interface{}{ "status": backfillStatus, "first_timestamp": backfillState.FirstExpectedTimestamp * 1000, }) diff --git a/main.go b/main.go index 9656be1..c8958af 100644 --- a/main.go +++ b/main.go @@ -19,6 +19,7 @@ package main import ( _ "embed" "net/http" + "net/url" "os" "strconv" "strings" @@ -27,19 +28,18 @@ import ( "google.golang.org/protobuf/proto" + "go.mau.fi/util/configupgrade" "go.mau.fi/whatsmeow" waProto "go.mau.fi/whatsmeow/binary/proto" "go.mau.fi/whatsmeow/store" "go.mau.fi/whatsmeow/store/sqlstore" "go.mau.fi/whatsmeow/types" - "maunium.net/go/mautrix" "maunium.net/go/mautrix/bridge" "maunium.net/go/mautrix/bridge/commands" "maunium.net/go/mautrix/bridge/status" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util/configupgrade" "maunium.net/go/mautrix-whatsapp/config" "maunium.net/go/mautrix-whatsapp/database" @@ -91,13 +91,18 @@ func (br *WABridge) Init() { br.EventProcessor.On(TypeMSC3381PollResponse, br.MatrixHandler.HandleMessage) br.EventProcessor.On(TypeMSC3381V2PollResponse, br.MatrixHandler.HandleMessage) - Segment.log = br.Log.Sub("Segment") - Segment.key = br.Config.SegmentKey - Segment.userID = br.Config.SegmentUserID - if Segment.IsEnabled() { - Segment.log.Infoln("Segment metrics are enabled") - if Segment.userID != "" { - Segment.log.Infoln("Overriding Segment user_id with %v", Segment.userID) + Analytics.log = br.Log.Sub("Analytics") + Analytics.url = (&url.URL{ + Scheme: "https", + Host: br.Config.Analytics.Host, + Path: "/v1/track", + }).String() + Analytics.key = br.Config.Analytics.Token + Analytics.userID = br.Config.Analytics.UserID + if Analytics.IsEnabled() { + Analytics.log.Infoln("Analytics metrics are enabled") + if Analytics.userID != "" { + Analytics.log.Infoln("Overriding analytics user_id with %v", Analytics.userID) } } @@ -248,20 +253,6 @@ func (br *WABridge) GetConfigPtr() interface{} { return br.Config } -const unstableFeatureBatchSending = "org.matrix.msc2716" - -func (br *WABridge) CheckFeatures(versions *mautrix.RespVersions) (string, bool) { - if br.Config.Bridge.HistorySync.Backfill { - supported, known := versions.UnstableFeatures[unstableFeatureBatchSending] - if !known { - return "Backfilling is enabled in bridge config, but homeserver does not support MSC2716 batch sending", false - } else if !supported { - return "Backfilling is enabled in bridge config, but MSC2716 batch sending is not enabled on homeserver", false - } - } - return "", true -} - func main() { br := &WABridge{ usersByMXID: make(map[id.UserID]*User), @@ -277,7 +268,7 @@ func main() { Name: "mautrix-whatsapp", URL: "https://github.com/mautrix/whatsapp", Description: "A Matrix-WhatsApp puppeting bridge.", - Version: "0.8.5", + Version: "0.10.5", ProtocolName: "WhatsApp", BeeperServiceName: "whatsapp", BeeperNetworkName: "whatsapp", diff --git a/messagetracking.go b/messagetracking.go index d1c91e6..9e79f66 100644 --- a/messagetracking.go +++ b/messagetracking.go @@ -37,6 +37,7 @@ var ( errUserNotConnected = errors.New("you are not connected to WhatsApp") errDifferentUser = errors.New("user is not the recipient of this private chat portal") errUserNotLoggedIn = errors.New("user is not logged in and chat has no relay bot") + errRelaybotNotLoggedIn = errors.New("neither user nor relay bot of chat are logged in") errMNoticeDisabled = errors.New("bridging m.notice messages is disabled") errUnexpectedParsedContentType = errors.New("unexpected parsed content type") errInvalidGeoURI = errors.New("invalid `geo:` URI in message") @@ -55,6 +56,9 @@ var ( errPollMissingQuestion = errors.New("poll message is missing question") errPollDuplicateOption = errors.New("poll options must be unique") + errGalleryRelay = errors.New("can't send gallery through relay user") + errGalleryCaption = errors.New("can't send gallery with caption") + errEditUnknownTarget = errors.New("unknown edit target message") errEditUnknownTargetType = errors.New("unsupported edited message type") errEditDifferentSender = errors.New("can't edit message sent by another user") @@ -108,7 +112,8 @@ func errorToStatusReason(err error) (reason event.MessageStatusReason, status ev errors.Is(err, errUserNotConnected): return event.MessageStatusGenericError, event.MessageStatusRetriable, true, true, "" case errors.Is(err, errUserNotLoggedIn), - errors.Is(err, errDifferentUser): + errors.Is(err, errDifferentUser), + errors.Is(err, errRelaybotNotLoggedIn): return event.MessageStatusGenericError, event.MessageStatusRetriable, true, false, "" case errors.Is(err, errMessageDisconnected), errors.Is(err, errMessageRetryDisconnected): @@ -147,7 +152,7 @@ func (portal *Portal) sendErrorMessage(evt *event.Event, err error, msgType stri return resp.EventID } -func (portal *Portal) sendStatusEvent(evtID, lastRetry id.EventID, err error) { +func (portal *Portal) sendStatusEvent(evtID, lastRetry id.EventID, err error, deliveredTo *[]id.UserID) { if !portal.bridge.Config.Bridge.MessageStatusEvents { return } @@ -165,7 +170,8 @@ func (portal *Portal) sendStatusEvent(evtID, lastRetry id.EventID, err error) { Type: event.RelReference, EventID: evtID, }, - LastRetry: lastRetry, + DeliveredToUsers: deliveredTo, + LastRetry: lastRetry, } if err == nil { content.Status = event.MessageStatusSuccess @@ -224,12 +230,16 @@ func (portal *Portal) sendMessageMetrics(evt *event.Event, err error, part strin if sendNotice { ms.setNoticeID(portal.sendErrorMessage(evt, err, msgType, isCertain, ms.getNoticeID())) } - portal.sendStatusEvent(origEvtID, evt.ID, err) + portal.sendStatusEvent(origEvtID, evt.ID, err, nil) } else { portal.log.Debugfln("Handled Matrix %s %s", msgType, evtDescription) portal.sendDeliveryReceipt(evt.ID) portal.bridge.SendMessageSuccessCheckpoint(evt, status.MsgStepRemote, ms.getRetryNum()) - portal.sendStatusEvent(origEvtID, evt.ID, nil) + var deliveredTo *[]id.UserID + if portal.IsPrivateChat() { + deliveredTo = &[]id.UserID{} + } + portal.sendStatusEvent(origEvtID, evt.ID, nil, deliveredTo) if prevNotice := ms.popNoticeID(); prevNotice != "" { _, _ = portal.MainIntent().RedactEvent(portal.MXID, prevNotice, mautrix.ReqRedact{ Reason: "error resolved", diff --git a/metrics.go b/metrics.go index 5613b50..6a93b04 100644 --- a/metrics.go +++ b/metrics.go @@ -52,6 +52,7 @@ type MetricsHandler struct { countCollection prometheus.Histogram disconnections *prometheus.CounterVec incomingRetryReceipts *prometheus.CounterVec + connectionFailures *prometheus.CounterVec puppetCount prometheus.Gauge userCount prometheus.Gauge messageCount prometheus.Gauge @@ -101,6 +102,10 @@ func NewMetricsHandler(address string, log log.Logger, db *database.Database) *M Name: "whatsapp_disconnections", Help: "Number of times a Matrix user has been disconnected from WhatsApp", }, []string{"user_id"}), + connectionFailures: promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "whatsapp_connection_failures", + Help: "Number of times a connection has failed to whatsapp", + }, []string{"reason"}), incomingRetryReceipts: promauto.NewCounterVec(prometheus.CounterOpts{ Name: "whatsapp_incoming_retry_receipts", Help: "Number of times a remote WhatsApp user has requested a retry from the bridge. retry_count = 5 is usually the last attempt (and very likely means a failed message)", @@ -173,6 +178,13 @@ func (mh *MetricsHandler) TrackDisconnection(userID id.UserID) { mh.disconnections.With(prometheus.Labels{"user_id": string(userID)}).Inc() } +func (mh *MetricsHandler) TrackConnectionFailure(reason string) { + if !mh.running { + return + } + mh.connectionFailures.With(prometheus.Labels{"reason": reason}).Inc() +} + func (mh *MetricsHandler) TrackRetryReceipt(count int, found bool) { if !mh.running { return diff --git a/portal.go b/portal.go index 8a142ce..d1373b8 100644 --- a/portal.go +++ b/portal.go @@ -41,31 +41,34 @@ import ( "sync" "time" - "github.com/chai2010/webp" + "github.com/rs/zerolog" "github.com/tidwall/gjson" - "golang.org/x/exp/slices" - "golang.org/x/image/draw" - "google.golang.org/protobuf/proto" - - log "maunium.net/go/maulogger/v2" - - "maunium.net/go/mautrix" - "maunium.net/go/mautrix/appservice" - "maunium.net/go/mautrix/bridge" - "maunium.net/go/mautrix/bridge/bridgeconfig" - "maunium.net/go/mautrix/crypto/attachment" - "maunium.net/go/mautrix/event" - "maunium.net/go/mautrix/format" - "maunium.net/go/mautrix/id" - "maunium.net/go/mautrix/util" - "maunium.net/go/mautrix/util/dbutil" - "maunium.net/go/mautrix/util/ffmpeg" - "maunium.net/go/mautrix/util/variationselector" - + "go.mau.fi/util/dbutil" + "go.mau.fi/util/exerrors" + "go.mau.fi/util/exmime" + "go.mau.fi/util/ffmpeg" + "go.mau.fi/util/jsontime" + "go.mau.fi/util/random" + "go.mau.fi/util/variationselector" + cwebp "go.mau.fi/webp" "go.mau.fi/whatsmeow" waProto "go.mau.fi/whatsmeow/binary/proto" "go.mau.fi/whatsmeow/types" "go.mau.fi/whatsmeow/types/events" + "golang.org/x/exp/slices" + "golang.org/x/image/draw" + "golang.org/x/image/webp" + "google.golang.org/protobuf/proto" + log "maunium.net/go/maulogger/v2" + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/appservice" + "maunium.net/go/mautrix/bridge" + "maunium.net/go/mautrix/bridge/bridgeconfig" + "maunium.net/go/mautrix/bridge/status" + "maunium.net/go/mautrix/crypto/attachment" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/format" + "maunium.net/go/mautrix/id" "maunium.net/go/mautrix-whatsapp/database" ) @@ -107,7 +110,13 @@ func (portal *Portal) MarkEncrypted() { func (portal *Portal) ReceiveMatrixEvent(user bridge.User, evt *event.Event) { if user.GetPermissionLevel() >= bridgeconfig.PermissionLevelUser || portal.HasRelaybot() { - portal.matrixMessages <- PortalMatrixMessage{user: user.(*User), evt: evt, receivedAt: time.Now()} + portal.events <- &PortalEvent{ + MatrixMessage: &PortalMatrixMessage{ + user: user.(*User), + evt: evt, + receivedAt: time.Now(), + }, + } } } @@ -194,10 +203,9 @@ func (br *WABridge) newBlankPortal(key database.PortalKey) *Portal { portal := &Portal{ bridge: br, log: br.Log.Sub(fmt.Sprintf("Portal/%s", key)), + zlog: br.ZLog.With().Str("portal_key", key.String()).Logger(), - messages: make(chan PortalMessage, br.Config.Bridge.PortalMessageBuffer), - matrixMessages: make(chan PortalMatrixMessage, br.Config.Bridge.PortalMessageBuffer), - mediaRetries: make(chan PortalMediaRetry, br.Config.Bridge.PortalMessageBuffer), + events: make(chan *PortalEvent, br.Config.Bridge.PortalMessageBuffer), mediaErrorCache: make(map[types.MessageID]*FailedMediaMeta), } @@ -228,6 +236,12 @@ type fakeMessage struct { Important bool } +type PortalEvent struct { + Message *PortalMessage + MatrixMessage *PortalMatrixMessage + MediaRetry *PortalMediaRetry +} + type PortalMessage struct { evt *events.Message undecryptable *events.UndecryptableMessage @@ -256,7 +270,9 @@ type Portal struct { *database.Portal bridge *WABridge - log log.Logger + // Deprecated: use zerolog + log log.Logger + zlog zerolog.Logger roomCreateLock sync.Mutex encryptLock sync.Mutex @@ -273,18 +289,54 @@ type Portal struct { currentlyTyping []id.UserID currentlyTypingLock sync.Mutex - messages chan PortalMessage - matrixMessages chan PortalMatrixMessage - mediaRetries chan PortalMediaRetry + events chan *PortalEvent mediaErrorCache map[types.MessageID]*FailedMediaMeta + galleryCache []*event.MessageEventContent + galleryCacheRootEvent id.EventID + galleryCacheStart time.Time + galleryCacheReplyTo *ReplyInfo + galleryCacheSender types.JID + currentlySleepingToDelete sync.Map relayUser *User parentPortal *Portal } +const GalleryMaxTime = 10 * time.Minute + +func (portal *Portal) stopGallery() { + if portal.galleryCache != nil { + portal.galleryCache = nil + portal.galleryCacheSender = types.EmptyJID + portal.galleryCacheReplyTo = nil + portal.galleryCacheStart = time.Time{} + portal.galleryCacheRootEvent = "" + } +} + +func (portal *Portal) startGallery(evt *events.Message, msg *ConvertedMessage) { + portal.galleryCache = []*event.MessageEventContent{msg.Content} + portal.galleryCacheSender = evt.Info.Sender.ToNonAD() + portal.galleryCacheReplyTo = msg.ReplyTo + portal.galleryCacheStart = time.Now() +} + +func (portal *Portal) extendGallery(msg *ConvertedMessage) int { + portal.galleryCache = append(portal.galleryCache, msg.Content) + msg.Content = &event.MessageEventContent{ + MsgType: event.MsgBeeperGallery, + Body: "Sent a gallery", + BeeperGalleryImages: portal.galleryCache, + } + msg.Content.SetEdit(portal.galleryCacheRootEvent) + // Don't set the gallery images in the edit fallback + msg.Content.BeeperGalleryImages = nil + return len(portal.galleryCache) - 1 +} + var ( _ bridge.Portal = (*Portal)(nil) _ bridge.ReadReceiptHandlingPortal = (*Portal)(nil) @@ -293,14 +345,14 @@ var ( _ bridge.TypingPortal = (*Portal)(nil) ) -func (portal *Portal) handleMessageLoopItem(msg PortalMessage) { +func (portal *Portal) handleWhatsAppMessageLoopItem(msg *PortalMessage) { if len(portal.MXID) == 0 { if msg.fake == nil && msg.undecryptable == nil && (msg.evt == nil || !containsSupportedMessage(msg.evt.Message)) { portal.log.Debugln("Not creating portal room for incoming message: message is not a chat message") return } portal.log.Debugln("Creating Matrix room from incoming message") - err := portal.CreateMatrixRoom(msg.source, nil, false, true) + err := portal.CreateMatrixRoom(msg.source, nil, nil, false, true) if err != nil { portal.log.Errorln("Failed to create portal room:", err) return @@ -310,12 +362,14 @@ func (portal *Portal) handleMessageLoopItem(msg PortalMessage) { defer portal.latestEventBackfillLock.Unlock() switch { case msg.evt != nil: - portal.handleMessage(msg.source, msg.evt) + portal.handleMessage(msg.source, msg.evt, false) case msg.receipt != nil: portal.handleReceipt(msg.receipt, msg.source) case msg.undecryptable != nil: + portal.stopGallery() portal.handleUndecryptableMessage(msg.source, msg.undecryptable) case msg.fake != nil: + portal.stopGallery() msg.fake.ID = "FAKE::" + msg.fake.ID portal.handleFakeMessage(*msg.fake) default: @@ -323,7 +377,7 @@ func (portal *Portal) handleMessageLoopItem(msg PortalMessage) { } } -func (portal *Portal) handleMatrixMessageLoopItem(msg PortalMatrixMessage) { +func (portal *Portal) handleMatrixMessageLoopItem(msg *PortalMatrixMessage) { portal.latestEventBackfillLock.Lock() defer portal.latestEventBackfillLock.Unlock() evtTS := time.UnixMilli(msg.evt.Timestamp) @@ -348,7 +402,38 @@ func (portal *Portal) handleMatrixMessageLoopItem(msg PortalMatrixMessage) { } } +func (portal *Portal) handleDeliveryReceipt(receipt *events.Receipt, source *User) { + if !portal.IsPrivateChat() { + return + } + for _, msgID := range receipt.MessageIDs { + msg := portal.bridge.DB.Message.GetByJID(portal.Key, msgID) + if msg == nil || msg.IsFakeMXID() { + continue + } + if msg.Sender == source.JID { + portal.bridge.SendRawMessageCheckpoint(&status.MessageCheckpoint{ + EventID: msg.MXID, + RoomID: portal.MXID, + Step: status.MsgStepRemote, + Timestamp: jsontime.UM(receipt.Timestamp), + Status: status.MsgStatusDelivered, + ReportedBy: status.MsgReportedByBridge, + }) + portal.sendStatusEvent(msg.MXID, "", nil, &[]id.UserID{portal.MainIntent().UserID}) + } + } +} + func (portal *Portal) handleReceipt(receipt *events.Receipt, source *User) { + if receipt.Sender.Server != types.DefaultUserServer { + // TODO handle lids + return + } + if receipt.Type == types.ReceiptTypeDelivered { + portal.handleDeliveryReceipt(receipt, source) + return + } // The order of the message ID array depends on the sender's platform, so we just have to find // the last message based on timestamp. Also, timestamps only have second precision, so if // there are many messages at the same second just mark them all as read, because we don't @@ -387,13 +472,34 @@ func (portal *Portal) handleReceipt(receipt *events.Receipt, source *User) { func (portal *Portal) handleMessageLoop() { for { - select { - case msg := <-portal.messages: - portal.handleMessageLoopItem(msg) - case msg := <-portal.matrixMessages: - portal.handleMatrixMessageLoopItem(msg) - case retry := <-portal.mediaRetries: - portal.handleMediaRetry(retry.evt, retry.source) + portal.handleOneMessageLoopItem() + } +} + +func (portal *Portal) handleOneMessageLoopItem() { + defer func() { + if err := recover(); err != nil { + logEvt := portal.zlog.WithLevel(zerolog.FatalLevel). + Str(zerolog.ErrorStackFieldName, string(debug.Stack())) + actualErr, ok := err.(error) + if ok { + logEvt = logEvt.Err(actualErr) + } else { + logEvt = logEvt.Any(zerolog.ErrorFieldName, err) + } + logEvt.Msg("Portal message handler panicked") + } + }() + select { + case msg := <-portal.events: + if msg.Message != nil { + portal.handleWhatsAppMessageLoopItem(msg.Message) + } else if msg.MatrixMessage != nil { + portal.handleMatrixMessageLoopItem(msg.MatrixMessage) + } else if msg.MediaRetry != nil { + portal.handleMediaRetry(msg.MediaRetry.evt, msg.MediaRetry.source) + } else { + portal.log.Warn("Portal event loop returned an event without any data") } } } @@ -403,7 +509,7 @@ func containsSupportedMessage(waMsg *waProto.Message) bool { return false } return waMsg.Conversation != nil || waMsg.ExtendedTextMessage != nil || waMsg.ImageMessage != nil || - waMsg.StickerMessage != nil || waMsg.AudioMessage != nil || waMsg.VideoMessage != nil || + waMsg.StickerMessage != nil || waMsg.AudioMessage != nil || waMsg.VideoMessage != nil || waMsg.PtvMessage != nil || waMsg.DocumentMessage != nil || waMsg.ContactMessage != nil || waMsg.LocationMessage != nil || waMsg.LiveLocationMessage != nil || waMsg.GroupInviteMessage != nil || waMsg.ContactsArrayMessage != nil || waMsg.HighlyStructuredMessage != nil || waMsg.TemplateMessage != nil || waMsg.TemplateButtonReplyMessage != nil || @@ -422,6 +528,8 @@ func getMessageType(waMsg *waProto.Message) string { return fmt.Sprintf("sticker %s", waMsg.GetStickerMessage().GetMimetype()) case waMsg.VideoMessage != nil: return fmt.Sprintf("video %s", waMsg.GetVideoMessage().GetMimetype()) + case waMsg.PtvMessage != nil: + return fmt.Sprintf("round video %s", waMsg.GetPtvMessage().GetMimetype()) case waMsg.AudioMessage != nil: return fmt.Sprintf("audio %s", waMsg.GetAudioMessage().GetMimetype()) case waMsg.DocumentMessage != nil: @@ -571,6 +679,8 @@ func (portal *Portal) convertMessage(intent *appservice.IntentAPI, source *User, return portal.convertMediaMessage(intent, source, info, waMsg.GetStickerMessage(), "sticker", isBackfill) case waMsg.VideoMessage != nil: return portal.convertMediaMessage(intent, source, info, waMsg.GetVideoMessage(), "video attachment", isBackfill) + case waMsg.PtvMessage != nil: + return portal.convertMediaMessage(intent, source, info, waMsg.GetPtvMessage(), "video message", isBackfill) case waMsg.AudioMessage != nil: typeName := "audio attachment" if waMsg.GetAudioMessage().GetPtt() { @@ -605,6 +715,23 @@ func (portal *Portal) convertMessage(intent *appservice.IntentAPI, source *User, } } +func (portal *Portal) implicitlyEnableDisappearingMessages(timer time.Duration) { + portal.ExpirationTime = uint32(timer.Seconds()) + portal.Update(nil) + intent := portal.MainIntent() + if portal.Encrypted { + intent = portal.bridge.Bot + } + duration := formatDuration(time.Duration(portal.ExpirationTime) * time.Second) + _, err := portal.sendMessage(intent, event.EventMessage, &event.MessageEventContent{ + MsgType: event.MsgNotice, + Body: fmt.Sprintf("Automatically enabled disappearing message timer (%s) because incoming message is disappearing", duration), + }, nil, 0) + if err != nil { + portal.zlog.Warn().Err(err).Msg("Failed to send notice about implicit disappearing timer") + } +} + func (portal *Portal) UpdateGroupDisappearingMessages(sender *types.JID, timestamp time.Time, timer uint32) { if portal.ExpirationTime == timer { return @@ -612,7 +739,7 @@ func (portal *Portal) UpdateGroupDisappearingMessages(sender *types.JID, timesta portal.ExpirationTime = timer portal.Update(nil) intent := portal.MainIntent() - if sender != nil { + if sender != nil && sender.Server == types.DefaultUserServer { intent = portal.bridge.GetPuppetByJID(sender.ToNonAD()).IntentFor(portal) } else { sender = &types.EmptyJID @@ -658,7 +785,7 @@ func (portal *Portal) handleUndecryptableMessage(source *User, evt *events.Undec if evt.IsUnavailable { metricType = "unavailable" } - Segment.Track(source.MXID, "WhatsApp undecryptable message", map[string]interface{}{ + Analytics.Track(source.MXID, "WhatsApp undecryptable message", map[string]interface{}{ "messageID": evt.Info.ID, "undecryptableType": metricType, }) @@ -672,7 +799,7 @@ func (portal *Portal) handleUndecryptableMessage(source *User, evt *events.Undec portal.log.Errorfln("Failed to send decryption error of %s to Matrix: %v", evt.Info.ID, err) return } - portal.finishHandling(nil, &evt.Info, resp.EventID, intent.UserID, database.MsgUnknown, database.MsgErrDecryptionFailed) + portal.finishHandling(nil, &evt.Info, resp.EventID, intent.UserID, database.MsgUnknown, 0, database.MsgErrDecryptionFailed) } func (portal *Portal) handleFakeMessage(msg fakeMessage) { @@ -683,6 +810,11 @@ func (portal *Portal) handleFakeMessage(msg fakeMessage) { portal.log.Debugfln("Not handling %s (fake): message is duplicate", msg.ID) return } + if msg.Sender.Server != types.DefaultUserServer { + portal.log.Debugfln("Not handling %s (fake): message is from a lid user (%s)", msg.ID, msg.Sender) + // TODO handle lids + return + } intent := portal.bridge.GetPuppetByJID(msg.Sender).IntentFor(portal) if !intent.IsCustomPuppet && portal.IsPrivateChat() && msg.Sender.User == portal.Key.Receiver.User && portal.Key.Receiver != portal.Key.JID { portal.log.Debugfln("Not handling %s (fake): user doesn't have double puppeting enabled", msg.ID) @@ -705,11 +837,11 @@ func (portal *Portal) handleFakeMessage(msg fakeMessage) { MessageSource: types.MessageSource{ Sender: msg.Sender, }, - }, resp.EventID, intent.UserID, database.MsgFake, database.MsgNoError) + }, resp.EventID, intent.UserID, database.MsgFake, 0, database.MsgNoError) } } -func (portal *Portal) handleMessage(source *User, evt *events.Message) { +func (portal *Portal) handleMessage(source *User, evt *events.Message, historical bool) { if len(portal.MXID) == 0 { portal.log.Warnln("handleMessage called even though portal.MXID is empty") return @@ -725,10 +857,15 @@ func (portal *Portal) handleMessage(source *User, evt *events.Message) { existingMsg := portal.bridge.DB.Message.GetByJID(portal.Key, msgID) if existingMsg != nil { if existingMsg.Error == database.MsgErrDecryptionFailed { - Segment.Track(source.MXID, "WhatsApp undecryptable message resolved", map[string]interface{}{ - "messageID": evt.Info.ID, + resolveType := "sender" + if evt.UnavailableRequestID != "" { + resolveType = "phone" + } + Analytics.Track(source.MXID, "WhatsApp undecryptable message resolved", map[string]interface{}{ + "messageID": evt.Info.ID, + "resolveType": resolveType, }) - portal.log.Debugfln("Got decryptable version of previously undecryptable message %s (%s)", msgID, msgType) + portal.log.Debugfln("Got decryptable version of previously undecryptable message %s (%s) via %s", msgID, msgType, resolveType) } else { portal.log.Debugfln("Not handling %s (%s): message is duplicate", msgID, msgType) return @@ -757,6 +894,25 @@ func (portal *Portal) handleMessage(source *User, evt *events.Message) { } converted := portal.convertMessage(intent, source, &evt.Info, evt.Message, false) if converted != nil { + isGalleriable := portal.bridge.Config.Bridge.BeeperGalleries && + (evt.Message.ImageMessage != nil || evt.Message.VideoMessage != nil) && + (portal.galleryCache == nil || + (evt.Info.Sender.ToNonAD() == portal.galleryCacheSender && + converted.ReplyTo.Equals(portal.galleryCacheReplyTo) && + time.Since(portal.galleryCacheStart) < GalleryMaxTime)) && + // Captions aren't allowed in galleries (this needs to be checked before the caption is merged) + converted.Caption == nil && + // Images can't be edited + editTargetMsg == nil + + if !historical && portal.IsPrivateChat() && evt.Info.Sender.Device == 0 && converted.ExpiresIn > 0 && portal.ExpirationTime == 0 { + portal.zlog.Info(). + Str("timer", converted.ExpiresIn.String()). + Str("sender_jid", evt.Info.Sender.String()). + Str("message_id", evt.Info.ID). + Msg("Implicitly enabling disappearing messages as incoming message is disappearing") + portal.implicitlyEnableDisappearingMessages(converted.ExpiresIn) + } if evt.Info.IsIncomingBroadcast() { if converted.Extra == nil { converted.Extra = map[string]interface{}{} @@ -772,13 +928,27 @@ func (portal *Portal) handleMessage(source *User, evt *events.Message) { portal.MarkDisappearing(nil, existingMsg.MXID, converted.ExpiresIn, evt.Info.Timestamp) converted.Content.SetEdit(existingMsg.MXID) } else if converted.ReplyTo != nil { - portal.SetReply(converted.Content, converted.ReplyTo, false) + portal.SetReply(evt.Info.ID, converted.Content, converted.ReplyTo, false) } dbMsgType := database.MsgNormal if editTargetMsg != nil { dbMsgType = database.MsgEdit converted.Content.SetEdit(editTargetMsg.MXID) } + galleryStarted := false + var galleryPart int + if isGalleriable { + if portal.galleryCache == nil { + portal.startGallery(evt, converted) + galleryStarted = true + } else { + galleryPart = portal.extendGallery(converted) + dbMsgType = database.MsgBeeperGallery + } + } else if editTargetMsg == nil { + // Stop collecting a gallery (except if it's an edit) + portal.stopGallery() + } resp, err := portal.sendMessage(converted.Intent, converted.Type, converted.Content, converted.Extra, evt.Info.Timestamp.UnixMilli()) if err != nil { portal.log.Errorfln("Failed to send %s to Matrix: %v", msgID, err) @@ -788,6 +958,11 @@ func (portal *Portal) handleMessage(source *User, evt *events.Message) { } eventID = resp.EventID lastEventID = eventID + if galleryStarted { + portal.galleryCacheRootEvent = eventID + } else if galleryPart != 0 { + eventID = portal.galleryCacheRootEvent + } } // TODO figure out how to handle captions with undecryptable messages turning decryptable if converted.Caption != nil && existingMsg == nil && editTargetMsg == nil { @@ -820,7 +995,7 @@ func (portal *Portal) handleMessage(source *User, evt *events.Message) { } } if len(eventID) != 0 { - portal.finishHandling(existingMsg, &evt.Info, eventID, intent.UserID, dbMsgType, converted.Error) + portal.finishHandling(existingMsg, &evt.Info, eventID, intent.UserID, dbMsgType, galleryPart, converted.Error) } } else if msgType == "reaction" || msgType == "encrypted reaction" { if evt.Message.GetEncReactionMessage() != nil { @@ -865,12 +1040,13 @@ func (portal *Portal) isRecentlyHandled(id types.MessageID, error database.Messa return false } -func (portal *Portal) markHandled(txn dbutil.Transaction, msg *database.Message, info *types.MessageInfo, mxid id.EventID, senderMXID id.UserID, isSent, recent bool, msgType database.MessageType, errType database.MessageErrorType) *database.Message { +func (portal *Portal) markHandled(txn dbutil.Transaction, msg *database.Message, info *types.MessageInfo, mxid id.EventID, senderMXID id.UserID, isSent, recent bool, msgType database.MessageType, galleryPart int, errType database.MessageErrorType) *database.Message { if msg == nil { msg = portal.bridge.DB.Message.New() msg.Chat = portal.Key msg.JID = info.ID msg.MXID = mxid + msg.GalleryPart = galleryPart msg.Timestamp = info.Timestamp msg.Sender = info.Sender msg.SenderMXID = senderMXID @@ -913,6 +1089,9 @@ func (portal *Portal) getMessagePuppet(user *User, info *types.MessageInfo) (pup } func (portal *Portal) getMessageIntent(user *User, info *types.MessageInfo, msgType string) *appservice.IntentAPI { + if portal.IsNewsletter() && info.Sender == info.Chat { + return portal.MainIntent() + } puppet := portal.getMessagePuppet(user, info) if puppet == nil { return nil @@ -925,8 +1104,8 @@ func (portal *Portal) getMessageIntent(user *User, info *types.MessageInfo, msgT return intent } -func (portal *Portal) finishHandling(existing *database.Message, message *types.MessageInfo, mxid id.EventID, senderMXID id.UserID, msgType database.MessageType, errType database.MessageErrorType) { - portal.markHandled(nil, existing, message, mxid, senderMXID, true, true, msgType, errType) +func (portal *Portal) finishHandling(existing *database.Message, message *types.MessageInfo, mxid id.EventID, senderMXID id.UserID, msgType database.MessageType, galleryPart int, errType database.MessageErrorType) { + portal.markHandled(nil, existing, message, mxid, senderMXID, true, true, msgType, galleryPart, errType) portal.sendDeliveryReceipt(mxid) var suffix string if errType == database.MsgErrDecryptionFailed { @@ -997,6 +1176,9 @@ func (portal *Portal) syncParticipant(source *User, participant types.GroupParti } func (portal *Portal) SyncParticipants(source *User, metadata *types.GroupInfo) ([]id.UserID, *event.PowerLevelsEventContent) { + if portal.IsNewsletter() { + return nil, nil + } changed := false var levels *event.PowerLevelsEventContent var err error @@ -1013,6 +1195,11 @@ func (portal *Portal) SyncParticipants(source *User, metadata *types.GroupInfo) participantMap := make(map[types.JID]bool) userIDs := make([]id.UserID, 0, len(metadata.Participants)) for _, participant := range metadata.Participants { + if participant.JID.IsEmpty() || participant.JID.Server != types.DefaultUserServer { + wg.Done() + // TODO handle lids + continue + } portal.log.Debugfln("Syncing participant %s (admin: %t)", participant.JID, participant.IsAdmin) participantMap[participant.JID] = true puppet := portal.bridge.GetPuppetByJID(participant.JID) @@ -1070,6 +1257,18 @@ func reuploadAvatar(intent *appservice.IntentAPI, url string) (id.ContentURI, er return resp.ContentURI, nil } +func (user *User) reuploadAvatarDirectPath(intent *appservice.IntentAPI, directPath string) (id.ContentURI, error) { + data, err := user.Client.DownloadMediaWithPath(directPath, nil, nil, nil, 0, "", "") + if err != nil { + return id.ContentURI{}, fmt.Errorf("failed to download avatar: %w", err) + } + resp, err := intent.UploadBytes(data, http.DetectContentType(data)) + if err != nil { + return id.ContentURI{}, fmt.Errorf("failed to upload avatar to Matrix: %w", err) + } + return resp.ContentURI, nil +} + func (user *User) updateAvatar(jid types.JID, isCommunity bool, avatarID *string, avatarURL *id.ContentURI, avatarSet *bool, log log.Logger, intent *appservice.IntentAPI) bool { currentID := "" if *avatarSet && *avatarID != "remove" && *avatarID != "unauthorized" { @@ -1104,14 +1303,23 @@ func (user *User) updateAvatar(jid types.JID, isCommunity bool, avatarID *string } if avatar.ID == *avatarID && *avatarSet { return false - } else if len(avatar.URL) == 0 { + } else if len(avatar.URL) == 0 && len(avatar.DirectPath) == 0 { log.Warnln("Didn't get URL in response to avatar query") return false } else if avatar.ID != *avatarID || avatarURL.IsEmpty() { - url, err := reuploadAvatar(intent, avatar.URL) - if err != nil { - log.Warnln("Failed to reupload avatar:", err) - return false + var url id.ContentURI + if len(avatar.URL) > 0 { + url, err = reuploadAvatar(intent, avatar.URL) + if err != nil { + log.Warnln("Failed to reupload avatar:", err) + return false + } + } else { + url, err = user.reuploadAvatarDirectPath(intent, avatar.DirectPath) + if err != nil { + log.Warnln("Failed to reupload avatar:", err) + return false + } } *avatarURL = url } @@ -1121,10 +1329,61 @@ func (user *User) updateAvatar(jid types.JID, isCommunity bool, avatarID *string return true } +func (portal *Portal) UpdateNewsletterAvatar(user *User, meta *types.NewsletterMetadata) bool { + portal.avatarLock.Lock() + defer portal.avatarLock.Unlock() + var picID string + picture := meta.ThreadMeta.Picture + if picture == nil { + picID = meta.ThreadMeta.Preview.ID + } else { + picID = picture.ID + } + if picID == "" { + picID = "remove" + } + if portal.Avatar != picID || !portal.AvatarSet { + if picID == "remove" { + portal.AvatarURL = id.ContentURI{} + } else if portal.Avatar != picID || portal.AvatarURL.IsEmpty() { + var err error + if picture == nil { + meta, err = user.Client.GetNewsletterInfo(portal.Key.JID) + if err != nil { + portal.log.Warnln("Failed to fetch full res avatar info for newsletter:", err) + return false + } + picture = meta.ThreadMeta.Picture + if picture == nil { + portal.log.Warnln("Didn't get full res avatar info for newsletter") + return false + } + picID = picture.ID + } + portal.AvatarURL, err = user.reuploadAvatarDirectPath(portal.MainIntent(), picture.DirectPath) + if err != nil { + portal.log.Warnln("Failed to reupload newsletter avatar:", err) + return false + } + } + portal.Avatar = picID + portal.AvatarSet = false + return portal.setRoomAvatar(true, types.EmptyJID, true) + } + return false +} + func (portal *Portal) UpdateAvatar(user *User, setBy types.JID, updateInfo bool) bool { + if portal.IsNewsletter() { + return false + } portal.avatarLock.Lock() defer portal.avatarLock.Unlock() changed := user.updateAvatar(portal.Key.JID, portal.IsParent, &portal.Avatar, &portal.AvatarURL, &portal.AvatarSet, portal.log, portal.MainIntent()) + return portal.setRoomAvatar(changed, setBy, updateInfo) +} + +func (portal *Portal) setRoomAvatar(changed bool, setBy types.JID, updateInfo bool) bool { if !changed || portal.Avatar == "unauthorized" { if changed || updateInfo { portal.Update(nil) @@ -1134,7 +1393,7 @@ func (portal *Portal) UpdateAvatar(user *User, setBy types.JID, updateInfo bool) if len(portal.MXID) > 0 { intent := portal.MainIntent() - if !setBy.IsEmpty() { + if !setBy.IsEmpty() && setBy.Server == types.DefaultUserServer { intent = portal.bridge.GetPuppetByJID(setBy).IntentFor(portal) } _, err := intent.SetRoomAvatar(portal.MXID, portal.AvatarURL) @@ -1172,7 +1431,7 @@ func (portal *Portal) UpdateName(name string, setBy types.JID, updateInfo bool) portal.UpdateBridgeInfo() } else if len(portal.MXID) > 0 { intent := portal.MainIntent() - if !setBy.IsEmpty() { + if !setBy.IsEmpty() && setBy.Server == types.DefaultUserServer { intent = portal.bridge.GetPuppetByJID(setBy).IntentFor(portal) } _, err := intent.SetRoomName(portal.MXID, name) @@ -1201,7 +1460,7 @@ func (portal *Portal) UpdateTopic(topic string, setBy types.JID, updateInfo bool portal.TopicSet = false intent := portal.MainIntent() - if !setBy.IsEmpty() { + if !setBy.IsEmpty() && setBy.Server == types.DefaultUserServer { intent = portal.bridge.GetPuppetByJID(setBy).IntentFor(portal) } _, err := intent.SetRoomTopic(portal.MXID, topic) @@ -1222,6 +1481,21 @@ func (portal *Portal) UpdateTopic(topic string, setBy types.JID, updateInfo bool return false } +func newsletterToGroupInfo(meta *types.NewsletterMetadata) *types.GroupInfo { + var out types.GroupInfo + out.JID = meta.ID + out.Name = meta.ThreadMeta.Name.Text + out.NameSetAt = meta.ThreadMeta.Name.UpdateTime.Time + out.Topic = meta.ThreadMeta.Description.Text + out.TopicSetAt = meta.ThreadMeta.Description.UpdateTime.Time + out.TopicID = meta.ThreadMeta.Description.ID + out.GroupCreated = meta.ThreadMeta.CreationTime.Time + out.IsAnnounce = true + out.IsLocked = true + out.IsIncognito = true + return &out +} + func (portal *Portal) UpdateParentGroup(source *User, parent types.JID, updateInfo bool) bool { portal.parentGroupUpdateLock.Lock() defer portal.parentGroupUpdateLock.Unlock() @@ -1243,7 +1517,7 @@ func (portal *Portal) UpdateParentGroup(source *User, parent types.JID, updateIn return false } -func (portal *Portal) UpdateMetadata(user *User, groupInfo *types.GroupInfo) bool { +func (portal *Portal) UpdateMetadata(user *User, groupInfo *types.GroupInfo, newsletterMetadata *types.NewsletterMetadata) bool { if portal.IsPrivateChat() { return false } else if portal.IsStatusBroadcastList() { @@ -1266,6 +1540,17 @@ func (portal *Portal) UpdateMetadata(user *User, groupInfo *types.GroupInfo) boo //update = portal.UpdateTopic(BroadcastTopic, "", nil, false) || update return update } + if groupInfo == nil && portal.IsNewsletter() { + if newsletterMetadata == nil { + var err error + newsletterMetadata, err = user.Client.GetNewsletterInfo(portal.Key.JID) + if err != nil { + portal.zlog.Err(err).Msg("Failed to get newsletter info") + return false + } + } + groupInfo = newsletterToGroupInfo(newsletterMetadata) + } if groupInfo == nil { var err error groupInfo, err = user.Client.GetGroupInfo(portal.Key.JID) @@ -1294,6 +1579,9 @@ func (portal *Portal) UpdateMetadata(user *User, groupInfo *types.GroupInfo) boo portal.RestrictMessageSending(groupInfo.IsAnnounce) portal.RestrictMetadataChanges(groupInfo.IsLocked) + if newsletterMetadata != nil && newsletterMetadata.ViewerMeta != nil { + portal.PromoteNewsletterUser(user, newsletterMetadata.ViewerMeta.Role) + } return update } @@ -1302,7 +1590,7 @@ func (portal *Portal) ensureUserInvited(user *User) bool { return user.ensureInvited(portal.MainIntent(), portal.MXID, portal.IsPrivateChat()) } -func (portal *Portal) UpdateMatrixRoom(user *User, groupInfo *types.GroupInfo) bool { +func (portal *Portal) UpdateMatrixRoom(user *User, groupInfo *types.GroupInfo, newsletterMetadata *types.NewsletterMetadata) bool { if len(portal.MXID) == 0 { return false } @@ -1311,10 +1599,16 @@ func (portal *Portal) UpdateMatrixRoom(user *User, groupInfo *types.GroupInfo) b portal.ensureUserInvited(user) go portal.addToPersonalSpace(user) + if groupInfo == nil && newsletterMetadata != nil { + groupInfo = newsletterToGroupInfo(newsletterMetadata) + } + update := false - update = portal.UpdateMetadata(user, groupInfo) || update - if !portal.IsPrivateChat() && !portal.IsBroadcastList() { + update = portal.UpdateMetadata(user, groupInfo, newsletterMetadata) || update + if !portal.IsPrivateChat() && !portal.IsBroadcastList() && !portal.IsNewsletter() { update = portal.UpdateAvatar(user, types.EmptyJID, false) || update + } else if newsletterMetadata != nil { + update = portal.UpdateNewsletterAvatar(user, newsletterMetadata) || update } if update || portal.LastSync.Add(24*time.Hour).Before(time.Now()) { portal.LastSync = time.Now() @@ -1372,6 +1666,10 @@ func (portal *Portal) ChangeAdminStatus(jids []types.JID, setAdmin bool) id.Even } changed := portal.applyPowerLevelFixes(levels) for _, jid := range jids { + if jid.Server != types.DefaultUserServer { + // TODO handle lids + continue + } puppet := portal.bridge.GetPuppetByJID(jid) changed = levels.EnsureUserLevel(puppet.MXID, newLevel) || changed @@ -1417,6 +1715,35 @@ func (portal *Portal) RestrictMessageSending(restrict bool) id.EventID { } } +func (portal *Portal) PromoteNewsletterUser(user *User, role types.NewsletterRole) id.EventID { + levels, err := portal.MainIntent().PowerLevels(portal.MXID) + if err != nil { + levels = portal.GetBasePowerLevels() + } + + newLevel := 0 + switch role { + case types.NewsletterRoleAdmin: + newLevel = 50 + case types.NewsletterRoleOwner: + newLevel = 95 + } + + changed := portal.applyPowerLevelFixes(levels) + changed = levels.EnsureUserLevel(user.MXID, newLevel) || changed + if !changed { + return "" + } + + resp, err := portal.MainIntent().SetPowerLevels(portal.MXID, levels) + if err != nil { + portal.log.Errorln("Failed to change power levels:", err) + return "" + } else { + return resp.EventID + } +} + func (portal *Portal) RestrictMetadataChanges(restrict bool) id.EventID { levels, err := portal.MainIntent().PowerLevels(portal.MXID) if err != nil { @@ -1518,7 +1845,7 @@ func (portal *Portal) GetEncryptionEventContent() (evt *event.EncryptionEventCon return } -func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, isFullInfo, backfill bool) error { +func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, newsletterMetadata *types.NewsletterMetadata, isFullInfo, backfill bool) error { portal.roomCreateLock.Lock() defer portal.roomCreateLock.Unlock() if len(portal.MXID) > 0 { @@ -1565,7 +1892,18 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, i portal.log.Debugln("Broadcast list is not yet supported, not creating room after all") return fmt.Errorf("broadcast list bridging is currently not supported") } else { - if groupInfo == nil || !isFullInfo { + if portal.IsNewsletter() { + if newsletterMetadata == nil { + var err error + newsletterMetadata, err = user.Client.GetNewsletterInfo(portal.Key.JID) + if err != nil { + return err + } + } + if groupInfo == nil { + groupInfo = newsletterToGroupInfo(newsletterMetadata) + } + } else if groupInfo == nil || !isFullInfo { foundInfo, err := user.Client.GetGroupInfo(portal.Key.JID) // Ensure that the user is actually a participant in the conversation @@ -1591,7 +1929,11 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, i portal.ExpirationTime = groupInfo.DisappearingTimer } } - portal.UpdateAvatar(user, types.EmptyJID, false) + if portal.IsNewsletter() { + portal.UpdateNewsletterAvatar(user, newsletterMetadata) + } else { + portal.UpdateAvatar(user, types.EmptyJID, false) + } } powerLevels := portal.GetBasePowerLevels() @@ -1606,6 +1948,14 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, i powerLevels.EnsureEventLevel(event.StateTopic, 50) } } + if newsletterMetadata != nil && newsletterMetadata.ViewerMeta != nil { + switch newsletterMetadata.ViewerMeta.Role { + case types.NewsletterRoleAdmin: + powerLevels.EnsureUserLevel(user.MXID, 50) + case types.NewsletterRoleOwner: + powerLevels.EnsureUserLevel(user.MXID, 95) + } + } bridgeInfoStateKey, bridgeInfo := portal.getBridgeInfo() @@ -1671,10 +2021,10 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, i }, }) } - autoJoinInvites := portal.bridge.Config.Homeserver.Software == bridgeconfig.SoftwareHungry + autoJoinInvites := portal.bridge.SpecVersions.Supports(mautrix.BeeperFeatureAutojoinInvites) if autoJoinInvites { portal.log.Debugfln("Hungryserv mode: adding all group members in create request") - if groupInfo != nil { + if groupInfo != nil && !portal.IsNewsletter() { // TODO non-hungryserv could also include all members in invites, and then send joins manually? participants, powerLevels := portal.SyncParticipants(user, groupInfo) invite = append(invite, participants...) @@ -1701,6 +2051,16 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, i if !portal.shouldSetDMRoomMetadata() { req.Name = "" } + legacyBackfill := user.bridge.Config.Bridge.HistorySync.Backfill && backfill && !user.bridge.SpecVersions.Supports(mautrix.BeeperFeatureBatchSending) + var backfillStarted bool + if legacyBackfill { + portal.latestEventBackfillLock.Lock() + defer func() { + if !backfillStarted { + portal.latestEventBackfillLock.Unlock() + } + }() + } resp, err := intent.CreateRoom(req) if err != nil { return err @@ -1732,7 +2092,7 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, i go portal.updateCommunitySpace(user, true, true) go portal.addToPersonalSpace(user) - if groupInfo != nil && !autoJoinInvites { + if !portal.IsNewsletter() && groupInfo != nil && !autoJoinInvites { portal.SyncParticipants(user, groupInfo) } //if broadcastMetadata != nil { @@ -1767,10 +2127,15 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, i } if user.bridge.Config.Bridge.HistorySync.Backfill && backfill { - portals := []*Portal{portal} - user.EnqueueImmediateBackfills(portals) - user.EnqueueDeferredBackfills(portals) - user.BackfillQueue.ReCheck() + if legacyBackfill { + backfillStarted = true + go portal.legacyBackfill(user) + } else { + portals := []*Portal{portal} + user.EnqueueImmediateBackfills(portals) + user.EnqueueDeferredBackfills(portals) + user.BackfillQueue.ReCheck() + } } return nil } @@ -1810,7 +2175,7 @@ func (portal *Portal) updateCommunitySpace(user *User, add, updateInfo bool) boo return false } portal.log.Debugfln("Creating portal for parent group %v", space.Key.JID) - err := space.CreateMatrixRoom(user, nil, false, false) + err := space.CreateMatrixRoom(user, nil, nil, false, false) if err != nil { portal.log.Debugfln("Failed to create portal for parent group: %v", err) return false @@ -1860,6 +2225,10 @@ func (portal *Portal) IsBroadcastList() bool { return portal.Key.JID.Server == types.BroadcastServer } +func (portal *Portal) IsNewsletter() bool { + return portal.Key.JID.Server == types.NewsletterServer +} + func (portal *Portal) IsStatusBroadcastList() bool { return portal.Key.JID == types.StatusBroadcastJID } @@ -1897,7 +2266,8 @@ func (portal *Portal) addReplyMention(content *event.MessageEventContent, sender if content.Mentions == nil || (sender.IsEmpty() && senderMXID == "") { return } - if senderMXID == "" { + // TODO handle lids + if senderMXID == "" && sender.Server == types.DefaultUserServer { if user := portal.bridge.GetUserByJID(sender); user != nil { senderMXID = user.MXID } else { @@ -1910,10 +2280,15 @@ func (portal *Portal) addReplyMention(content *event.MessageEventContent, sender } } -func (portal *Portal) SetReply(content *event.MessageEventContent, replyTo *ReplyInfo, isBackfill bool) bool { +func (portal *Portal) SetReply(msgID string, content *event.MessageEventContent, replyTo *ReplyInfo, isHungryBackfill bool) bool { if replyTo == nil { return false } + log := portal.zlog.With(). + Str("message_id", msgID). + Object("reply_to", replyTo). + Str("action", "SetReply"). + Logger() key := portal.Key targetPortal := portal defer func() { @@ -1936,10 +2311,12 @@ func (portal *Portal) SetReply(content *event.MessageEventContent, replyTo *Repl } message := portal.bridge.DB.Message.GetByJID(key, replyTo.MessageID) if message == nil || message.IsFakeMXID() { - if isBackfill && portal.bridge.Config.Homeserver.Software == bridgeconfig.SoftwareHungry { + if isHungryBackfill { content.RelatesTo = (&event.RelatesTo{}).SetReplyTo(targetPortal.deterministicEventID(replyTo.Sender, replyTo.MessageID, "")) portal.addReplyMention(content, replyTo.Sender, "") return true + } else { + log.Warn().Msg("Failed to find reply target") } return false } @@ -1950,14 +2327,14 @@ func (portal *Portal) SetReply(content *event.MessageEventContent, replyTo *Repl } evt, err := targetPortal.MainIntent().GetEvent(targetPortal.MXID, message.MXID) if err != nil { - portal.log.Warnln("Failed to get reply target:", err) + log.Warn().Err(err).Msg("Failed to get reply target event") return true } _ = evt.Content.ParseRaw(evt.Type) if evt.Type == event.EventEncrypted { decryptedEvt, err := portal.bridge.Crypto.Decrypt(evt) if err != nil { - portal.log.Warnln("Failed to decrypt reply target:", err) + log.Warn().Err(err).Msg("Failed to decrypt reply target event") } else { evt = decryptedEvt } @@ -1985,7 +2362,7 @@ func (portal *Portal) HandleMessageReaction(intent *appservice.IntentAPI, user * if err != nil { portal.log.Errorfln("Failed to redact reaction %s/%s from %s to %s: %v", existing.MXID, existing.JID, info.Sender, targetJID, err) } - portal.finishHandling(existingMsg, info, resp.EventID, intent.UserID, database.MsgReaction, database.MsgNoError) + portal.finishHandling(existingMsg, info, resp.EventID, intent.UserID, database.MsgReaction, 0, database.MsgNoError) existing.Delete() } else { target := portal.bridge.DB.Message.GetByJID(portal.Key, targetJID) @@ -2006,7 +2383,7 @@ func (portal *Portal) HandleMessageReaction(intent *appservice.IntentAPI, user * return } - portal.finishHandling(existingMsg, info, resp.EventID, intent.UserID, database.MsgReaction, database.MsgNoError) + portal.finishHandling(existingMsg, info, resp.EventID, intent.UserID, database.MsgReaction, 0, database.MsgNoError) portal.upsertReaction(nil, intent, target.JID, info.Sender, resp.EventID, info.ID) } } @@ -2094,6 +2471,21 @@ type ReplyInfo struct { Sender types.JID } +func (r *ReplyInfo) Equals(other *ReplyInfo) bool { + if r == nil { + return other == nil + } else if other == nil { + return false + } + return r.MessageID == other.MessageID && r.Chat == other.Chat && r.Sender == other.Sender +} + +func (r ReplyInfo) MarshalZerologObject(e *zerolog.Event) { + e.Str("message_id", r.MessageID) + e.Str("chat_jid", r.Chat.String()) + e.Str("sender_jid", r.Sender.String()) +} + type Replyable interface { GetStanzaId() string GetParticipant() string @@ -2282,7 +2674,7 @@ func (portal *Portal) convertListMessage(intent *appservice.IntentAPI, source *U body = fmt.Sprintf("%s\n\n%s", msg.GetTitle(), body) } } - randomID := util.RandomString(64) + randomID := random.String(64) body = fmt.Sprintf("%s\n%s", body, randomID) if msg.GetFooterText() != "" { body = fmt.Sprintf("%s\n\n%s", body, msg.GetFooterText()) @@ -2692,6 +3084,10 @@ func (portal *Portal) HandleWhatsAppKick(source *User, senderJID types.JID, jids sender := portal.bridge.GetPuppetByJID(senderJID) senderIntent := sender.IntentFor(portal) for _, jid := range jids { + if jid.Server != types.DefaultUserServer { + // TODO handle lids + continue + } //if source != nil && source.JID.User == jid.User { // portal.log.Debugln("Ignoring self-kick by", source.MXID) // continue @@ -2719,6 +3115,10 @@ func (portal *Portal) HandleWhatsAppInvite(source *User, senderJID *types.JID, j intent = sender.IntentFor(portal) } for _, jid := range jids { + if jid.Server != types.DefaultUserServer { + // TODO handle lids + continue + } puppet := portal.bridge.GetPuppetByJID(jid) puppet.SyncContact(source, true, false, "handling whatsapp invite") resp, err := intent.SendStateEvent(portal.MXID, event.StateMember, puppet.MXID.String(), &event.MemberEventContent{ @@ -2890,7 +3290,7 @@ func (portal *Portal) convertMediaMessageContent(intent *appservice.IntentAPI, m content.Body = mimeClass } - content.Body += util.ExtensionFromMimetype(msg.GetMimetype()) + content.Body += exmime.ExtensionFromMimetype(msg.GetMimetype()) } msgWithDuration, ok := msg.(MediaMessageWithDuration) @@ -2991,7 +3391,7 @@ func (portal *Portal) convertMediaMessageContent(intent *appservice.IntentAPI, m "duration": int(audioMessage.GetSeconds()) * 1000, "waveform": waveform, } - if audioMessage.GetPtt() { + if audioMessage.GetPtt() || audioMessage.GetMimetype() == "audio/ogg; codecs/opus" { extraContent["org.matrix.msc3245.voice"] = map[string]interface{}{} } } @@ -3027,7 +3427,7 @@ func (portal *Portal) uploadMedia(intent *appservice.IntentAPI, data []byte, con } var mxc id.ContentURI if portal.bridge.Config.Homeserver.AsyncMedia { - uploaded, err := intent.UnstableUploadAsync(req) + uploaded, err := intent.UploadAsync(req) if err != nil { return err } @@ -3192,6 +3592,10 @@ func (portal *Portal) handleMediaRetry(retry *events.MediaRetry, source *User) { } else { puppet = portal.bridge.GetPuppetByJID(retry.SenderID) } + if puppet == nil { + // TODO handle lids? + return + } intent := puppet.IntentFor(portal) retryData, err := whatsmeow.DecryptMediaRetryNotification(retry, meta.Media.Key) @@ -3408,7 +3812,7 @@ func (portal *Portal) convertToWebP(img []byte) ([]byte, error) { } var webpBuffer bytes.Buffer - if err = webp.Encode(&webpBuffer, decodedImg, nil); err != nil { + if err = cwebp.Encode(&webpBuffer, decodedImg, nil); err != nil { return img, fmt.Errorf("failed to encode webp image: %w", err) } @@ -3442,15 +3846,18 @@ func (portal *Portal) preprocessMatrixMedia(ctx context.Context, sender *User, r } data, err := portal.MainIntent().DownloadBytesContext(ctx, mxc) if err != nil { - return nil, util.NewDualError(errMediaDownloadFailed, err) + return nil, exerrors.NewDualError(errMediaDownloadFailed, err) } if file != nil { err = file.DecryptInPlace(data) if err != nil { - return nil, util.NewDualError(errMediaDecryptFailed, err) + return nil, exerrors.NewDualError(errMediaDecryptFailed, err) } } mimeType := content.GetInfo().MimeType + if mimeType == "" { + content.Info.MimeType = "application/octet-stream" + } var convertErr error // Allowed mime types from https://developers.facebook.com/docs/whatsapp/on-premises/reference/media switch { @@ -3502,15 +3909,20 @@ func (portal *Portal) preprocessMatrixMedia(ctx context.Context, sender *User, r } if convertErr != nil { if content.Info.MimeType != mimeType || data == nil { - return nil, util.NewDualError(fmt.Errorf("%w (%s to %s)", errMediaConvertFailed, mimeType, content.Info.MimeType), convertErr) + return nil, exerrors.NewDualError(fmt.Errorf("%w (%s to %s)", errMediaConvertFailed, mimeType, content.Info.MimeType), convertErr) } else { // If the mime type didn't change and the errored conversion function returned the original data, just log a warning and continue portal.log.Warnfln("Failed to re-encode %s media: %v, continuing with original file", mimeType, convertErr) } } - uploadResp, err := sender.Client.Upload(ctx, data, mediaType) + var uploadResp whatsmeow.UploadResponse + if portal.Key.JID.Server == types.NewsletterServer { + uploadResp, err = sender.Client.UploadNewsletter(ctx, data, mediaType) + } else { + uploadResp, err = sender.Client.Upload(ctx, data, mediaType) + } if err != nil { - return nil, util.NewDualError(errMediaWhatsAppUploadFailed, err) + return nil, exerrors.NewDualError(errMediaWhatsAppUploadFailed, err) } // Audio doesn't have thumbnails @@ -3754,7 +4166,6 @@ func (portal *Portal) convertMatrixPollStart(_ context.Context, sender *User, ev if maxAnswers >= len(content.PollStart.Answers) || maxAnswers < 0 { maxAnswers = 0 } - fmt.Printf("%+v\n", content.PollStart) ctxInfo := portal.generateContextInfo(content.RelatesTo) var question string question, ctxInfo.MentionedJid = portal.msc1767ToWhatsApp(content.PollStart.Question, true) @@ -3795,7 +4206,7 @@ func (portal *Portal) generateContextInfo(relatesTo *event.RelatesTo) *waProto.C replyToID := relatesTo.GetReplyTo() if len(replyToID) > 0 { replyToMsg := portal.bridge.DB.Message.GetByMXID(replyToID) - if replyToMsg != nil && !replyToMsg.IsFakeJID() && (replyToMsg.Type == database.MsgNormal || replyToMsg.Type == database.MsgMatrixPoll) { + if replyToMsg != nil && !replyToMsg.IsFakeJID() && (replyToMsg.Type == database.MsgNormal || replyToMsg.Type == database.MsgMatrixPoll || replyToMsg.Type == database.MsgBeeperGallery) { ctxInfo.StanzaId = &replyToMsg.JID ctxInfo.Participant = proto.String(replyToMsg.Sender.ToNonAD().String()) // Using blank content here seems to work fine on all official WhatsApp apps. @@ -3815,6 +4226,10 @@ func (portal *Portal) generateContextInfo(relatesTo *event.RelatesTo) *waProto.C type extraConvertMeta struct { PollOptions map[[32]byte]string EditRootMsg *database.Message + + GalleryExtraParts []*waProto.Message + + MediaHandle string } func getEditError(rootMsg *database.Message, editer *User) error { @@ -3850,6 +4265,9 @@ func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, ev return nil, sender, extraMeta, errUserNotLoggedIn } sender = portal.GetRelayUser() + if !sender.IsLoggedIn() { + return nil, sender, extraMeta, errRelaybotNotLoggedIn + } isRelay = true } var editRootMsg *database.Message @@ -3913,6 +4331,7 @@ func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, ev if media == nil { return nil, sender, extraMeta, err } + extraMeta.MediaHandle = media.Handle ctxInfo.MentionedJid = media.MentionedJIDs msg.ImageMessage = &waProto.ImageMessage{ ContextInfo: ctxInfo, @@ -3926,11 +4345,46 @@ func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, ev FileSha256: media.FileSHA256, FileLength: proto.Uint64(uint64(media.FileLength)), } + case event.MsgBeeperGallery: + if isRelay { + return nil, sender, extraMeta, errGalleryRelay + } else if content.BeeperGalleryCaption != "" { + return nil, sender, extraMeta, errGalleryCaption + } else if portal.Key.JID.Server == types.NewsletterServer { + // We don't handle the media handles properly for multiple messages + return nil, sender, extraMeta, fmt.Errorf("can't send gallery to newsletter") + } + for i, part := range content.BeeperGalleryImages { + // TODO support videos + media, err := portal.preprocessMatrixMedia(ctx, sender, false, part, evt.ID, whatsmeow.MediaImage) + if media == nil { + return nil, sender, extraMeta, fmt.Errorf("failed to handle image #%d: %w", i+1, err) + } + imageMsg := &waProto.ImageMessage{ + ContextInfo: ctxInfo, + JpegThumbnail: media.Thumbnail, + Url: &media.URL, + DirectPath: &media.DirectPath, + MediaKey: media.MediaKey, + Mimetype: &part.GetInfo().MimeType, + FileEncSha256: media.FileEncSHA256, + FileSha256: media.FileSHA256, + FileLength: proto.Uint64(uint64(media.FileLength)), + } + if i == 0 { + msg.ImageMessage = imageMsg + } else { + extraMeta.GalleryExtraParts = append(extraMeta.GalleryExtraParts, &waProto.Message{ + ImageMessage: imageMsg, + }) + } + } case event.MessageType(event.EventSticker.Type): media, err := portal.preprocessMatrixMedia(ctx, sender, relaybotFormatted, content, evt.ID, whatsmeow.MediaImage) if media == nil { return nil, sender, extraMeta, err } + extraMeta.MediaHandle = media.Handle ctxInfo.MentionedJid = media.MentionedJIDs msg.StickerMessage = &waProto.StickerMessage{ ContextInfo: ctxInfo, @@ -3950,6 +4404,7 @@ func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, ev return nil, sender, extraMeta, err } duration := uint32(content.GetInfo().Duration / 1000) + extraMeta.MediaHandle = media.Handle ctxInfo.MentionedJid = media.MentionedJIDs msg.VideoMessage = &waProto.VideoMessage{ ContextInfo: ctxInfo, @@ -3970,6 +4425,7 @@ func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, ev if media == nil { return nil, sender, extraMeta, err } + extraMeta.MediaHandle = media.Handle duration := uint32(content.GetInfo().Duration / 1000) msg.AudioMessage = &waProto.AudioMessage{ ContextInfo: ctxInfo, @@ -3994,6 +4450,7 @@ func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, ev if media == nil { return nil, sender, extraMeta, err } + extraMeta.MediaHandle = media.Handle msg.DocumentMessage = &waProto.DocumentMessage{ ContextInfo: ctxInfo, Caption: &media.Caption, @@ -4055,7 +4512,7 @@ func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, ev func (portal *Portal) generateMessageInfo(sender *User) *types.MessageInfo { return &types.MessageInfo{ - ID: whatsmeow.GenerateMessageID(), + ID: sender.Client.GenerateMessageID(), Timestamp: time.Now(), MessageSource: types.MessageSource{ Sender: sender.JID, @@ -4069,9 +4526,13 @@ func (portal *Portal) generateMessageInfo(sender *User) *types.MessageInfo { func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event, timings messageTimings) { start := time.Now() ms := metricSender{portal: portal, timings: &timings} + log := portal.zlog.With(). + Str("event_id", evt.ID.String()). + Str("action", "handle matrix message"). + Logger() allowRelay := evt.Type != TypeMSC3381PollResponse && evt.Type != TypeMSC3381V2PollResponse && evt.Type != TypeMSC3381PollStart - if err := portal.canBridgeFrom(sender, allowRelay); err != nil { + if err := portal.canBridgeFrom(sender, allowRelay, true); err != nil { go ms.sendMessageMetrics(evt, err, "Ignoring", true) return } else if portal.Key.JID == types.StatusBroadcastJID && portal.bridge.Config.Bridge.DisableStatusBroadcastSend { @@ -4127,6 +4588,7 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event, timing ctx, cancel = context.WithTimeout(ctx, deadline) defer cancel() } + ctx = log.WithContext(ctx) timings.preproc = time.Since(start) start = time.Now() @@ -4136,6 +4598,9 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event, timing go ms.sendMessageMetrics(evt, err, "Error converting", true) return } + if extraMeta == nil { + extraMeta = &extraConvertMeta{} + } dbMsgType := database.MsgNormal if msg.PollCreationMessage != nil || msg.PollCreationMessageV2 != nil || msg.PollCreationMessageV3 != nil { dbMsgType = database.MsgMatrixPoll @@ -4146,26 +4611,45 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event, timing } info := portal.generateMessageInfo(sender) if dbMsg == nil { - dbMsg = portal.markHandled(nil, nil, info, evt.ID, evt.Sender, false, true, dbMsgType, database.MsgNoError) + dbMsg = portal.markHandled(nil, nil, info, evt.ID, evt.Sender, false, true, dbMsgType, 0, database.MsgNoError) } else { info.ID = dbMsg.JID } - if dbMsgType == database.MsgMatrixPoll && extraMeta != nil && extraMeta.PollOptions != nil { + if dbMsgType == database.MsgMatrixPoll && extraMeta.PollOptions != nil { dbMsg.PutPollOptions(extraMeta.PollOptions) } portal.log.Debugln("Sending event", evt.ID, "to WhatsApp", info.ID) start = time.Now() - resp, err := sender.Client.SendMessage(ctx, portal.Key.JID, msg, whatsmeow.SendRequestExtra{ID: info.ID}) + resp, err := sender.Client.SendMessage(ctx, portal.Key.JID, msg, whatsmeow.SendRequestExtra{ + ID: info.ID, + MediaHandle: extraMeta.MediaHandle, + }) timings.totalSend = time.Since(start) timings.whatsmeow = resp.DebugTimings - go ms.sendMessageMetrics(evt, err, "Error sending", true) - if err == nil { - dbMsg.MarkSent(resp.Timestamp) + if err != nil { + go ms.sendMessageMetrics(evt, err, "Error sending", true) + return } + dbMsg.MarkSent(resp.Timestamp) + if extraMeta != nil && len(extraMeta.GalleryExtraParts) > 0 { + for i, part := range extraMeta.GalleryExtraParts { + partInfo := portal.generateMessageInfo(sender) + partDBMsg := portal.markHandled(nil, nil, partInfo, evt.ID, evt.Sender, false, true, database.MsgBeeperGallery, i+1, database.MsgNoError) + portal.log.Debugln("Sending gallery part", i+2, "of event", evt.ID, "to WhatsApp", partInfo.ID) + resp, err = sender.Client.SendMessage(ctx, portal.Key.JID, part, whatsmeow.SendRequestExtra{ID: partInfo.ID}) + if err != nil { + go ms.sendMessageMetrics(evt, err, "Error sending", true) + return + } + portal.log.Debugfln("Sent gallery part", i+2, "of event", evt.ID) + partDBMsg.MarkSent(resp.Timestamp) + } + } + go ms.sendMessageMetrics(evt, nil, "", true) } func (portal *Portal) HandleMatrixReaction(sender *User, evt *event.Event) { - if err := portal.canBridgeFrom(sender, false); err != nil { + if err := portal.canBridgeFrom(sender, false, true); err != nil { go portal.sendMessageMetrics(evt, err, "Ignoring", nil) return } else if portal.Key.JID.Server == types.BroadcastServer { @@ -4201,7 +4685,7 @@ func (portal *Portal) handleMatrixReaction(sender *User, evt *event.Event) error return fmt.Errorf("unknown target event %s", content.RelatesTo.EventID) } info := portal.generateMessageInfo(sender) - dbMsg := portal.markHandled(nil, nil, info, evt.ID, evt.Sender, false, true, database.MsgReaction, database.MsgNoError) + dbMsg := portal.markHandled(nil, nil, info, evt.ID, evt.Sender, false, true, database.MsgReaction, 0, database.MsgNoError) portal.upsertReaction(nil, nil, target.JID, sender.JID, evt.ID, info.ID) portal.log.Debugln("Sending reaction", evt.ID, "to WhatsApp", info.ID) resp, err := portal.sendReactionToWhatsApp(sender, info.ID, target, content.RelatesTo.Key, evt.Timestamp) @@ -4217,7 +4701,9 @@ func (portal *Portal) sendReactionToWhatsApp(sender *User, id types.MessageID, t messageKeyParticipant = proto.String(target.Sender.ToNonAD().String()) } key = variationselector.Remove(key) - return sender.Client.SendMessage(context.TODO(), portal.Key.JID, &waProto.Message{ + ctx, cancel := context.WithTimeout(context.TODO(), 60*time.Second) + defer cancel() + return sender.Client.SendMessage(ctx, portal.Key.JID, &waProto.Message{ ReactionMessage: &waProto.ReactionMessage{ Key: &waProto.MessageKey{ RemoteJid: proto.String(portal.Key.JID.String()), @@ -4257,7 +4743,7 @@ func (portal *Portal) upsertReaction(txn dbutil.Transaction, intent *appservice. } func (portal *Portal) HandleMatrixRedaction(sender *User, evt *event.Event) { - if err := portal.canBridgeFrom(sender, true); err != nil { + if err := portal.canBridgeFrom(sender, true, true); err != nil { go portal.sendMessageMetrics(evt, err, "Ignoring", nil) return } @@ -4303,7 +4789,9 @@ func (portal *Portal) HandleMatrixRedaction(sender *User, evt *event.Event) { key.Participant = proto.String(msg.Sender.ToNonAD().String()) } portal.log.Debugfln("Sending redaction %s of %s/%s to WhatsApp", evt.ID, msg.MXID, msg.JID) - _, err := sender.Client.SendMessage(context.TODO(), portal.Key.JID, &waProto.Message{ + ctx, cancel := context.WithTimeout(context.TODO(), 60*time.Second) + defer cancel() + _, err := sender.Client.SendMessage(ctx, portal.Key.JID, &waProto.Message{ ProtocolMessage: &waProto.ProtocolMessage{ Type: waProto.ProtocolMessage_REVOKE.Enum(), Key: key, @@ -4426,12 +4914,16 @@ func (portal *Portal) HandleMatrixTyping(newTyping []id.UserID) { portal.setTyping(stoppedTyping, types.ChatPresencePaused) } -func (portal *Portal) canBridgeFrom(sender *User, allowRelay bool) error { +func (portal *Portal) canBridgeFrom(sender *User, allowRelay, reconnectWait bool) error { if !sender.IsLoggedIn() { if allowRelay && portal.HasRelaybot() { return nil } else if sender.Session != nil { return errUserNotConnected + } else if reconnectWait { + // If a message was received exactly during a disconnection, wait a second for the socket to reconnect + time.Sleep(1 * time.Second) + return portal.canBridgeFrom(sender, allowRelay, false) } else { return errUserNotLoggedIn } @@ -4500,7 +4992,7 @@ func (portal *Portal) Cleanup(puppetsOnly bool) { return } intent := portal.MainIntent() - if portal.bridge.SpecVersions.UnstableFeatures["com.beeper.room_yeeting"] { + if portal.bridge.SpecVersions.Supports(mautrix.BeeperFeatureRoomYeeting) { err := intent.BeeperDeleteRoom(portal.MXID) if err == nil || errors.Is(err, mautrix.MNotFound) { return @@ -4556,9 +5048,7 @@ func (portal *Portal) HandleMatrixLeave(brSender bridge.User) { func (portal *Portal) HandleMatrixKick(brSender bridge.User, brTarget bridge.Ghost) { sender := brSender.(*User) target := brTarget.(*Puppet) - _, err := sender.Client.UpdateGroupParticipants(portal.Key.JID, map[types.JID]whatsmeow.ParticipantChange{ - target.JID: whatsmeow.ParticipantChangeRemove, - }) + _, err := sender.Client.UpdateGroupParticipants(portal.Key.JID, []types.JID{target.JID}, whatsmeow.ParticipantChangeRemove) if err != nil { portal.log.Errorfln("Failed to kick %s from group as %s: %v", target.JID, sender.MXID, err) return @@ -4569,9 +5059,7 @@ func (portal *Portal) HandleMatrixKick(brSender bridge.User, brTarget bridge.Gho func (portal *Portal) HandleMatrixInvite(brSender bridge.User, brTarget bridge.Ghost) { sender := brSender.(*User) target := brTarget.(*Puppet) - _, err := sender.Client.UpdateGroupParticipants(portal.Key.JID, map[types.JID]whatsmeow.ParticipantChange{ - target.JID: whatsmeow.ParticipantChangeAdd, - }) + _, err := sender.Client.UpdateGroupParticipants(portal.Key.JID, []types.JID{target.JID}, whatsmeow.ParticipantChangeAdd) if err != nil { portal.log.Errorfln("Failed to add %s to group as %s: %v", target.JID, sender.MXID, err) return diff --git a/provisioning.go b/provisioning.go index 7abef77..911338d 100644 --- a/provisioning.go +++ b/provisioning.go @@ -24,7 +24,7 @@ import ( "fmt" "net" "net/http" - "strconv" + _ "net/http/pprof" "strings" "time" @@ -32,7 +32,6 @@ import ( "github.com/gorilla/websocket" "go.mau.fi/whatsmeow/appstate" - waBinary "go.mau.fi/whatsmeow/binary" "go.mau.fi/whatsmeow/types" "go.mau.fi/whatsmeow" @@ -61,7 +60,6 @@ func (prov *ProvisioningAPI) Init() { r.HandleFunc("/v1/disconnect", prov.Disconnect).Methods(http.MethodPost) r.HandleFunc("/v1/reconnect", prov.Reconnect).Methods(http.MethodPost) r.HandleFunc("/v1/debug/appstate/{name}", prov.SyncAppState).Methods(http.MethodPost) - r.HandleFunc("/v1/debug/retry", prov.SendRetryReceipt).Methods(http.MethodPost) r.HandleFunc("/v1/contacts", prov.ListContacts).Methods(http.MethodGet) r.HandleFunc("/v1/groups", prov.ListGroups).Methods(http.MethodGet, http.MethodPost) r.HandleFunc("/v1/resolve_identifier/{number}", prov.ResolveIdentifier).Methods(http.MethodGet) @@ -74,6 +72,13 @@ func (prov *ProvisioningAPI) Init() { prov.bridge.AS.Router.HandleFunc("/_matrix/app/com.beeper.asmux/ping", prov.BridgeStatePing).Methods(http.MethodPost) prov.bridge.AS.Router.HandleFunc("/_matrix/app/com.beeper.bridge_state", prov.BridgeStatePing).Methods(http.MethodPost) + if prov.bridge.Config.Bridge.Provisioning.DebugEndpoints { + prov.log.Debugln("Enabling debug API at /debug") + r := prov.bridge.AS.Router.PathPrefix("/debug").Subrouter() + r.Use(prov.AuthMiddleware) + r.PathPrefix("/pprof").Handler(http.DefaultServeMux) + } + // Deprecated, just use /disconnect r.HandleFunc("/v1/delete_connection", prov.Disconnect).Methods(http.MethodPost) } @@ -191,55 +196,6 @@ func (prov *ProvisioningAPI) Reconnect(w http.ResponseWriter, r *http.Request) { } } -type debugRetryReceiptContent struct { - ID types.MessageID `json:"id"` - From types.JID `json:"from"` - Recipient types.JID `json:"recipient"` - Participant types.JID `json:"participant"` - Timestamp int64 `json:"timestamp"` - Count int `json:"count"` - - ForceIncludeIdentity bool `json:"force_include_identity"` -} - -func (prov *ProvisioningAPI) SendRetryReceipt(w http.ResponseWriter, r *http.Request) { - var req debugRetryReceiptContent - user := r.Context().Value("user").(*User) - if user == nil || user.Client == nil { - jsonResponse(w, http.StatusNotFound, Error{ - Error: "User is not connected to WhatsApp", - ErrCode: "no session", - }) - return - } else if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - jsonResponse(w, http.StatusBadRequest, Error{ - Error: "Failed to parse request JSON", - ErrCode: "bad json", - }) - } else { - node := &waBinary.Node{ - Attrs: waBinary.Attrs{ - "id": string(req.ID), - "from": req.From, - "t": strconv.FormatInt(req.Timestamp, 10), - }, - } - if !req.Recipient.IsEmpty() { - node.Attrs["recipient"] = req.Recipient - } - if !req.Participant.IsEmpty() { - node.Attrs["participant"] = req.Participant - } - if req.Count > 0 { - node.Content = []waBinary.Node{{ - Tag: "enc", - Attrs: waBinary.Attrs{"count": strconv.Itoa(req.Count)}, - }} - } - user.Client.DangerousInternals().SendRetryReceipt(node, req.ForceIncludeIdentity) - } -} - func (prov *ProvisioningAPI) SyncAppState(w http.ResponseWriter, r *http.Request) { user := r.Context().Value("user").(*User) if user == nil || user.Client == nil { @@ -503,7 +459,7 @@ func (prov *ProvisioningAPI) OpenGroup(w http.ResponseWriter, r *http.Request) { portal := user.GetPortalByJID(info.JID) status := http.StatusOK if len(portal.MXID) == 0 { - err = portal.CreateMatrixRoom(user, info, true, true) + err = portal.CreateMatrixRoom(user, info, nil, true, true) if err != nil { jsonResponse(w, http.StatusInternalServerError, Error{ Error: fmt.Sprintf("Failed to create portal: %v", err), @@ -584,7 +540,7 @@ func (prov *ProvisioningAPI) JoinGroup(w http.ResponseWriter, r *http.Request) { status := http.StatusOK if len(portal.MXID) == 0 { time.Sleep(500 * time.Millisecond) // Wait for incoming group info to create the portal automatically - err = portal.CreateMatrixRoom(user, info, true, true) + err = portal.CreateMatrixRoom(user, info, nil, true, true) if err != nil { jsonResponse(w, http.StatusInternalServerError, Error{ Error: fmt.Sprintf("Failed to create portal: %v", err), @@ -744,7 +700,7 @@ func (prov *ProvisioningAPI) Login(w http.ResponseWriter, r *http.Request) { } } user.log.Debugln("Started login via provisioning API") - Segment.Track(user.MXID, "$login_start") + Analytics.Track(user.MXID, "$login_start") for { select { @@ -753,7 +709,7 @@ func (prov *ProvisioningAPI) Login(w http.ResponseWriter, r *http.Request) { case whatsmeow.QRChannelSuccess.Event: jid := user.Client.Store.ID user.log.Debugln("Successful login as", jid, "via provisioning API") - Segment.Track(user.MXID, "$login_success") + Analytics.Track(user.MXID, "$login_success") _ = c.WriteJSON(map[string]interface{}{ "success": true, "jid": jid, @@ -763,7 +719,7 @@ func (prov *ProvisioningAPI) Login(w http.ResponseWriter, r *http.Request) { case whatsmeow.QRChannelTimeout.Event: user.log.Debugln("Login via provisioning API timed out") errCode := "login timed out" - Segment.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) + Analytics.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) _ = c.WriteJSON(Error{ Error: "QR code scan timed out. Please try again.", ErrCode: errCode, @@ -771,7 +727,7 @@ func (prov *ProvisioningAPI) Login(w http.ResponseWriter, r *http.Request) { case whatsmeow.QRChannelErrUnexpectedEvent.Event: user.log.Debugln("Login via provisioning API failed due to unexpected event") errCode := "unexpected event" - Segment.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) + Analytics.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) _ = c.WriteJSON(Error{ Error: "Got unexpected event while waiting for QRs, perhaps you're already logged in?", ErrCode: errCode, @@ -779,14 +735,14 @@ func (prov *ProvisioningAPI) Login(w http.ResponseWriter, r *http.Request) { case whatsmeow.QRChannelClientOutdated.Event: user.log.Debugln("Login via provisioning API failed due to outdated client") errCode := "bridge outdated" - Segment.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) + Analytics.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) _ = c.WriteJSON(Error{ Error: "Got client outdated error while waiting for QRs. The bridge must be updated to continue.", ErrCode: errCode, }) case whatsmeow.QRChannelScannedWithoutMultidevice.Event: errCode := "multidevice not enabled" - Segment.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) + Analytics.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) _ = c.WriteJSON(Error{ Error: "Please enable the WhatsApp multidevice beta and scan the QR code again.", ErrCode: errCode, @@ -794,13 +750,13 @@ func (prov *ProvisioningAPI) Login(w http.ResponseWriter, r *http.Request) { continue case "error": errCode := "fatal error" - Segment.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) + Analytics.Track(user.MXID, "$login_failure", map[string]interface{}{"error": errCode}) _ = c.WriteJSON(Error{ Error: "Fatal error while logging in", ErrCode: errCode, }) case "code": - Segment.Track(user.MXID, "$qrcode_retrieved") + Analytics.Track(user.MXID, "$qrcode_retrieved") _ = c.WriteJSON(map[string]interface{}{ "code": evt.Code, "timeout": int(evt.Timeout.Seconds()), diff --git a/puppet.go b/puppet.go index 7f2d46e..0c3f9ac 100644 --- a/puppet.go +++ b/puppet.go @@ -26,9 +26,9 @@ import ( log "maunium.net/go/maulogger/v2" + "maunium.net/go/mautrix" "maunium.net/go/mautrix/appservice" "maunium.net/go/mautrix/bridge" - "maunium.net/go/mautrix/bridge/bridgeconfig" "maunium.net/go/mautrix/id" "maunium.net/go/mautrix-whatsapp/config" @@ -264,7 +264,7 @@ func (puppet *Puppet) UpdateName(contact types.ContactInfo, forcePortalSync bool } func (puppet *Puppet) UpdateContactInfo() bool { - if puppet.bridge.Config.Homeserver.Software != bridgeconfig.SoftwareHungry { + if !puppet.bridge.SpecVersions.Supports(mautrix.BeeperFeatureArbitraryProfileMeta) { return false } @@ -330,6 +330,9 @@ func (puppet *Puppet) updatePortalName() { } func (puppet *Puppet) SyncContact(source *User, onlyIfNoName, shouldHavePushName bool, reason string) { + if puppet == nil { + return + } if onlyIfNoName && len(puppet.Displayname) > 0 && (!shouldHavePushName || puppet.NameQuality > config.NameQualityPhone) { source.EnqueuePuppetResync(puppet) return diff --git a/user.go b/user.go index cc895ed..a6f9f68 100644 --- a/user.go +++ b/user.go @@ -327,7 +327,7 @@ func (user *User) doPuppetResync() { user.log.Warnfln("Failed to get group info for %s to do background sync: %v", portal.Key.JID, err) } else { user.log.Debugfln("Doing background sync for %s", portal.Key.JID) - portal.UpdateMatrixRoom(user, groupInfo) + portal.UpdateMatrixRoom(user, groupInfo, nil) } } if len(puppetJIDs) == 0 { @@ -496,8 +496,9 @@ func (user *User) createClient(sess *store.Device) { user.Client = whatsmeow.NewClient(sess, &waLogger{user.log.Sub("Client")}) user.Client.AddEventHandler(user.HandleEvent) user.Client.SetForceActiveDeliveryReceipts(user.bridge.Config.Bridge.ForceActiveDeliveryReceipts) + user.Client.AutomaticMessageRerequestFromPhone = true user.Client.GetMessageForRetry = func(requester, to types.JID, id types.MessageID) *waProto.Message { - Segment.Track(user.MXID, "WhatsApp incoming retry (message not found)", map[string]interface{}{ + Analytics.Track(user.MXID, "WhatsApp incoming retry (message not found)", map[string]interface{}{ "requester": user.obfuscateJID(requester), "messageID": id, }) @@ -505,7 +506,7 @@ func (user *User) createClient(sess *store.Device) { return nil } user.Client.PreRetryCallback = func(receipt *events.Receipt, messageID types.MessageID, retryCount int, msg *waProto.Message) bool { - Segment.Track(user.MXID, "WhatsApp incoming retry (accepted)", map[string]interface{}{ + Analytics.Track(user.MXID, "WhatsApp incoming retry (accepted)", map[string]interface{}{ "requester": user.obfuscateJID(receipt.Sender), "messageID": messageID, "retryCount": retryCount, @@ -611,30 +612,6 @@ func (user *User) IsLoggedIn() bool { return user.IsConnected() && user.Client.IsLoggedIn() } -func (user *User) tryAutomaticDoublePuppeting() { - if !user.bridge.Config.CanAutoDoublePuppet(user.MXID) { - return - } - user.log.Debugln("Checking if double puppeting needs to be enabled") - puppet := user.bridge.GetPuppetByJID(user.JID) - if len(puppet.CustomMXID) > 0 { - user.log.Debugln("User already has double-puppeting enabled") - // Custom puppet already enabled - return - } - accessToken, err := puppet.loginWithSharedSecret(user.MXID) - if err != nil { - user.log.Warnln("Failed to login with shared secret:", err) - return - } - err = puppet.SwitchCustomMXID(accessToken, user.MXID) - if err != nil { - puppet.log.Warnln("Failed to switch to auto-logined custom puppet:", err) - return - } - user.log.Infoln("Successfully automatically enabled custom puppet") -} - func (user *User) sendMarkdownBridgeAlert(formatString string, args ...interface{}) { if user.bridge.Config.Bridge.DisableBridgeAlerts { return @@ -654,19 +631,21 @@ func (user *User) handleCallStart(sender types.JID, id, callType string, ts time return } portal := user.GetPortalByJID(sender) - text := "Incoming call" + text := "Incoming call. Use the WhatsApp app to answer." if callType != "" { - text = fmt.Sprintf("Incoming %s call", callType) + text = fmt.Sprintf("Incoming %s call. Use the WhatsApp app to answer.", callType) } - portal.messages <- PortalMessage{ - fake: &fakeMessage{ - Sender: sender, - Text: text, - ID: id, - Time: ts, - Important: true, + portal.events <- &PortalEvent{ + Message: &PortalMessage{ + fake: &fakeMessage{ + Sender: sender, + Text: text, + ID: id, + Time: ts, + Important: true, + }, + source: user, }, - source: user, } } @@ -676,7 +655,7 @@ const PhoneMinPingInterval = 24 * time.Hour func (user *User) sendHackyPhonePing() { user.PhoneLastPinged = time.Now() - msgID := whatsmeow.GenerateMessageID() + msgID := user.Client.GenerateMessageID() keyIDs := make([]*waProto.AppStateSyncKeyId, 0, 1) lastKeyID, err := user.GetLastAppStateKeyID() if lastKeyID != nil { @@ -842,15 +821,18 @@ func (user *User) HandleEvent(event interface{}) { user.sendMarkdownBridgeAlert("The bridge was started in another location. Use `reconnect` to reconnect this one.") } case *events.ConnectFailure: - user.BridgeState.Send(status.BridgeState{StateEvent: status.StateUnknownError, Message: fmt.Sprintf("Unknown connection failure: %s", v.Reason)}) + user.BridgeState.Send(status.BridgeState{StateEvent: status.StateUnknownError, Message: fmt.Sprintf("Unknown connection failure: %s (%s)", v.Reason, v.Message)}) user.bridge.Metrics.TrackConnectionState(user.JID, false) + user.bridge.Metrics.TrackConnectionFailure(fmt.Sprintf("status-%d", v.Reason)) case *events.ClientOutdated: user.log.Errorfln("Got a client outdated connect failure. The bridge is likely out of date, please update immediately.") user.BridgeState.Send(status.BridgeState{StateEvent: status.StateUnknownError, Message: "Connect failure: 405 client outdated"}) user.bridge.Metrics.TrackConnectionState(user.JID, false) + user.bridge.Metrics.TrackConnectionFailure("client-outdated") case *events.TemporaryBan: user.BridgeState.Send(status.BridgeState{StateEvent: status.StateBadCredentials, Message: v.String()}) user.bridge.Metrics.TrackConnectionState(user.JID, false) + user.bridge.Metrics.TrackConnectionFailure("temporary-ban") case *events.Disconnected: // Don't send the normal transient disconnect state if we're already in a different transient disconnect state. // TODO remove this if/when the phone offline state is moved to a sub-state of CONNECTED @@ -870,6 +852,10 @@ func (user *User) HandleEvent(event interface{}) { case *events.JoinedGroup: user.groupListCache = nil go user.handleGroupCreate(v) + case *events.NewsletterJoin: + go user.handleNewsletterJoin(v) + case *events.NewsletterLeave: + go user.handleNewsletterLeave(v) case *events.Picture: go user.handlePictureUpdate(v) case *events.Receipt: @@ -881,39 +867,50 @@ func (user *User) HandleEvent(event interface{}) { go user.handleChatPresence(v) case *events.Message: portal := user.GetPortalByMessageSource(v.Info.MessageSource) - portal.messages <- PortalMessage{evt: v, source: user} + portal.events <- &PortalEvent{ + Message: &PortalMessage{evt: v, source: user}, + } case *events.MediaRetry: user.phoneSeen(v.Timestamp) portal := user.GetPortalByJID(v.ChatID) - portal.mediaRetries <- PortalMediaRetry{evt: v, source: user} + portal.events <- &PortalEvent{ + MediaRetry: &PortalMediaRetry{evt: v, source: user}, + } case *events.CallOffer: user.handleCallStart(v.CallCreator, v.CallID, "", v.Timestamp) case *events.CallOfferNotice: user.handleCallStart(v.CallCreator, v.CallID, v.Type, v.Timestamp) case *events.IdentityChange: puppet := user.bridge.GetPuppetByJID(v.JID) + if puppet == nil { + return + } portal := user.GetPortalByJID(v.JID) if len(portal.MXID) > 0 && user.bridge.Config.Bridge.IdentityChangeNotices { text := fmt.Sprintf("Your security code with %s changed.", puppet.Displayname) if v.Implicit { text = fmt.Sprintf("Your security code with %s (device #%d) changed.", puppet.Displayname, v.JID.Device) } - portal.messages <- PortalMessage{ - fake: &fakeMessage{ - Sender: v.JID, - Text: text, - ID: strconv.FormatInt(v.Timestamp.Unix(), 10), - Time: v.Timestamp, - Important: false, + portal.events <- &PortalEvent{ + Message: &PortalMessage{ + fake: &fakeMessage{ + Sender: v.JID, + Text: text, + ID: strconv.FormatInt(v.Timestamp.Unix(), 10), + Time: v.Timestamp, + Important: false, + }, + source: user, }, - source: user, } } case *events.CallTerminate, *events.CallRelayLatency, *events.CallAccept, *events.UnknownCallEvent: // ignore case *events.UndecryptableMessage: portal := user.GetPortalByMessageSource(v.Info.MessageSource) - portal.messages <- PortalMessage{undecryptable: v, source: user} + portal.events <- &PortalEvent{ + Message: &PortalMessage{undecryptable: v, source: user}, + } case *events.HistorySync: if user.bridge.Config.Bridge.HistorySync.Backfill { user.historySyncs <- v @@ -1201,13 +1198,13 @@ func (user *User) ResyncGroups(createPortals bool) error { portal := user.GetPortalByJID(group.JID) if len(portal.MXID) == 0 { if createPortals { - err = portal.CreateMatrixRoom(user, group, true, true) + err = portal.CreateMatrixRoom(user, group, nil, true, true) if err != nil { return fmt.Errorf("failed to create room for %s: %w", group.JID, err) } } } else { - portal.UpdateMatrixRoom(user, group) + portal.UpdateMatrixRoom(user, group, nil) } } return nil @@ -1217,6 +1214,9 @@ const WATypingTimeout = 15 * time.Second func (user *User) handleChatPresence(presence *events.ChatPresence) { puppet := user.bridge.GetPuppetByJID(presence.Sender) + if puppet == nil { + return + } portal := user.GetPortalByJID(presence.Chat) if puppet == nil || portal == nil || len(portal.MXID) == 0 { return @@ -1238,14 +1238,16 @@ func (user *User) handleChatPresence(presence *events.ChatPresence) { } func (user *User) handleReceipt(receipt *events.Receipt) { - if receipt.Type != events.ReceiptTypeRead && receipt.Type != events.ReceiptTypeReadSelf { + if receipt.Type != types.ReceiptTypeRead && receipt.Type != types.ReceiptTypeReadSelf && receipt.Type != types.ReceiptTypeDelivered { return } portal := user.GetPortalByMessageSource(receipt.MessageSource) if portal == nil || len(portal.MXID) == 0 { return } - portal.messages <- PortalMessage{receipt: receipt, source: user} + portal.events <- &PortalEvent{ + Message: &PortalMessage{receipt: receipt, source: user}, + } } func (user *User) makeReadMarkerContent(eventID id.EventID, doublePuppet bool) CustomReadMarkers { @@ -1315,12 +1317,12 @@ func (user *User) handleGroupCreate(evt *events.JoinedGroup) { user.log.Debugfln("Ignoring group create event with key %s", evt.CreateKey) return } - err := portal.CreateMatrixRoom(user, &evt.GroupInfo, true, true) + err := portal.CreateMatrixRoom(user, &evt.GroupInfo, nil, true, true) if err != nil { user.log.Errorln("Failed to create Matrix room after join notification: %v", err) } } else { - portal.UpdateMatrixRoom(user, &evt.GroupInfo) + portal.UpdateMatrixRoom(user, &evt.GroupInfo, nil) } } @@ -1337,6 +1339,10 @@ func (user *User) handleGroupUpdate(evt *events.GroupInfo) { log.Debug().Msg("Ignoring group info update in chat with no portal") return } + if evt.Sender != nil && evt.Sender.Server == types.HiddenUserServer { + log.Debug().Str("sender", evt.Sender.String()).Msg("Ignoring group info update from @lid user") + return + } switch { case evt.Announce != nil: log.Debug().Msg("Group announcement mode (message send permission) changed") @@ -1386,6 +1392,25 @@ func (user *User) handleGroupUpdate(evt *events.GroupInfo) { } } +func (user *User) handleNewsletterJoin(evt *events.NewsletterJoin) { + portal := user.GetPortalByJID(evt.ID) + if portal.MXID == "" { + err := portal.CreateMatrixRoom(user, nil, &evt.NewsletterMetadata, true, false) + if err != nil { + user.zlog.Err(err).Msg("Failed to create room on newsletter join event") + } + } else { + portal.UpdateMatrixRoom(user, nil, &evt.NewsletterMetadata) + } +} + +func (user *User) handleNewsletterLeave(evt *events.NewsletterLeave) { + portal := user.GetPortalByJID(evt.ID) + if portal.MXID != "" { + portal.HandleWhatsAppKick(user, user.JID, []types.JID{user.JID}) + } +} + func (user *User) handlePictureUpdate(evt *events.Picture) { if evt.JID.Server == types.DefaultUserServer { puppet := user.bridge.GetPuppetByJID(evt.JID) @@ -1415,7 +1440,7 @@ func (user *User) StartPM(jid types.JID, reason string) (*Portal, *Puppet, bool, return portal, puppet, false, nil } } - err := portal.CreateMatrixRoom(user, nil, false, true) + err := portal.CreateMatrixRoom(user, nil, nil, false, true) return portal, puppet, true, err }