From acc25a02e40ef5b5b50f5da6dd333c49cf4a5ad4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 8 May 2020 22:32:22 +0300 Subject: [PATCH 01/22] Update mautrix-go --- commands.go | 15 +- community.go | 9 +- config/bridge.go | 52 ++-- custompuppet.go | 156 ++++------ database/message.go | 7 +- database/portal.go | 27 +- database/puppet.go | 17 +- database/statestore.go | 57 ++-- .../2019-08-25-move-state-store-to-db.go | 12 +- database/user.go | 9 +- formatting.go | 16 +- go.mod | 8 +- go.sum | 22 ++ main.go | 34 +-- matrix.go | 101 +++---- portal.go | 266 +++++++++--------- provisioning.go | 6 +- puppet.go | 22 +- types/types.go | 9 - user.go | 74 ++--- 20 files changed, 454 insertions(+), 465 deletions(-) diff --git a/commands.go b/commands.go index ac4ddbb..ec4fb9b 100644 --- a/commands.go +++ b/commands.go @@ -1,5 +1,5 @@ // mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. -// Copyright (C) 2019 Tulir Asokan +// Copyright (C) 2020 Tulir Asokan // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by @@ -26,10 +26,11 @@ import ( "maunium.net/go/mautrix" "maunium.net/go/mautrix-appservice" + "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/format" + "maunium.net/go/mautrix/id" "maunium.net/go/mautrix-whatsapp/database" - "maunium.net/go/mautrix-whatsapp/types" "maunium.net/go/mautrix-whatsapp/whatsapp-ext" ) @@ -51,7 +52,7 @@ type CommandEvent struct { Bot *appservice.IntentAPI Bridge *Bridge Handler *CommandHandler - RoomID types.MatrixRoomID + RoomID id.RoomID User *User Command string Args []string @@ -59,20 +60,20 @@ type CommandEvent struct { // Reply sends a reply to command as notice func (ce *CommandEvent) Reply(msg string, args ...interface{}) { - content := format.RenderMarkdown(fmt.Sprintf(msg, args...)) - content.MsgType = mautrix.MsgNotice + content := format.RenderMarkdown(fmt.Sprintf(msg, args...), true, false) + content.MsgType = event.MsgNotice room := ce.User.ManagementRoom if len(room) == 0 { room = ce.RoomID } - _, err := ce.Bot.SendMessageEvent(room, mautrix.EventMessage, content) + _, err := ce.Bot.SendMessageEvent(room, event.EventMessage, content) if err != nil { ce.Handler.log.Warnfln("Failed to reply to command from %s: %v", ce.User.MXID, err) } } // Handle handles messages to the bridge -func (handler *CommandHandler) Handle(roomID types.MatrixRoomID, user *User, message string) { +func (handler *CommandHandler) Handle(roomID id.RoomID, user *User, message string) { args := strings.Split(message, " ") ce := &CommandEvent{ Bot: handler.bridge.Bot, diff --git a/community.go b/community.go index 3f841f1..6ed97ee 100644 --- a/community.go +++ b/community.go @@ -1,5 +1,5 @@ // mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. -// Copyright (C) 2019 Tulir Asokan +// Copyright (C) 2020 Tulir Asokan // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by @@ -21,7 +21,6 @@ import ( "net/http" "maunium.net/go/mautrix" - appservice "maunium.net/go/mautrix-appservice" ) func (user *User) inviteToCommunity() { @@ -51,7 +50,7 @@ func (user *User) createCommunity() { return } - localpart, server := appservice.ParseUserID(user.MXID) + localpart, server, _ := user.MXID.Parse() community := user.bridge.Config.Bridge.FormatCommunity(localpart, server) user.log.Debugln("Creating personal filtering community", community) bot := user.bridge.Bot @@ -100,8 +99,8 @@ func (user *User) addPuppetToCommunity(puppet *Puppet) bool { "type": "private", }, } - url = bot.BuildURLWithQuery([]string{"groups", user.CommunityID, "self", "accept_invite"}, map[string]string{ - "user_id": puppet.MXID, + url = bot.BuildURLWithQuery(mautrix.URLPath{"groups", user.CommunityID, "self", "accept_invite"}, map[string]string{ + "user_id": puppet.MXID.String(), }) _, err = bot.MakeRequest(http.MethodPut, url, &reqBody, nil) if err != nil { diff --git a/config/bridge.go b/config/bridge.go index baba694..b7efad5 100644 --- a/config/bridge.go +++ b/config/bridge.go @@ -1,5 +1,5 @@ // mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. -// Copyright (C) 2019 Tulir Asokan +// Copyright (C) 2020 Tulir Asokan // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by @@ -24,8 +24,8 @@ import ( "github.com/Rhymen/go-whatsapp" - "maunium.net/go/mautrix" - "maunium.net/go/mautrix-appservice" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" "maunium.net/go/mautrix-whatsapp/types" ) @@ -54,8 +54,8 @@ type BridgeConfig struct { RecoverHistory bool `yaml:"recovery_history_backfill"` SyncChatMaxAge uint64 `yaml:"sync_max_chat_age"` - SyncWithCustomPuppets bool `yaml:"sync_with_custom_puppets"` - LoginSharedSecret string `yaml:"login_shared_secret"` + SyncWithCustomPuppets bool `yaml:"sync_with_custom_puppets"` + LoginSharedSecret string `yaml:"login_shared_secret"` InviteOwnPuppetForBackfilling bool `yaml:"invite_own_puppet_for_backfilling"` PrivateChatPortalMeta bool `yaml:"private_chat_portal_meta"` @@ -127,7 +127,7 @@ func (bc *BridgeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { } type UsernameTemplateArgs struct { - UserID string + UserID id.UserID } func (bc BridgeConfig) FormatDisplayname(contact whatsapp.Contact) (string, int8) { @@ -232,25 +232,25 @@ func (pc *PermissionConfig) MarshalYAML() (interface{}, error) { return rawPC, nil } -func (pc PermissionConfig) IsRelaybotWhitelisted(userID string) bool { +func (pc PermissionConfig) IsRelaybotWhitelisted(userID id.UserID) bool { return pc.GetPermissionLevel(userID) >= PermissionLevelRelaybot } -func (pc PermissionConfig) IsWhitelisted(userID string) bool { +func (pc PermissionConfig) IsWhitelisted(userID id.UserID) bool { return pc.GetPermissionLevel(userID) >= PermissionLevelUser } -func (pc PermissionConfig) IsAdmin(userID string) bool { +func (pc PermissionConfig) IsAdmin(userID id.UserID) bool { return pc.GetPermissionLevel(userID) >= PermissionLevelAdmin } -func (pc PermissionConfig) GetPermissionLevel(userID string) PermissionLevel { - permissions, ok := pc[userID] +func (pc PermissionConfig) GetPermissionLevel(userID id.UserID) PermissionLevel { + permissions, ok := pc[string(userID)] if ok { return permissions } - _, homeserver := appservice.ParseUserID(userID) + _, homeserver, _ := userID.Parse() permissions, ok = pc[homeserver] if len(homeserver) > 0 && ok { return permissions @@ -265,12 +265,12 @@ func (pc PermissionConfig) GetPermissionLevel(userID string) PermissionLevel { } type RelaybotConfig struct { - Enabled bool `yaml:"enabled"` - ManagementRoom string `yaml:"management"` - InviteUsers []types.MatrixUserID `yaml:"invites"` + Enabled bool `yaml:"enabled"` + ManagementRoom id.RoomID `yaml:"management"` + InviteUsers []id.UserID `yaml:"invites"` - MessageFormats map[mautrix.MessageType]string `yaml:"message_formats"` - messageTemplates *template.Template `yaml:"-"` + MessageFormats map[event.MessageType]string `yaml:"message_formats"` + messageTemplates *template.Template `yaml:"-"` } type umRelaybotConfig RelaybotConfig @@ -293,25 +293,25 @@ func (rc *RelaybotConfig) UnmarshalYAML(unmarshal func(interface{}) error) error } type Sender struct { - UserID types.MatrixUserID - mautrix.Member + UserID id.UserID + *event.MemberEventContent } type formatData struct { Sender Sender Message string - Content mautrix.Content + Content *event.MessageEventContent } -func (rc *RelaybotConfig) FormatMessage(evt *mautrix.Event, member mautrix.Member) (string, error) { +func (rc *RelaybotConfig) FormatMessage(content *event.MessageEventContent, sender id.UserID, member *event.MemberEventContent) (string, error) { var output strings.Builder - err := rc.messageTemplates.ExecuteTemplate(&output, string(evt.Content.MsgType), formatData{ + err := rc.messageTemplates.ExecuteTemplate(&output, string(content.MsgType), formatData{ Sender: Sender{ - UserID: evt.Sender, - Member: member, + UserID: sender, + MemberEventContent: member, }, - Content: evt.Content, - Message: evt.Content.FormattedBody, + Content: content, + Message: content.FormattedBody, }) return output.String(), err } diff --git a/custompuppet.go b/custompuppet.go index ac98ee9..1f2c520 100644 --- a/custompuppet.go +++ b/custompuppet.go @@ -1,5 +1,5 @@ // mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. -// Copyright (C) 2019 Tulir Asokan +// Copyright (C) 2020 Tulir Asokan // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by @@ -20,17 +20,16 @@ import ( "crypto/hmac" "crypto/sha512" "encoding/hex" - "encoding/json" - "fmt" - "os" - "strings" "time" "github.com/pkg/errors" "github.com/Rhymen/go-whatsapp" + "maunium.net/go/mautrix" appservice "maunium.net/go/mautrix-appservice" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" ) var ( @@ -38,7 +37,7 @@ var ( ErrMismatchingMXID = errors.New("whoami result does not match custom mxid") ) -func (puppet *Puppet) SwitchCustomMXID(accessToken string, mxid string) error { +func (puppet *Puppet) SwitchCustomMXID(accessToken string, mxid id.UserID) error { prevCustomMXID := puppet.CustomMXID if puppet.customIntent != nil { puppet.stopSyncing() @@ -63,12 +62,12 @@ func (puppet *Puppet) SwitchCustomMXID(accessToken string, mxid string) error { return nil } -func (puppet *Puppet) loginWithSharedSecret(mxid string) (string, error) { +func (puppet *Puppet) loginWithSharedSecret(mxid id.UserID) (string, error) { mac := hmac.New(sha512.New, []byte(puppet.bridge.Config.Bridge.LoginSharedSecret)) mac.Write([]byte(mxid)) resp, err := puppet.bridge.AS.BotClient().Login(&mautrix.ReqLogin{ Type: "m.login.password", - Identifier: mautrix.UserIdentifier{Type: "m.id.user", User: mxid}, + Identifier: mautrix.UserIdentifier{Type: "m.id.user", User: string(mxid)}, Password: hex.EncodeToString(mac.Sum(nil)), DeviceID: "WhatsApp Bridge", InitialDeviceDisplayName: "WhatsApp Bridge", @@ -87,13 +86,13 @@ func (puppet *Puppet) newCustomIntent() (*appservice.IntentAPI, error) { if err != nil { return nil, err } - client.Logger = puppet.bridge.AS.Log.Sub(puppet.CustomMXID) + client.Logger = puppet.bridge.AS.Log.Sub(string(puppet.CustomMXID)) client.Syncer = puppet client.Store = puppet ia := puppet.bridge.AS.NewIntentAPI("custom") ia.Client = client - ia.Localpart = puppet.CustomMXID[1:strings.IndexRune(puppet.CustomMXID, ':')] + ia.Localpart, _, _ = puppet.CustomMXID.Parse() ia.UserID = puppet.CustomMXID ia.IsCustomPuppet = true return ia, nil @@ -117,11 +116,7 @@ func (puppet *Puppet) StartCustomMXID() error { puppet.clearCustomMXID() return err } - urlPath := intent.BuildURL("account", "whoami") - var resp struct { - UserID string `json:"user_id"` - } - _, err = intent.MakeRequest("GET", urlPath, nil, &resp) + resp, err := intent.Whoami() if err != nil { puppet.clearCustomMXID() return err @@ -131,7 +126,7 @@ func (puppet *Puppet) StartCustomMXID() error { return ErrMismatchingMXID } puppet.customIntent = intent - puppet.customTypingIn = make(map[string]bool) + puppet.customTypingIn = make(map[id.RoomID]bool) puppet.customUser = puppet.bridge.GetUserByMXID(puppet.CustomMXID) puppet.startSyncing() return nil @@ -158,28 +153,6 @@ func (puppet *Puppet) stopSyncing() { puppet.customIntent.StopSync() } -func parseEvent(roomID string, data json.RawMessage) *mautrix.Event { - event := &mautrix.Event{} - err := json.Unmarshal(data, event) - if err != nil { - // TODO add separate handler for these - _, _ = fmt.Fprintf(os.Stderr, "Failed to unmarshal event: %v\n%s\n", err, string(data)) - return nil - } - return event -} - -func parsePresenceEvent(data json.RawMessage) *mautrix.Event { - event := &mautrix.Event{} - err := json.Unmarshal(data, event) - if err != nil { - // TODO add separate handler for these - _, _ = fmt.Fprintf(os.Stderr, "Failed to unmarshal event: %v\n%s\n", err, string(data)) - return nil - } - return event -} - func (puppet *Puppet) ProcessResponse(resp *mautrix.RespSync, since string) error { if !puppet.customUser.IsConnected() { puppet.log.Debugln("Skipping sync processing: custom user not connected to whatsapp") @@ -190,31 +163,33 @@ func (puppet *Puppet) ProcessResponse(resp *mautrix.RespSync, since string) erro if portal == nil { continue } - for _, data := range events.Ephemeral.Events { - event := parseEvent(roomID, data) - if event != nil { - switch event.Type { - case mautrix.EphemeralEventReceipt: - go puppet.handleReceiptEvent(portal, event) - case mautrix.EphemeralEventTyping: - go puppet.handleTypingEvent(portal, event) - } + for _, evt := range events.Ephemeral.Events { + err := evt.Content.ParseRaw(evt.Type) + if err != nil { + continue + } + switch evt.Type { + case event.EphemeralEventReceipt: + go puppet.handleReceiptEvent(portal, evt) + case event.EphemeralEventTyping: + go puppet.handleTypingEvent(portal, evt) } } } - for _, data := range resp.Presence.Events { - event := parsePresenceEvent(data) - if event != nil { - if event.Sender != puppet.CustomMXID { - continue - } - go puppet.handlePresenceEvent(event) + for _, evt := range resp.Presence.Events { + if evt.Sender != puppet.CustomMXID { + continue } + err := evt.Content.ParseRaw(evt.Type) + if err != nil { + continue + } + go puppet.handlePresenceEvent(evt) } return nil } -func (puppet *Puppet) handlePresenceEvent(event *mautrix.Event) { +func (puppet *Puppet) handlePresenceEvent(event *event.Event) { presence := whatsapp.PresenceAvailable if event.Content.Raw["presence"].(string) != "online" { presence = whatsapp.PresenceUnavailable @@ -228,13 +203,9 @@ func (puppet *Puppet) handlePresenceEvent(event *mautrix.Event) { } } -func (puppet *Puppet) handleReceiptEvent(portal *Portal, event *mautrix.Event) { - for eventID, rawReceipts := range event.Content.Raw { - if receipts, ok := rawReceipts.(map[string]interface{}); !ok { - continue - } else if readReceipt, ok := receipts["m.read"].(map[string]interface{}); !ok { - continue - } else if _, ok = readReceipt[puppet.CustomMXID].(map[string]interface{}); !ok { +func (puppet *Puppet) handleReceiptEvent(portal *Portal, event *event.Event) { + for eventID, receipts := range *event.Content.AsReceipt() { + if _, ok := receipts.Read[puppet.CustomMXID]; !ok { continue } message := puppet.bridge.DB.Message.GetByMXID(eventID) @@ -249,16 +220,16 @@ func (puppet *Puppet) handleReceiptEvent(portal *Portal, event *mautrix.Event) { } } -func (puppet *Puppet) handleTypingEvent(portal *Portal, event *mautrix.Event) { +func (puppet *Puppet) handleTypingEvent(portal *Portal, evt *event.Event) { isTyping := false - for _, userID := range event.Content.TypingUserIDs { + for _, userID := range evt.Content.AsTyping().UserIDs { if userID == puppet.CustomMXID { isTyping = true break } } - if puppet.customTypingIn[event.RoomID] != isTyping { - puppet.customTypingIn[event.RoomID] = isTyping + if puppet.customTypingIn[evt.RoomID] != isTyping { + puppet.customTypingIn[evt.RoomID] = isTyping presence := whatsapp.PresenceComposing if !isTyping { puppet.customUser.log.Infofln("Marking not typing in %s/%s", portal.Key.JID, portal.MXID) @@ -278,36 +249,27 @@ func (puppet *Puppet) OnFailedSync(res *mautrix.RespSync, err error) (time.Durat return 10 * time.Second, nil } -func (puppet *Puppet) GetFilterJSON(_ string) json.RawMessage { - mxid, _ := json.Marshal(puppet.CustomMXID) - return json.RawMessage(fmt.Sprintf(`{ - "account_data": { "types": [] }, - "presence": { - "senders": [ - %s - ], - "types": [ - "m.presence" - ] - }, - "room": { - "ephemeral": { - "types": [ - "m.typing", - "m.receipt" - ] - }, - "include_leave": false, - "account_data": { "types": [] }, - "state": { "types": [] }, - "timeline": { "types": [] } - } -}`, mxid)) +func (puppet *Puppet) GetFilterJSON(_ id.UserID) *mautrix.Filter { + everything := []event.Type{{Type: "*"}} + return &mautrix.Filter{ + Presence: mautrix.FilterPart{ + Senders: []id.UserID{puppet.CustomMXID}, + Types: []event.Type{event.EphemeralEventPresence}, + }, + AccountData: mautrix.FilterPart{NotTypes: everything}, + Room: mautrix.RoomFilter{ + Ephemeral: mautrix.FilterPart{Types: []event.Type{event.EphemeralEventTyping, event.EphemeralEventReceipt}}, + IncludeLeave: false, + AccountData: mautrix.FilterPart{NotTypes: everything}, + State: mautrix.FilterPart{NotTypes: everything}, + Timeline: mautrix.FilterPart{NotTypes: everything}, + }, + } } -func (puppet *Puppet) SaveFilterID(_, _ string) {} -func (puppet *Puppet) SaveNextBatch(_, nbt string) { puppet.NextBatch = nbt; puppet.Update() } -func (puppet *Puppet) SaveRoom(room *mautrix.Room) {} -func (puppet *Puppet) LoadFilterID(_ string) string { return "" } -func (puppet *Puppet) LoadNextBatch(_ string) string { return puppet.NextBatch } -func (puppet *Puppet) LoadRoom(roomID string) *mautrix.Room { return nil } +func (puppet *Puppet) SaveFilterID(_ id.UserID, _ string) {} +func (puppet *Puppet) SaveNextBatch(_ id.UserID, nbt string) { puppet.NextBatch = nbt; puppet.Update() } +func (puppet *Puppet) SaveRoom(room *mautrix.Room) {} +func (puppet *Puppet) LoadFilterID(_ id.UserID) string { return "" } +func (puppet *Puppet) LoadNextBatch(_ id.UserID) string { return puppet.NextBatch } +func (puppet *Puppet) LoadRoom(roomID id.RoomID) *mautrix.Room { return nil } diff --git a/database/message.go b/database/message.go index f7976fd..b6bd5a4 100644 --- a/database/message.go +++ b/database/message.go @@ -1,5 +1,5 @@ // mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. -// Copyright (C) 2019 Tulir Asokan +// Copyright (C) 2020 Tulir Asokan // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by @@ -26,6 +26,7 @@ import ( log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix-whatsapp/types" + "maunium.net/go/mautrix/id" ) type MessageQuery struct { @@ -57,7 +58,7 @@ func (mq *MessageQuery) GetByJID(chat PortalKey, jid types.WhatsAppMessageID) *M "FROM message WHERE chat_jid=$1 AND chat_receiver=$2 AND jid=$3", chat.JID, chat.Receiver, jid) } -func (mq *MessageQuery) GetByMXID(mxid types.MatrixEventID) *Message { +func (mq *MessageQuery) GetByMXID(mxid id.EventID) *Message { return mq.get("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, content " + "FROM message WHERE mxid=$1", mxid) } @@ -86,7 +87,7 @@ type Message struct { Chat PortalKey JID types.WhatsAppMessageID - MXID types.MatrixEventID + MXID id.EventID Sender types.WhatsAppID Timestamp uint64 Content *waProto.Message diff --git a/database/portal.go b/database/portal.go index 77fb162..6164266 100644 --- a/database/portal.go +++ b/database/portal.go @@ -1,5 +1,5 @@ // mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. -// Copyright (C) 2019 Tulir Asokan +// Copyright (C) 2020 Tulir Asokan // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by @@ -23,6 +23,7 @@ import ( log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix-whatsapp/types" + "maunium.net/go/mautrix/id" ) type PortalKey struct { @@ -74,7 +75,7 @@ func (pq *PortalQuery) GetByJID(key PortalKey) *Portal { return pq.get("SELECT * FROM portal WHERE jid=$1 AND receiver=$2", key.JID, key.Receiver) } -func (pq *PortalQuery) GetByMXID(mxid types.MatrixRoomID) *Portal { +func (pq *PortalQuery) GetByMXID(mxid id.RoomID) *Portal { return pq.get("SELECT * FROM portal WHERE mxid=$1", mxid) } @@ -107,12 +108,12 @@ type Portal struct { log log.Logger Key PortalKey - MXID types.MatrixRoomID + MXID id.RoomID Name string Topic string Avatar string - AvatarURL string + AvatarURL id.ContentURI } func (portal *Portal) Scan(row Scannable) *Portal { @@ -124,12 +125,12 @@ func (portal *Portal) Scan(row Scannable) *Portal { } return nil } - portal.MXID = mxid.String - portal.AvatarURL = avatarURL.String + portal.MXID = id.RoomID(mxid.String) + portal.AvatarURL, _ = id.ParseContentURI(avatarURL.String) return portal } -func (portal *Portal) mxidPtr() *string { +func (portal *Portal) mxidPtr() *id.RoomID { if len(portal.MXID) > 0 { return &portal.MXID } @@ -138,19 +139,19 @@ func (portal *Portal) mxidPtr() *string { func (portal *Portal) Insert() { _, err := portal.db.Exec("INSERT INTO portal VALUES ($1, $2, $3, $4, $5, $6, $7)", - portal.Key.JID, portal.Key.Receiver, portal.mxidPtr(), portal.Name, portal.Topic, portal.Avatar, portal.AvatarURL) + portal.Key.JID, portal.Key.Receiver, portal.mxidPtr(), portal.Name, portal.Topic, portal.Avatar, portal.AvatarURL.String()) if err != nil { portal.log.Warnfln("Failed to insert %s: %v", portal.Key, err) } } func (portal *Portal) Update() { - var mxid *string + var mxid *id.RoomID if len(portal.MXID) > 0 { mxid = &portal.MXID } _, err := portal.db.Exec("UPDATE portal SET mxid=$1, name=$2, topic=$3, avatar=$4, avatar_url=$5 WHERE jid=$6 AND receiver=$7", - mxid, portal.Name, portal.Topic, portal.Avatar, portal.AvatarURL, portal.Key.JID, portal.Key.Receiver) + mxid, portal.Name, portal.Topic, portal.Avatar, portal.AvatarURL.String(), portal.Key.JID, portal.Key.Receiver) if err != nil { portal.log.Warnfln("Failed to update %s: %v", portal.Key, err) } @@ -163,7 +164,7 @@ func (portal *Portal) Delete() { } } -func (portal *Portal) GetUserIDs() []types.MatrixUserID { +func (portal *Portal) GetUserIDs() []id.UserID { rows, err := portal.db.Query(`SELECT "user".mxid FROM "user", user_portal WHERE "user".jid=user_portal.user_jid AND user_portal.portal_jid=$1 @@ -173,9 +174,9 @@ func (portal *Portal) GetUserIDs() []types.MatrixUserID { portal.log.Debugln("Failed to get portal user ids:", err) return nil } - var userIDs []types.MatrixUserID + var userIDs []id.UserID for rows.Next() { - var userID types.MatrixUserID + var userID id.UserID err = rows.Scan(&userID) if err != nil { portal.log.Warnln("Failed to scan row:", err) diff --git a/database/puppet.go b/database/puppet.go index bb35923..6b9ea21 100644 --- a/database/puppet.go +++ b/database/puppet.go @@ -1,5 +1,5 @@ // mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. -// Copyright (C) 2019 Tulir Asokan +// Copyright (C) 2020 Tulir Asokan // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by @@ -22,6 +22,7 @@ import ( log "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix-whatsapp/types" + "maunium.net/go/mautrix/id" ) type PuppetQuery struct { @@ -56,7 +57,7 @@ func (pq *PuppetQuery) Get(jid types.WhatsAppID) *Puppet { return pq.New().Scan(row) } -func (pq *PuppetQuery) GetByCustomMXID(mxid types.MatrixUserID) *Puppet { +func (pq *PuppetQuery) GetByCustomMXID(mxid id.UserID) *Puppet { row := pq.db.QueryRow("SELECT jid, avatar, avatar_url, displayname, name_quality, custom_mxid, access_token, next_batch FROM puppet WHERE custom_mxid=$1", mxid) if row == nil { return nil @@ -82,11 +83,11 @@ type Puppet struct { JID types.WhatsAppID Avatar string - AvatarURL string + AvatarURL id.ContentURI Displayname string NameQuality int8 - CustomMXID string + CustomMXID id.UserID AccessToken string NextBatch string } @@ -103,9 +104,9 @@ func (puppet *Puppet) Scan(row Scannable) *Puppet { } puppet.Displayname = displayname.String puppet.Avatar = avatar.String - puppet.AvatarURL = avatarURL.String + puppet.AvatarURL, _ = id.ParseContentURI(avatarURL.String) puppet.NameQuality = int8(quality.Int64) - puppet.CustomMXID = customMXID.String + puppet.CustomMXID = id.UserID(customMXID.String) puppet.AccessToken = accessToken.String puppet.NextBatch = nextBatch.String return puppet @@ -113,7 +114,7 @@ func (puppet *Puppet) Scan(row Scannable) *Puppet { func (puppet *Puppet) Insert() { _, err := puppet.db.Exec("INSERT INTO puppet (jid, avatar, avatar_url, displayname, name_quality, custom_mxid, access_token, next_batch) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", - puppet.JID, puppet.Avatar, puppet.AvatarURL, puppet.Displayname, puppet.NameQuality, puppet.CustomMXID, puppet.AccessToken, puppet.NextBatch) + puppet.JID, puppet.Avatar, puppet.AvatarURL.String(), puppet.Displayname, puppet.NameQuality, puppet.CustomMXID, puppet.AccessToken, puppet.NextBatch) if err != nil { puppet.log.Warnfln("Failed to insert %s: %v", puppet.JID, err) } @@ -121,7 +122,7 @@ func (puppet *Puppet) Insert() { func (puppet *Puppet) Update() { _, err := puppet.db.Exec("UPDATE puppet SET displayname=$1, name_quality=$2, avatar=$3, avatar_url=$4, custom_mxid=$5, access_token=$6, next_batch=$7 WHERE jid=$8", - puppet.Displayname, puppet.NameQuality, puppet.Avatar, puppet.AvatarURL, puppet.CustomMXID, puppet.AccessToken, puppet.NextBatch, puppet.JID) + puppet.Displayname, puppet.NameQuality, puppet.Avatar, puppet.AvatarURL.String(), puppet.CustomMXID, puppet.AccessToken, puppet.NextBatch, puppet.JID) if err != nil { puppet.log.Warnfln("Failed to update %s->%s: %v", puppet.JID, err) } diff --git a/database/statestore.go b/database/statestore.go index 0768219..ff2e41f 100644 --- a/database/statestore.go +++ b/database/statestore.go @@ -1,5 +1,5 @@ // mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. -// Copyright (C) 2019 Tulir Asokan +// Copyright (C) 2020 Tulir Asokan // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by @@ -24,8 +24,9 @@ import ( log "maunium.net/go/maulogger/v2" - "maunium.net/go/mautrix" "maunium.net/go/mautrix-appservice" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" ) type SQLStateStore struct { @@ -34,7 +35,7 @@ type SQLStateStore struct { db *Database log log.Logger - Typing map[string]map[string]int64 + Typing map[id.RoomID]map[id.UserID]int64 typingLock sync.RWMutex } @@ -46,7 +47,7 @@ func NewSQLStateStore(db *Database) *SQLStateStore { } } -func (store *SQLStateStore) IsRegistered(userID string) bool { +func (store *SQLStateStore) IsRegistered(userID id.UserID) bool { row := store.db.QueryRow("SELECT EXISTS(SELECT 1 FROM mx_registrations WHERE user_id=$1)", userID) var isRegistered bool err := row.Scan(&isRegistered) @@ -56,7 +57,7 @@ func (store *SQLStateStore) IsRegistered(userID string) bool { return isRegistered } -func (store *SQLStateStore) MarkRegistered(userID string) { +func (store *SQLStateStore) MarkRegistered(userID id.UserID) { var err error if store.db.dialect == "postgres" { _, err = store.db.Exec("INSERT INTO mx_registrations (user_id) VALUES ($1) ON CONFLICT (user_id) DO NOTHING", userID) @@ -70,28 +71,28 @@ func (store *SQLStateStore) MarkRegistered(userID string) { } } -func (store *SQLStateStore) GetRoomMembers(roomID string) map[string]mautrix.Member { - members := make(map[string]mautrix.Member) +func (store *SQLStateStore) GetRoomMembers(roomID id.RoomID) map[id.UserID]*event.MemberEventContent { + members := make(map[id.UserID]*event.MemberEventContent) rows, err := store.db.Query("SELECT user_id, membership, displayname, avatar_url FROM mx_user_profile WHERE room_id=$1", roomID) if err != nil { return members } - var userID string - var member mautrix.Member + var userID id.UserID + var member event.MemberEventContent for rows.Next() { err := rows.Scan(&userID, &member.Membership, &member.Displayname, &member.AvatarURL) if err != nil { store.log.Warnfln("Failed to scan member in %s: %v", roomID, err) } else { - members[userID] = member + members[userID] = &member } } return members } -func (store *SQLStateStore) GetMembership(roomID, userID string) mautrix.Membership { +func (store *SQLStateStore) GetMembership(roomID id.RoomID, userID id.UserID) event.Membership { row := store.db.QueryRow("SELECT membership FROM mx_user_profile WHERE room_id=$1 AND user_id=$2", roomID, userID) - membership := mautrix.MembershipLeave + membership := event.MembershipLeave err := row.Scan(&membership) if err != nil && err != sql.ErrNoRows { store.log.Warnfln("Failed to scan membership of %s in %s: %v", userID, roomID, err) @@ -99,33 +100,33 @@ func (store *SQLStateStore) GetMembership(roomID, userID string) mautrix.Members return membership } -func (store *SQLStateStore) GetMember(roomID, userID string) mautrix.Member { +func (store *SQLStateStore) GetMember(roomID id.RoomID, userID id.UserID) *event.MemberEventContent { member, ok := store.TryGetMember(roomID, userID) if !ok { - member.Membership = mautrix.MembershipLeave + member.Membership = event.MembershipLeave } return member } -func (store *SQLStateStore) TryGetMember(roomID, userID string) (mautrix.Member, bool) { +func (store *SQLStateStore) TryGetMember(roomID id.RoomID, userID id.UserID) (*event.MemberEventContent, bool) { row := store.db.QueryRow("SELECT membership, displayname, avatar_url FROM mx_user_profile WHERE room_id=$1 AND user_id=$2", roomID, userID) - var member mautrix.Member + var member event.MemberEventContent err := row.Scan(&member.Membership, &member.Displayname, &member.AvatarURL) if err != nil && err != sql.ErrNoRows { store.log.Warnfln("Failed to scan member info of %s in %s: %v", userID, roomID, err) } - return member, err == nil + return &member, err == nil } -func (store *SQLStateStore) IsInRoom(roomID, userID string) bool { +func (store *SQLStateStore) IsInRoom(roomID id.RoomID, userID id.UserID) bool { return store.IsMembership(roomID, userID, "join") } -func (store *SQLStateStore) IsInvited(roomID, userID string) bool { +func (store *SQLStateStore) IsInvited(roomID id.RoomID, userID id.UserID) bool { return store.IsMembership(roomID, userID, "join", "invite") } -func (store *SQLStateStore) IsMembership(roomID, userID string, allowedMemberships ...mautrix.Membership) bool { +func (store *SQLStateStore) IsMembership(roomID id.RoomID, userID id.UserID, allowedMemberships ...event.Membership) bool { membership := store.GetMembership(roomID, userID) for _, allowedMembership := range allowedMemberships { if allowedMembership == membership { @@ -135,7 +136,7 @@ func (store *SQLStateStore) IsMembership(roomID, userID string, allowedMembershi return false } -func (store *SQLStateStore) SetMembership(roomID, userID string, membership mautrix.Membership) { +func (store *SQLStateStore) SetMembership(roomID id.RoomID, userID id.UserID, membership event.Membership) { var err error if store.db.dialect == "postgres" { _, err = store.db.Exec(`INSERT INTO mx_user_profile (room_id, user_id, membership) VALUES ($1, $2, $3) @@ -150,7 +151,7 @@ func (store *SQLStateStore) SetMembership(roomID, userID string, membership maut } } -func (store *SQLStateStore) SetMember(roomID, userID string, member mautrix.Member) { +func (store *SQLStateStore) SetMember(roomID id.RoomID, userID id.UserID, member *event.MemberEventContent) { var err error if store.db.dialect == "postgres" { _, err = store.db.Exec(`INSERT INTO mx_user_profile (room_id, user_id, membership, displayname, avatar_url) VALUES ($1, $2, $3, $4, $5) @@ -166,7 +167,7 @@ func (store *SQLStateStore) SetMember(roomID, userID string, member mautrix.Memb } } -func (store *SQLStateStore) SetPowerLevels(roomID string, levels *mautrix.PowerLevels) { +func (store *SQLStateStore) SetPowerLevels(roomID id.RoomID, levels *event.PowerLevelsEventContent) { levelsBytes, err := json.Marshal(levels) if err != nil { store.log.Errorfln("Failed to marshal power levels of %s: %v", roomID, err) @@ -185,7 +186,7 @@ func (store *SQLStateStore) SetPowerLevels(roomID string, levels *mautrix.PowerL } } -func (store *SQLStateStore) GetPowerLevels(roomID string) (levels *mautrix.PowerLevels) { +func (store *SQLStateStore) GetPowerLevels(roomID id.RoomID) (levels *event.PowerLevelsEventContent) { row := store.db.QueryRow("SELECT power_levels FROM mx_room_state WHERE room_id=$1", roomID) if row == nil { return @@ -196,7 +197,7 @@ func (store *SQLStateStore) GetPowerLevels(roomID string) (levels *mautrix.Power store.log.Errorln("Failed to scan power levels of %s: %v", roomID, err) return } - levels = &mautrix.PowerLevels{} + levels = &event.PowerLevelsEventContent{} err = json.Unmarshal(data, levels) if err != nil { store.log.Errorln("Failed to parse power levels of %s: %v", roomID, err) @@ -205,7 +206,7 @@ func (store *SQLStateStore) GetPowerLevels(roomID string) (levels *mautrix.Power return } -func (store *SQLStateStore) GetPowerLevel(roomID, userID string) int { +func (store *SQLStateStore) GetPowerLevel(roomID id.RoomID, userID id.UserID) int { if store.db.dialect == "postgres" { row := store.db.QueryRow(`SELECT COALESCE((power_levels->'users'->$2)::int, (power_levels->'users_default')::int, 0) @@ -224,7 +225,7 @@ func (store *SQLStateStore) GetPowerLevel(roomID, userID string) int { return store.GetPowerLevels(roomID).GetUserLevel(userID) } -func (store *SQLStateStore) GetPowerLevelRequirement(roomID string, eventType mautrix.EventType) int { +func (store *SQLStateStore) GetPowerLevelRequirement(roomID id.RoomID, eventType event.Type) int { if store.db.dialect == "postgres" { defaultType := "events_default" defaultValue := 0 @@ -249,7 +250,7 @@ func (store *SQLStateStore) GetPowerLevelRequirement(roomID string, eventType ma return store.GetPowerLevels(roomID).GetEventLevel(eventType) } -func (store *SQLStateStore) HasPowerLevel(roomID, userID string, eventType mautrix.EventType) bool { +func (store *SQLStateStore) HasPowerLevel(roomID id.RoomID, userID id.UserID, eventType event.Type) bool { if store.db.dialect == "postgres" { defaultType := "events_default" defaultValue := 0 diff --git a/database/upgrades/2019-08-25-move-state-store-to-db.go b/database/upgrades/2019-08-25-move-state-store-to-db.go index 608ad81..cbb6001 100644 --- a/database/upgrades/2019-08-25-move-state-store-to-db.go +++ b/database/upgrades/2019-08-25-move-state-store-to-db.go @@ -8,7 +8,7 @@ import ( "os" "strings" - "maunium.net/go/mautrix" + "maunium.net/go/mautrix/event" ) func init() { @@ -46,7 +46,7 @@ func init() { return executeBatch(tx, valueStrings, values...) } - migrateMemberships := func(tx *sql.Tx, rooms map[string]map[string]mautrix.Membership) error { + migrateMemberships := func(tx *sql.Tx, rooms map[string]map[string]event.Membership) error { for roomID, members := range rooms { if len(members) == 0 { continue @@ -68,7 +68,7 @@ func init() { return nil } - migratePowerLevels := func(tx *sql.Tx, rooms map[string]*mautrix.PowerLevels) error { + migratePowerLevels := func(tx *sql.Tx, rooms map[string]*event.PowerLevelsEventContent) error { if len(rooms) == 0 { return nil } @@ -106,9 +106,9 @@ func init() { )` type TempStateStore struct { - Registrations map[string]bool `json:"registrations"` - Members map[string]map[string]mautrix.Membership `json:"memberships"` - PowerLevels map[string]*mautrix.PowerLevels `json:"power_levels"` + Registrations map[string]bool `json:"registrations"` + Members map[string]map[string]event.Membership `json:"memberships"` + PowerLevels map[string]*event.PowerLevelsEventContent `json:"power_levels"` } upgrades[9] = upgrade{"Move state store to main DB", func(tx *sql.Tx, ctx context) error { diff --git a/database/user.go b/database/user.go index 9840778..2e820ff 100644 --- a/database/user.go +++ b/database/user.go @@ -1,5 +1,5 @@ // mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. -// Copyright (C) 2019 Tulir Asokan +// Copyright (C) 2020 Tulir Asokan // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by @@ -28,6 +28,7 @@ import ( "maunium.net/go/mautrix-whatsapp/types" "maunium.net/go/mautrix-whatsapp/whatsapp-ext" + "maunium.net/go/mautrix/id" ) type UserQuery struct { @@ -54,7 +55,7 @@ func (uq *UserQuery) GetAll() (users []*User) { return } -func (uq *UserQuery) GetByMXID(userID types.MatrixUserID) *User { +func (uq *UserQuery) GetByMXID(userID id.UserID) *User { row := uq.db.QueryRow(`SELECT mxid, jid, management_room, last_connection, client_id, client_token, server_token, enc_key, mac_key FROM "user" WHERE mxid=$1`, userID) if row == nil { return nil @@ -74,9 +75,9 @@ type User struct { db *Database log log.Logger - MXID types.MatrixUserID + MXID id.UserID JID types.WhatsAppID - ManagementRoom types.MatrixRoomID + ManagementRoom id.RoomID Session *whatsapp.Session LastConnection uint64 } diff --git a/formatting.go b/formatting.go index cef32bc..7880790 100644 --- a/formatting.go +++ b/formatting.go @@ -1,5 +1,5 @@ // mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. -// Copyright (C) 2019 Tulir Asokan +// Copyright (C) 2020 Tulir Asokan // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by @@ -22,8 +22,9 @@ import ( "regexp" "strings" - "maunium.net/go/mautrix" + "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/format" + "maunium.net/go/mautrix/id" "maunium.net/go/mautrix-whatsapp/types" "maunium.net/go/mautrix-whatsapp/whatsapp-ext" @@ -54,8 +55,7 @@ func NewFormatter(bridge *Bridge) *Formatter { PillConverter: func(mxid, eventID string) string { if mxid[0] == '@' { - puppet := bridge.GetPuppetByMXID(mxid) - fmt.Println(mxid, puppet) + puppet := bridge.GetPuppetByMXID(id.UserID(mxid)) if puppet != nil { return "@" + puppet.PhoneNumber() } @@ -106,10 +106,10 @@ func NewFormatter(bridge *Bridge) *Formatter { return formatter } -func (formatter *Formatter) getMatrixInfoByJID(jid types.WhatsAppID) (mxid, displayname string) { +func (formatter *Formatter) getMatrixInfoByJID(jid types.WhatsAppID) (mxid id.UserID, displayname string) { if user := formatter.bridge.GetUserByJID(jid); user != nil { mxid = user.MXID - displayname = user.MXID + displayname = string(user.MXID) } else if puppet := formatter.bridge.GetPuppetByJID(jid); puppet != nil { mxid = puppet.MXID displayname = puppet.Displayname @@ -117,7 +117,7 @@ func (formatter *Formatter) getMatrixInfoByJID(jid types.WhatsAppID) (mxid, disp return } -func (formatter *Formatter) ParseWhatsApp(content *mautrix.Content) { +func (formatter *Formatter) ParseWhatsApp(content *event.MessageEventContent) { output := html.EscapeString(content.Body) for regex, replacement := range formatter.waReplString { output = regex.ReplaceAllString(output, replacement) @@ -128,7 +128,7 @@ func (formatter *Formatter) ParseWhatsApp(content *mautrix.Content) { if output != content.Body { output = strings.Replace(output, "\n", "
", -1) content.FormattedBody = output - content.Format = mautrix.FormatHTML + content.Format = event.FormatHTML for regex, replacer := range formatter.waReplFuncText { content.Body = regex.ReplaceAllStringFunc(content.Body, replacer) } diff --git a/go.mod b/go.mod index 4a6559c..2b6cc1c 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.14 require ( github.com/Rhymen/go-whatsapp v0.1.0 github.com/chai2010/webp v1.1.0 - github.com/gorilla/websocket v1.4.1 - github.com/lib/pq v1.3.0 + github.com/gorilla/websocket v1.4.2 + github.com/lib/pq v1.5.2 github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/pkg/errors v0.9.1 github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect @@ -14,8 +14,8 @@ require ( gopkg.in/yaml.v2 v2.2.8 maunium.net/go/mauflag v1.0.0 maunium.net/go/maulogger/v2 v2.1.1 - maunium.net/go/mautrix v0.1.0-beta.2 - maunium.net/go/mautrix-appservice v0.1.0-alpha.6 + maunium.net/go/mautrix v0.3.6 + maunium.net/go/mautrix-appservice v0.2.0 ) replace github.com/Rhymen/go-whatsapp => github.com/tulir/go-whatsapp v0.2.6 diff --git a/go.sum b/go.sum index 19eaba1..3d63304 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,10 @@ github.com/chai2010/webp v1.1.0 h1:4Ei0/BRroMF9FaXDG2e4OxwFcuW2vcXd+A6tyqTJUQQ= github.com/chai2010/webp v1.1.0/go.mod h1:LP12PG5IFmLGHUU26tBiCBKnghxx3toZFwDjOYvd3Ow= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= @@ -12,6 +15,8 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.5.2 h1:yTSXVswvWUOQ3k1sd7vJfDrbSl8lKuscqFJRqjC0ifw= +github.com/lib/pq v1.5.2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -22,12 +27,24 @@ github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/skip2/go-qrcode v0.0.0-20191027152451-9434209cb086 h1:RYiqpb2ii2Z6J4x0wxK46kvPBbFuZcdhS+CIztmYgZs= github.com/skip2/go-qrcode v0.0.0-20191027152451-9434209cb086/go.mod h1:PLPIyL7ikehBD1OAjmKKiOEhbvWyHGaNDjquXMcYABo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= +github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= +github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= +github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8= +github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/sjson v1.1.1 h1:7h1vk049Jnd5EH9NyzNiEuwYW4b5qgreBbqRC19AS3U= +github.com/tidwall/sjson v1.1.1/go.mod h1:yvVuSnpEQv5cYIrO+AT6kw4QVfd5SDZoGIS7/5+fZFs= github.com/tulir/go-whatsapp v0.2.0 h1:JWK/Xxrc1qsZsVz6gYVX5AtvzYmqaHNjt34Ipnrgz88= github.com/tulir/go-whatsapp v0.2.0/go.mod h1:gyw9zGup1/Y3ZQUueZaqz3iR/WX9a2Lth4aqEbXjkok= github.com/tulir/go-whatsapp v0.2.1 h1:Owoss2AbvZMgt3nxoFlsG+bqLHDnO+PhXNhhoCmb/3M= @@ -52,6 +69,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= @@ -60,5 +78,9 @@ maunium.net/go/maulogger/v2 v2.1.1 h1:NAZNc6XUFJzgzfewCzVoGkxNAsblLCSSEdtDuIjP0X maunium.net/go/maulogger/v2 v2.1.1/go.mod h1:TYWy7wKwz/tIXTpsx8G3mZseIRiC5DoMxSZazOHy68A= maunium.net/go/mautrix v0.1.0-beta.2 h1:RxYTqTzW6iXu83gf8ucqGwYx8JLa+a17LWjiPkVV/fU= maunium.net/go/mautrix v0.1.0-beta.2/go.mod h1:YFMU9DBeXH7cqx7sJLg0DkVxwNPbih8QbpUTYf/IjMM= +maunium.net/go/mautrix v0.3.6 h1:bXUo8WFdv7sUpvr7jgJ6TVMEQgVHtw1z1T3eUcLpPCA= +maunium.net/go/mautrix v0.3.6/go.mod h1:SkGZzch8CvU2qKtNpYxtzZ0sQxfVEJ3IsVVLSUBUx9Y= maunium.net/go/mautrix-appservice v0.1.0-alpha.6 h1:dNE+RykOC0UhSyRNbMHXEk3BzSOp3dj8aQwKuNMELWM= maunium.net/go/mautrix-appservice v0.1.0-alpha.6/go.mod h1:Dfiwiuicvn8s2VKrBDrZ9eCjlKUMbuCi91TE6xeEHRM= +maunium.net/go/mautrix-appservice v0.2.0 h1:HmEpBSdGK7/8/xqOhxNP6viSQPkgjFVTfMI33moz51A= +maunium.net/go/mautrix-appservice v0.2.0/go.mod h1:55u7GKZBfxIs6tfAQTpvLvKLUjaJvll5HLcmx5Set1A= diff --git a/main.go b/main.go index ca4aa11..cb9667c 100644 --- a/main.go +++ b/main.go @@ -1,5 +1,5 @@ // mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. -// Copyright (C) 2019 Tulir Asokan +// Copyright (C) 2020 Tulir Asokan // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by @@ -18,7 +18,6 @@ package main import ( "fmt" - "net/http" "os" "os/signal" "sync" @@ -30,6 +29,7 @@ import ( "maunium.net/go/mautrix" "maunium.net/go/mautrix-appservice" + "maunium.net/go/mautrix/id" "maunium.net/go/mautrix-whatsapp/config" "maunium.net/go/mautrix-whatsapp/database" @@ -107,28 +107,28 @@ type Bridge struct { Formatter *Formatter Relaybot *User - usersByMXID map[types.MatrixUserID]*User + usersByMXID map[id.UserID]*User usersByJID map[types.WhatsAppID]*User usersLock sync.Mutex - managementRooms map[types.MatrixRoomID]*User + managementRooms map[id.RoomID]*User managementRoomsLock sync.Mutex - portalsByMXID map[types.MatrixRoomID]*Portal + portalsByMXID map[id.RoomID]*Portal portalsByJID map[database.PortalKey]*Portal portalsLock sync.Mutex puppets map[types.WhatsAppID]*Puppet - puppetsByCustomMXID map[types.MatrixUserID]*Puppet + puppetsByCustomMXID map[id.UserID]*Puppet puppetsLock sync.Mutex } func NewBridge() *Bridge { bridge := &Bridge{ - usersByMXID: make(map[types.MatrixUserID]*User), + usersByMXID: make(map[id.UserID]*User), usersByJID: make(map[types.WhatsAppID]*User), - managementRooms: make(map[types.MatrixRoomID]*User), - portalsByMXID: make(map[types.MatrixRoomID]*Portal), + managementRooms: make(map[id.RoomID]*User), + portalsByMXID: make(map[id.RoomID]*Portal), portalsByJID: make(map[database.PortalKey]*Portal), puppets: make(map[types.WhatsAppID]*Puppet), - puppetsByCustomMXID: make(map[types.MatrixUserID]*Puppet), + puppetsByCustomMXID: make(map[id.UserID]*Puppet), } var err error @@ -141,12 +141,8 @@ func NewBridge() *Bridge { } func (bridge *Bridge) ensureConnection() { - url := bridge.Bot.BuildURL("account", "whoami") - resp := struct { - UserID string `json:"user_id"` - }{} for { - _, err := bridge.Bot.MakeRequest(http.MethodGet, url, nil, &resp) + resp, err := bridge.Bot.Whoami() if err != nil { if httpErr, ok := err.(mautrix.HTTPError); ok && httpErr.RespError != nil && httpErr.RespError.ErrCode == "M_UNKNOWN_ACCESS_TOKEN" { bridge.Log.Fatalln("Access token invalid. Is the registration installed in your homeserver correctly?") @@ -262,10 +258,14 @@ func (bridge *Bridge) UpdateBotProfile() { botConfig := bridge.Config.AppService.Bot var err error + var mxc id.ContentURI if botConfig.Avatar == "remove" { - err = bridge.Bot.SetAvatarURL("") + err = bridge.Bot.SetAvatarURL(mxc) } else if len(botConfig.Avatar) > 0 { - err = bridge.Bot.SetAvatarURL(botConfig.Avatar) + mxc, err = id.ParseContentURI(botConfig.Avatar) + if err == nil { + err = bridge.Bot.SetAvatarURL(mxc) + } } if err != nil { bridge.Log.Warnln("Failed to update bot avatar:", err) diff --git a/matrix.go b/matrix.go index dfb6668..3f30be7 100644 --- a/matrix.go +++ b/matrix.go @@ -1,5 +1,5 @@ // mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. -// Copyright (C) 2019 Tulir Asokan +// Copyright (C) 2020 Tulir Asokan // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by @@ -22,11 +22,10 @@ import ( "maunium.net/go/maulogger/v2" - "maunium.net/go/mautrix" "maunium.net/go/mautrix-appservice" + "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/format" - - "maunium.net/go/mautrix-whatsapp/types" + "maunium.net/go/mautrix/id" ) type MatrixHandler struct { @@ -43,17 +42,17 @@ func NewMatrixHandler(bridge *Bridge) *MatrixHandler { log: bridge.Log.Sub("Matrix"), cmd: NewCommandHandler(bridge), } - bridge.EventProcessor.On(mautrix.EventMessage, handler.HandleMessage) - bridge.EventProcessor.On(mautrix.EventSticker, handler.HandleMessage) - bridge.EventProcessor.On(mautrix.EventRedaction, handler.HandleRedaction) - bridge.EventProcessor.On(mautrix.StateMember, handler.HandleMembership) - bridge.EventProcessor.On(mautrix.StateRoomName, handler.HandleRoomMetadata) - bridge.EventProcessor.On(mautrix.StateRoomAvatar, handler.HandleRoomMetadata) - bridge.EventProcessor.On(mautrix.StateTopic, handler.HandleRoomMetadata) + bridge.EventProcessor.On(event.EventMessage, handler.HandleMessage) + bridge.EventProcessor.On(event.EventSticker, handler.HandleMessage) + bridge.EventProcessor.On(event.EventRedaction, handler.HandleRedaction) + bridge.EventProcessor.On(event.StateMember, handler.HandleMembership) + bridge.EventProcessor.On(event.StateRoomName, handler.HandleRoomMetadata) + bridge.EventProcessor.On(event.StateRoomAvatar, handler.HandleRoomMetadata) + bridge.EventProcessor.On(event.StateTopic, handler.HandleRoomMetadata) return handler } -func (mx *MatrixHandler) HandleBotInvite(evt *mautrix.Event) { +func (mx *MatrixHandler) HandleBotInvite(evt *event.Event) { intent := mx.as.BotIntent() user := mx.bridge.GetUserByMXID(evt.Sender) @@ -61,7 +60,7 @@ func (mx *MatrixHandler) HandleBotInvite(evt *mautrix.Event) { return } - resp, err := intent.JoinRoom(evt.RoomID, "", nil) + resp, err := intent.JoinRoomByID(evt.RoomID) if err != nil { mx.log.Debugln("Failed to join room", evt.RoomID, "with invite from", evt.Sender) return @@ -97,7 +96,7 @@ func (mx *MatrixHandler) HandleBotInvite(evt *mautrix.Event) { for mxid, _ := range members.Joined { if mxid == intent.UserID || mxid == evt.Sender { continue - } else if _, ok := mx.bridge.ParsePuppetMXID(types.MatrixUserID(mxid)); ok { + } else if _, ok := mx.bridge.ParsePuppetMXID(mxid); ok { hasPuppets = true continue } @@ -108,15 +107,16 @@ func (mx *MatrixHandler) HandleBotInvite(evt *mautrix.Event) { } if !hasPuppets { - user := mx.bridge.GetUserByMXID(types.MatrixUserID(evt.Sender)) - user.SetManagementRoom(types.MatrixRoomID(resp.RoomID)) - intent.SendNotice(string(user.ManagementRoom), "This room has been registered as your bridge management/status room. Send `help` to get a list of commands.") + user := mx.bridge.GetUserByMXID(evt.Sender) + user.SetManagementRoom(resp.RoomID) + intent.SendNotice(user.ManagementRoom, "This room has been registered as your bridge management/status room. Send `help` to get a list of commands.") mx.log.Debugln(resp.RoomID, "registered as a management room with", evt.Sender) } } -func (mx *MatrixHandler) HandleMembership(evt *mautrix.Event) { - if evt.Content.Membership == "invite" && evt.GetStateKey() == mx.as.BotMXID() { +func (mx *MatrixHandler) HandleMembership(evt *event.Event) { + content := evt.Content.AsMember() + if content.Membership == event.MembershipInvite && id.UserID(evt.GetStateKey()) == mx.as.BotMXID() { mx.HandleBotInvite(evt) } @@ -125,15 +125,21 @@ func (mx *MatrixHandler) HandleMembership(evt *mautrix.Event) { return } - user := mx.bridge.GetUserByMXID(types.MatrixUserID(evt.Sender)) + user := mx.bridge.GetUserByMXID(id.UserID(evt.Sender)) if user == nil || !user.Whitelisted || !user.IsConnected() { return } - if evt.Content.Membership == "leave" { - if evt.GetStateKey() == evt.Sender { - if portal.IsPrivateChat() || evt.Unsigned.PrevContent.Membership == "join" { - portal.HandleMatrixLeave(user) + if content.Membership == event.MembershipLeave { + if id.UserID(evt.GetStateKey()) == evt.Sender { + if evt.Unsigned.PrevContent != nil { + _ = evt.Unsigned.PrevContent.ParseRaw(evt.Type) + prevContent, ok := evt.Unsigned.PrevContent.Parsed.(*event.MemberEventContent) + if ok { + if portal.IsPrivateChat() || prevContent.Membership == "join" { + portal.HandleMatrixLeave(user) + } + } } } else { portal.HandleMatrixKick(user, evt) @@ -141,8 +147,8 @@ func (mx *MatrixHandler) HandleMembership(evt *mautrix.Event) { } } -func (mx *MatrixHandler) HandleRoomMetadata(evt *mautrix.Event) { - user := mx.bridge.GetUserByMXID(types.MatrixUserID(evt.Sender)) +func (mx *MatrixHandler) HandleRoomMetadata(evt *event.Event) { + user := mx.bridge.GetUserByMXID(id.UserID(evt.Sender)) if user == nil || !user.Whitelisted || !user.IsConnected() { return } @@ -154,12 +160,12 @@ func (mx *MatrixHandler) HandleRoomMetadata(evt *mautrix.Event) { var resp <-chan string var err error - switch evt.Type { - case mautrix.StateRoomName: - resp, err = user.Conn.UpdateGroupSubject(evt.Content.Name, portal.Key.JID) - case mautrix.StateTopic: - resp, err = user.Conn.UpdateGroupDescription(portal.Key.JID, evt.Content.Topic) - case mautrix.StateRoomAvatar: + switch content := evt.Content.Parsed.(type) { + case *event.RoomNameEventContent: + resp, err = user.Conn.UpdateGroupSubject(content.Name, portal.Key.JID) + case *event.TopicEventContent: + resp, err = user.Conn.UpdateGroupDescription(portal.Key.JID, content.Topic) + case *event.RoomAvatarEventContent: return } if err != nil { @@ -170,7 +176,7 @@ func (mx *MatrixHandler) HandleRoomMetadata(evt *mautrix.Event) { } } -func (mx *MatrixHandler) HandleMessage(evt *mautrix.Event) { +func (mx *MatrixHandler) HandleMessage(evt *event.Event) { if _, isPuppet := mx.bridge.ParsePuppetMXID(evt.Sender); evt.Sender == mx.bridge.Bot.UserID || isPuppet { return } @@ -179,38 +185,37 @@ func (mx *MatrixHandler) HandleMessage(evt *mautrix.Event) { return } - roomID := types.MatrixRoomID(evt.RoomID) - user := mx.bridge.GetUserByMXID(types.MatrixUserID(evt.Sender)) + user := mx.bridge.GetUserByMXID(evt.Sender) if !user.RelaybotWhitelisted { return } - if user.Whitelisted && evt.Content.MsgType == mautrix.MsgText { + content := evt.Content.AsMessage() + if user.Whitelisted && content.MsgType == event.MsgText { commandPrefix := mx.bridge.Config.Bridge.CommandPrefix - hasCommandPrefix := strings.HasPrefix(evt.Content.Body, commandPrefix) + hasCommandPrefix := strings.HasPrefix(content.Body, commandPrefix) if hasCommandPrefix { - evt.Content.Body = strings.TrimLeft(evt.Content.Body[len(commandPrefix):], " ") + content.Body = strings.TrimLeft(content.Body[len(commandPrefix):], " ") } - if hasCommandPrefix || roomID == user.ManagementRoom { - mx.cmd.Handle(roomID, user, evt.Content.Body) + if hasCommandPrefix || evt.RoomID == user.ManagementRoom { + mx.cmd.Handle(evt.RoomID, user, content.Body) return } } - portal := mx.bridge.GetPortalByMXID(roomID) + portal := mx.bridge.GetPortalByMXID(evt.RoomID) if portal != nil && (user.Whitelisted || portal.HasRelaybot()) { portal.HandleMatrixMessage(user, evt) } } -func (mx *MatrixHandler) HandleRedaction(evt *mautrix.Event) { +func (mx *MatrixHandler) HandleRedaction(evt *event.Event) { if _, isPuppet := mx.bridge.ParsePuppetMXID(evt.Sender); evt.Sender == mx.bridge.Bot.UserID || isPuppet { return } - roomID := types.MatrixRoomID(evt.RoomID) - user := mx.bridge.GetUserByMXID(types.MatrixUserID(evt.Sender)) + user := mx.bridge.GetUserByMXID(id.UserID(evt.Sender)) if !user.Whitelisted { return @@ -221,13 +226,13 @@ func (mx *MatrixHandler) HandleRedaction(evt *mautrix.Event) { } else if !user.IsConnected() { msg := format.RenderMarkdown(fmt.Sprintf("[%[1]s](https://matrix.to/#/%[1]s): \u26a0 "+ "You are not connected to WhatsApp, so your redaction was not bridged. "+ - "Use `%[2]s reconnect` to reconnect.", user.MXID, mx.bridge.Config.Bridge.CommandPrefix)) - msg.MsgType = mautrix.MsgNotice - _, _ = mx.bridge.Bot.SendMessageEvent(roomID, mautrix.EventMessage, msg) + "Use `%[2]s reconnect` to reconnect.", user.MXID, mx.bridge.Config.Bridge.CommandPrefix), true, false) + msg.MsgType = event.MsgNotice + _, _ = mx.bridge.Bot.SendMessageEvent(evt.RoomID, event.EventMessage, msg) return } - portal := mx.bridge.GetPortalByMXID(roomID) + portal := mx.bridge.GetPortalByMXID(evt.RoomID) if portal != nil { portal.HandleMatrixRedaction(user, evt) } diff --git a/portal.go b/portal.go index b63fc7e..1fb6a7a 100644 --- a/portal.go +++ b/portal.go @@ -1,5 +1,5 @@ // mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. -// Copyright (C) 2019 Tulir Asokan +// Copyright (C) 2020 Tulir Asokan // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by @@ -20,7 +20,6 @@ import ( "bytes" "encoding/gob" "encoding/hex" - "encoding/json" "fmt" "html" "image" @@ -43,14 +42,16 @@ import ( "maunium.net/go/mautrix" "maunium.net/go/mautrix-appservice" + "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/format" + "maunium.net/go/mautrix/id" "maunium.net/go/mautrix-whatsapp/database" "maunium.net/go/mautrix-whatsapp/types" "maunium.net/go/mautrix-whatsapp/whatsapp-ext" ) -func (bridge *Bridge) GetPortalByMXID(mxid types.MatrixRoomID) *Portal { +func (bridge *Bridge) GetPortalByMXID(mxid id.RoomID) *Portal { bridge.portalsLock.Lock() defer bridge.portalsLock.Unlock() portal, ok := bridge.portalsByMXID[mxid] @@ -233,7 +234,7 @@ func init() { gob.Register(&waProto.Message{}) } -func (portal *Portal) markHandled(source *User, message *waProto.WebMessageInfo, mxid types.MatrixEventID) { +func (portal *Portal) markHandled(source *User, message *waProto.WebMessageInfo, mxid id.EventID) { msg := portal.bridge.DB.Message.New() msg.Chat = portal.Key msg.JID = message.GetKey().GetId() @@ -269,7 +270,7 @@ func (portal *Portal) startHandling(info whatsapp.MessageInfo) bool { return true } -func (portal *Portal) finishHandling(source *User, message *waProto.WebMessageInfo, mxid types.MatrixEventID) { +func (portal *Portal) finishHandling(source *User, message *waProto.WebMessageInfo, mxid id.EventID) { portal.markHandled(source, message, mxid) portal.log.Debugln("Handled message", message.GetKey().GetId(), "->", mxid) } @@ -416,7 +417,7 @@ func (portal *Portal) UpdateMetadata(user *User) bool { return update } -func (portal *Portal) userMXIDAction(user *User, fn func(mxid types.MatrixUserID)) { +func (portal *Portal) userMXIDAction(user *User, fn func(mxid id.UserID)) { if user == nil { return } @@ -430,7 +431,7 @@ func (portal *Portal) userMXIDAction(user *User, fn func(mxid types.MatrixUserID } } -func (portal *Portal) ensureMXIDInvited(mxid types.MatrixUserID) { +func (portal *Portal) ensureMXIDInvited(mxid id.UserID) { err := portal.MainIntent().EnsureInvited(portal.MXID, mxid) if err != nil { portal.log.Warnfln("Failed to ensure %s is invited to %s: %v", mxid, portal.MXID, err) @@ -481,27 +482,27 @@ func (portal *Portal) Sync(user *User, contact whatsapp.Contact) { } } -func (portal *Portal) GetBasePowerLevels() *mautrix.PowerLevels { +func (portal *Portal) GetBasePowerLevels() *event.PowerLevelsEventContent { anyone := 0 nope := 99 invite := 99 if portal.bridge.Config.Bridge.AllowUserInvite { invite = 0 } - return &mautrix.PowerLevels{ + return &event.PowerLevelsEventContent{ UsersDefault: anyone, EventsDefault: anyone, RedactPtr: &anyone, StateDefaultPtr: &nope, BanPtr: &nope, InvitePtr: &invite, - Users: map[string]int{ + Users: map[id.UserID]int{ portal.MainIntent().UserID: 100, }, Events: map[string]int{ - mautrix.StateRoomName.Type: anyone, - mautrix.StateRoomAvatar.Type: anyone, - mautrix.StateTopic.Type: anyone, + event.StateRoomName.Type: anyone, + event.StateRoomAvatar.Type: anyone, + event.StateTopic.Type: anyone, }, } } @@ -559,9 +560,9 @@ func (portal *Portal) RestrictMetadataChanges(restrict bool) { newLevel = 50 } changed := false - changed = levels.EnsureEventLevel(mautrix.StateRoomName, newLevel) || changed - changed = levels.EnsureEventLevel(mautrix.StateRoomAvatar, newLevel) || changed - changed = levels.EnsureEventLevel(mautrix.StateTopic, newLevel) || changed + changed = levels.EnsureEventLevel(event.StateRoomName, newLevel) || changed + changed = levels.EnsureEventLevel(event.StateRoomAvatar, newLevel) || changed + changed = levels.EnsureEventLevel(event.StateTopic, newLevel) || changed if changed { _, err = portal.MainIntent().SetPowerLevels(portal.MXID, levels) if err != nil { @@ -749,22 +750,22 @@ func (portal *Portal) CreateMatrixRoom(user *User) error { portal.UpdateAvatar(user, nil) } - initialState := []*mautrix.Event{{ - Type: mautrix.StatePowerLevels, - Content: mautrix.Content{ - PowerLevels: portal.GetBasePowerLevels(), + initialState := []*event.Event{{ + Type: event.StatePowerLevels, + Content: event.Content{ + Parsed: portal.GetBasePowerLevels(), }, }} - if len(portal.AvatarURL) > 0 { - initialState = append(initialState, &mautrix.Event{ - Type: mautrix.StateRoomAvatar, - Content: mautrix.Content{ - URL: portal.AvatarURL, + if !portal.AvatarURL.IsEmpty() { + initialState = append(initialState, &event.Event{ + Type: event.StateRoomAvatar, + Content: event.Content{ + Parsed: event.RoomAvatarEventContent{URL: portal.AvatarURL}, }, }) } - invite := []string{user.MXID} + invite := []id.UserID{user.MXID} if user.IsRelaybot { invite = portal.bridge.Config.Bridge.Relaybot.InviteUsers } @@ -847,19 +848,18 @@ func (portal *Portal) GetMessageIntent(user *User, info whatsapp.MessageInfo) *a return portal.bridge.GetPuppetByJID(info.SenderJid).IntentFor(portal) } -func (portal *Portal) SetReply(content *mautrix.Content, info whatsapp.ContextInfo) { +func (portal *Portal) SetReply(content *event.MessageEventContent, info whatsapp.ContextInfo) { if len(info.QuotedMessageID) == 0 { return } message := portal.bridge.DB.Message.GetByJID(portal.Key, info.QuotedMessageID) if message != nil { - event, err := portal.MainIntent().GetEvent(portal.MXID, message.MXID) + evt, err := portal.MainIntent().GetEvent(portal.MXID, message.MXID) if err != nil { portal.log.Warnln("Failed to get reply target:", err) return } - event.Content.RemoveReplyFallback() - content.SetReply(event) + content.SetReply(evt) } return } @@ -908,32 +908,6 @@ func (portal *Portal) HandleFakeMessage(source *User, message FakeMessage) { portal.recentlyHandled[index] = message.ID } -type MessageContent struct { - *mautrix.Content - IsCustomPuppet bool `json:"net.maunium.whatsapp.puppet,omitempty"` -} - -type serializableContent mautrix.Content - -type serializableMessageContent struct { - *serializableContent - IsCustomPuppet bool `json:"net.maunium.whatsapp.puppet,omitempty"` -} - -// Hacky bypass for mautrix.Content's MarshalSJSON -func (content *MessageContent) MarshalJSON() ([]byte, error) { - if mautrix.DisableFancyEventParsing { - if content.IsCustomPuppet { - content.Raw["net.maunium.whatsapp.puppet"] = content.IsCustomPuppet - } - return json.Marshal(content.Raw) - } - return json.Marshal(&serializableMessageContent{ - serializableContent: (*serializableContent)(content.Content), - IsCustomPuppet: content.IsCustomPuppet, - }) -} - func (portal *Portal) HandleTextMessage(source *User, message whatsapp.TextMessage) { if !portal.startHandling(message.Info) { return @@ -944,16 +918,21 @@ func (portal *Portal) HandleTextMessage(source *User, message whatsapp.TextMessa return } - content := &mautrix.Content{ + content := &event.MessageEventContent{ Body: message.Text, - MsgType: mautrix.MsgText, + MsgType: event.MsgText, } portal.bridge.Formatter.ParseWhatsApp(content) portal.SetReply(content, message.ContextInfo) _, _ = intent.UserTyping(portal.MXID, false, 0) - resp, err := intent.SendMassagedMessageEvent(portal.MXID, mautrix.EventMessage, &MessageContent{content, intent.IsCustomPuppet}, int64(message.Info.Timestamp*1000)) + resp, err := intent.SendMassagedMessageEvent(portal.MXID, event.EventMessage, &event.Content{ + Parsed: content, + Raw: map[string]interface{}{ + "net.maunium.whatsapp.puppet": intent.IsCustomPuppet, + }, + }, int64(message.Info.Timestamp*1000)) if err != nil { portal.log.Errorfln("Failed to handle message %s: %v", message.Info.Id, err) return @@ -1016,10 +995,10 @@ func (portal *Portal) HandleMediaMessage(source *User, download func() ([]byte, fileName += exts[0] } - content := &mautrix.Content{ + content := &event.MessageEventContent{ Body: fileName, - URL: uploaded.ContentURI, - Info: &mautrix.FileInfo{ + URL: uploaded.ContentURI.CUString(), + Info: &event.FileInfo{ Size: len(data), MimeType: mimeType, }, @@ -1030,9 +1009,9 @@ func (portal *Portal) HandleMediaMessage(source *User, download func() ([]byte, thumbnailMime := http.DetectContentType(thumbnail) uploadedThumbnail, _ := intent.UploadBytes(thumbnail, thumbnailMime) if uploadedThumbnail != nil { - content.Info.ThumbnailURL = uploadedThumbnail.ContentURI + content.Info.ThumbnailURL = uploadedThumbnail.ContentURI.CUString() cfg, _, _ := image.DecodeConfig(bytes.NewReader(data)) - content.Info.ThumbnailInfo = &mautrix.FileInfo{ + content.Info.ThumbnailInfo = &event.FileInfo{ Size: len(thumbnail), Width: cfg.Width, Height: cfg.Height, @@ -1044,40 +1023,50 @@ func (portal *Portal) HandleMediaMessage(source *User, download func() ([]byte, switch strings.ToLower(strings.Split(mimeType, "/")[0]) { case "image": if !sendAsSticker { - content.MsgType = mautrix.MsgImage + content.MsgType = event.MsgImage } cfg, _, _ := image.DecodeConfig(bytes.NewReader(data)) content.Info.Width = cfg.Width content.Info.Height = cfg.Height case "video": - content.MsgType = mautrix.MsgVideo + content.MsgType = event.MsgVideo case "audio": - content.MsgType = mautrix.MsgAudio + content.MsgType = event.MsgAudio default: - content.MsgType = mautrix.MsgFile + content.MsgType = event.MsgFile } _, _ = intent.UserTyping(portal.MXID, false, 0) ts := int64(info.Timestamp * 1000) - eventType := mautrix.EventMessage + eventType := event.EventMessage if sendAsSticker { - eventType = mautrix.EventSticker + eventType = event.EventSticker } - resp, err := intent.SendMassagedMessageEvent(portal.MXID, eventType, &MessageContent{content, intent.IsCustomPuppet}, ts) + resp, err := intent.SendMassagedMessageEvent(portal.MXID, eventType, &event.Content{ + Parsed: content, + Raw: map[string]interface{}{ + "net.maunium.whatsapp.puppet": intent.IsCustomPuppet, + }, + }, ts) if err != nil { portal.log.Errorfln("Failed to handle message %s: %v", info.Id, err) return } if len(caption) > 0 { - captionContent := &mautrix.Content{ + captionContent := &event.MessageEventContent{ Body: caption, - MsgType: mautrix.MsgNotice, + MsgType: event.MsgNotice, } portal.bridge.Formatter.ParseWhatsApp(captionContent) - _, err := intent.SendMassagedMessageEvent(portal.MXID, mautrix.EventMessage, &MessageContent{captionContent, intent.IsCustomPuppet}, ts) + _, err := intent.SendMassagedMessageEvent(portal.MXID, event.EventMessage, &event.Content{ + Parsed: content, + Raw: map[string]interface{}{ + "net.maunium.whatsapp.puppet": intent.IsCustomPuppet, + }, + }, ts) if err != nil { portal.log.Warnfln("Failed to handle caption of message %s: %v", info.Id, err) } @@ -1094,14 +1083,17 @@ func makeMessageID() *string { return &str } -func (portal *Portal) downloadThumbnail(evt *mautrix.Event) []byte { - if evt.Content.Info == nil || len(evt.Content.Info.ThumbnailURL) == 0 { +func (portal *Portal) downloadThumbnail(content *event.MessageEventContent, id id.EventID) []byte { + if len(content.GetInfo().ThumbnailURL) == 0 { return nil } - - thumbnail, err := portal.MainIntent().DownloadBytes(evt.Content.Info.ThumbnailURL) + mxc, err := content.GetInfo().ThumbnailURL.Parse() if err != nil { - portal.log.Errorln("Failed to download thumbnail in %s: %v", evt.ID, err) + portal.log.Errorln("Malformed thumbnail URL in %s: %v", id, err) + } + thumbnail, err := portal.MainIntent().DownloadBytes(mxc) + if err != nil { + portal.log.Errorln("Failed to download thumbnail in %s: %v", id, err) return nil } thumbnailType := http.DetectContentType(thumbnail) @@ -1121,30 +1113,31 @@ func (portal *Portal) downloadThumbnail(evt *mautrix.Event) []byte { Quality: jpeg.DefaultQuality, }) if err != nil { - portal.log.Errorln("Failed to re-encode thumbnail in %s: %v", evt.ID, err) + portal.log.Errorln("Failed to re-encode thumbnail in %s: %v", id, err) return nil } return buf.Bytes() } -func (portal *Portal) preprocessMatrixMedia(sender *User, relaybotFormatted bool, evt *mautrix.Event, mediaType whatsapp.MediaType) *MediaUpload { - if evt.Content.Info == nil { - evt.Content.Info = &mautrix.FileInfo{} - } +func (portal *Portal) preprocessMatrixMedia(sender *User, relaybotFormatted bool, content *event.MessageEventContent, id id.EventID, mediaType whatsapp.MediaType) *MediaUpload { var caption string if relaybotFormatted { - caption = portal.bridge.Formatter.ParseMatrix(evt.Content.FormattedBody) + caption = portal.bridge.Formatter.ParseMatrix(content.FormattedBody) } - content, err := portal.MainIntent().DownloadBytes(evt.Content.URL) + mxc, err := content.URL.Parse() if err != nil { - portal.log.Errorfln("Failed to download media in %s: %v", evt.ID, err) + portal.log.Errorln("Malformed content URL in %s: %v", id, err) + } + data, err := portal.MainIntent().DownloadBytes(mxc) + if err != nil { + portal.log.Errorfln("Failed to download media in %s: %v", id, err) return nil } - url, mediaKey, fileEncSHA256, fileSHA256, fileLength, err := sender.Conn.Upload(bytes.NewReader(content), mediaType) + url, mediaKey, fileEncSHA256, fileSHA256, fileLength, err := sender.Conn.Upload(bytes.NewReader(data), mediaType) if err != nil { - portal.log.Errorfln("Failed to upload media in %s: %v", evt.ID, err) + portal.log.Errorfln("Failed to upload media in %s: %v", id, err) return nil } @@ -1155,7 +1148,7 @@ func (portal *Portal) preprocessMatrixMedia(sender *User, relaybotFormatted bool FileEncSHA256: fileEncSHA256, FileSHA256: fileSHA256, FileLength: fileLength, - Thumbnail: portal.downloadThumbnail(evt), + Thumbnail: portal.downloadThumbnail(content, id), } } @@ -1169,7 +1162,7 @@ type MediaUpload struct { Thumbnail []byte } -func (portal *Portal) sendMatrixConnectionError(sender *User, eventID string) bool { +func (portal *Portal) sendMatrixConnectionError(sender *User, eventID id.EventID) bool { if !sender.HasSession() { portal.log.Debugln("Ignoring event", eventID, "from", sender.MXID, "as user has no session") return true @@ -1183,9 +1176,9 @@ func (portal *Portal) sendMatrixConnectionError(sender *User, eventID string) bo if sender.IsLoginInProgress() { reconnect = "You have a login attempt in progress, please wait." } - msg := format.RenderMarkdown("\u26a0 You are not connected to WhatsApp, so your message was not bridged. " + reconnect) - msg.MsgType = mautrix.MsgNotice - _, err := portal.MainIntent().SendMessageEvent(portal.MXID, mautrix.EventMessage, msg) + msg := format.RenderMarkdown("\u26a0 You are not connected to WhatsApp, so your message was not bridged. " + reconnect, true, false) + msg.MsgType = event.MsgNotice + _, err := portal.MainIntent().SendMessageEvent(portal.MXID, event.EventMessage, msg) if err != nil { portal.log.Errorln("Failed to send bridging failure message:", err) } @@ -1194,32 +1187,37 @@ func (portal *Portal) sendMatrixConnectionError(sender *User, eventID string) bo return false } -func (portal *Portal) addRelaybotFormat(user *User, evt *mautrix.Event) bool { - member := portal.MainIntent().Member(portal.MXID, evt.Sender) +func (portal *Portal) addRelaybotFormat(sender *User, content *event.MessageEventContent) bool { + member := portal.MainIntent().Member(portal.MXID, sender.MXID) if len(member.Displayname) == 0 { - member.Displayname = evt.Sender + member.Displayname = string(sender.MXID) } - if evt.Content.Format != mautrix.FormatHTML { - evt.Content.FormattedBody = strings.Replace(html.EscapeString(evt.Content.Body), "\n", "
", -1) - evt.Content.Format = mautrix.FormatHTML + if content.Format != event.FormatHTML { + content.FormattedBody = strings.Replace(html.EscapeString(content.Body), "\n", "
", -1) + content.Format = event.FormatHTML } - data, err := portal.bridge.Config.Bridge.Relaybot.FormatMessage(evt, member) + data, err := portal.bridge.Config.Bridge.Relaybot.FormatMessage(content, sender.MXID, member) if err != nil { portal.log.Errorln("Failed to apply relaybot format:", err) } - evt.Content.FormattedBody = data + content.FormattedBody = data return true } -func (portal *Portal) HandleMatrixMessage(sender *User, evt *mautrix.Event) { +func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event) { if !portal.HasRelaybot() && ( (portal.IsPrivateChat() && sender.JID != portal.Key.Receiver) || portal.sendMatrixConnectionError(sender, evt.ID)) { return } + content := evt.Content.AsMessage() + if content == nil { + return + } portal.log.Debugfln("Received event %s", evt.ID) + ts := uint64(evt.Timestamp / 1000) status := waProto.WebMessageInfo_ERROR fromMe := true @@ -1234,9 +1232,9 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *mautrix.Event) { Status: &status, } ctxInfo := &waProto.ContextInfo{} - replyToID := evt.Content.GetReplyTo() + replyToID := content.GetReplyTo() if len(replyToID) > 0 { - evt.Content.RemoveReplyFallback() + content.RemoveReplyFallback() msg := portal.bridge.DB.Message.GetByMXID(replyToID) if msg != nil && msg.Content != nil { ctxInfo.StanzaId = &msg.JID @@ -1254,21 +1252,21 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *mautrix.Event) { return } } else { - relaybotFormatted = portal.addRelaybotFormat(sender, evt) + relaybotFormatted = portal.addRelaybotFormat(sender, content) sender = portal.bridge.Relaybot } } - if evt.Type == mautrix.EventSticker { - evt.Content.MsgType = mautrix.MsgImage + if evt.Type == event.EventSticker { + content.MsgType = event.MsgImage } var err error - switch evt.Content.MsgType { - case mautrix.MsgText, mautrix.MsgEmote, mautrix.MsgNotice: - text := evt.Content.Body - if evt.Content.Format == mautrix.FormatHTML { - text = portal.bridge.Formatter.ParseMatrix(evt.Content.FormattedBody) + switch content.MsgType { + case event.MsgText, event.MsgEmote, event.MsgNotice: + text := content.Body + if content.Format == event.FormatHTML { + text = portal.bridge.Formatter.ParseMatrix(content.FormattedBody) } - if evt.Content.MsgType == mautrix.MsgEmote && !relaybotFormatted { + if content.MsgType == event.MsgEmote && !relaybotFormatted { text = "/me " + text } ctxInfo.MentionedJid = mentionRegex.FindAllString(text, -1) @@ -1283,8 +1281,8 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *mautrix.Event) { } else { info.Message.Conversation = &text } - case mautrix.MsgImage: - media := portal.preprocessMatrixMedia(sender, relaybotFormatted, evt, whatsapp.MediaImage) + case event.MsgImage: + media := portal.preprocessMatrixMedia(sender, relaybotFormatted, content, evt.ID, whatsapp.MediaImage) if media == nil { return } @@ -1293,53 +1291,53 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *mautrix.Event) { JpegThumbnail: media.Thumbnail, Url: &media.URL, MediaKey: media.MediaKey, - Mimetype: &evt.Content.GetInfo().MimeType, + Mimetype: &content.GetInfo().MimeType, FileEncSha256: media.FileEncSHA256, FileSha256: media.FileSHA256, FileLength: &media.FileLength, } - case mautrix.MsgVideo: - media := portal.preprocessMatrixMedia(sender, relaybotFormatted, evt, whatsapp.MediaVideo) + case event.MsgVideo: + media := portal.preprocessMatrixMedia(sender, relaybotFormatted, content, evt.ID, whatsapp.MediaVideo) if media == nil { return } - duration := uint32(evt.Content.GetInfo().Duration) + duration := uint32(content.GetInfo().Duration) info.Message.VideoMessage = &waProto.VideoMessage{ Caption: &media.Caption, JpegThumbnail: media.Thumbnail, Url: &media.URL, MediaKey: media.MediaKey, - Mimetype: &evt.Content.GetInfo().MimeType, + Mimetype: &content.GetInfo().MimeType, Seconds: &duration, FileEncSha256: media.FileEncSHA256, FileSha256: media.FileSHA256, FileLength: &media.FileLength, } - case mautrix.MsgAudio: - media := portal.preprocessMatrixMedia(sender, relaybotFormatted, evt, whatsapp.MediaAudio) + case event.MsgAudio: + media := portal.preprocessMatrixMedia(sender, relaybotFormatted, content, evt.ID, whatsapp.MediaAudio) if media == nil { return } - duration := uint32(evt.Content.GetInfo().Duration) + duration := uint32(content.GetInfo().Duration) info.Message.AudioMessage = &waProto.AudioMessage{ Url: &media.URL, MediaKey: media.MediaKey, - Mimetype: &evt.Content.GetInfo().MimeType, + Mimetype: &content.GetInfo().MimeType, Seconds: &duration, FileEncSha256: media.FileEncSHA256, FileSha256: media.FileSHA256, FileLength: &media.FileLength, } - case mautrix.MsgFile: - media := portal.preprocessMatrixMedia(sender, relaybotFormatted, evt, whatsapp.MediaDocument) + case event.MsgFile: + media := portal.preprocessMatrixMedia(sender, relaybotFormatted, content, evt.ID, whatsapp.MediaDocument) if media == nil { return } info.Message.DocumentMessage = &waProto.DocumentMessage{ Url: &media.URL, - FileName: &evt.Content.Body, + FileName: &content.Body, MediaKey: media.MediaKey, - Mimetype: &evt.Content.GetInfo().MimeType, + Mimetype: &content.GetInfo().MimeType, FileEncSha256: media.FileEncSHA256, FileSha256: media.FileSHA256, FileLength: &media.FileLength, @@ -1353,9 +1351,9 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *mautrix.Event) { _, err = sender.Conn.Send(info) if err != nil { portal.log.Errorfln("Error handling Matrix event %s: %v", evt.ID, err) - msg := format.RenderMarkdown(fmt.Sprintf("\u26a0 Your message may not have been bridged: %v", err)) - msg.MsgType = mautrix.MsgNotice - _, err := portal.MainIntent().SendMessageEvent(portal.MXID, mautrix.EventMessage, msg) + msg := format.RenderMarkdown(fmt.Sprintf("\u26a0 Your message may not have been bridged: %v", err), false, false) + msg.MsgType = event.MsgNotice + _, err := portal.MainIntent().SendMessageEvent(portal.MXID, event.EventMessage, msg) if err != nil { portal.log.Errorln("Failed to send bridging failure message:", err) } @@ -1364,7 +1362,7 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *mautrix.Event) { } } -func (portal *Portal) HandleMatrixRedaction(sender *User, evt *mautrix.Event) { +func (portal *Portal) HandleMatrixRedaction(sender *User, evt *event.Event) { if portal.IsPrivateChat() && sender.JID != portal.Key.Receiver { return } @@ -1462,6 +1460,6 @@ func (portal *Portal) HandleMatrixLeave(sender *User) { } } -func (portal *Portal) HandleMatrixKick(sender *User, event *mautrix.Event) { +func (portal *Portal) HandleMatrixKick(sender *User, event *event.Event) { // TODO } diff --git a/provisioning.go b/provisioning.go index 0b17666..a08560d 100644 --- a/provisioning.go +++ b/provisioning.go @@ -26,8 +26,8 @@ import ( "github.com/gorilla/websocket" log "maunium.net/go/maulogger/v2" - "maunium.net/go/mautrix-whatsapp/types" whatsappExt "maunium.net/go/mautrix-whatsapp/whatsapp-ext" + "maunium.net/go/mautrix/id" ) type ProvisioningAPI struct { @@ -61,7 +61,7 @@ func (prov *ProvisioningAPI) AuthMiddleware(h http.Handler) http.Handler { return } userID := r.URL.Query().Get("user_id") - user := prov.bridge.GetUserByMXID(types.MatrixUserID(userID)) + user := prov.bridge.GetUserByMXID(id.UserID(userID)) h.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), "user", user))) }) } @@ -300,7 +300,7 @@ var upgrader = websocket.Upgrader{} func (prov *ProvisioningAPI) Login(w http.ResponseWriter, r *http.Request) { userID := r.URL.Query().Get("user_id") - user := prov.bridge.GetUserByMXID(types.MatrixUserID(userID)) + user := prov.bridge.GetUserByMXID(id.UserID(userID)) c, err := upgrader.Upgrade(w, r, nil) if err != nil { diff --git a/puppet.go b/puppet.go index 641e4de..6b56a50 100644 --- a/puppet.go +++ b/puppet.go @@ -1,5 +1,5 @@ // mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. -// Copyright (C) 2019 Tulir Asokan +// Copyright (C) 2020 Tulir Asokan // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by @@ -25,14 +25,16 @@ import ( "github.com/Rhymen/go-whatsapp" log "maunium.net/go/maulogger/v2" + "maunium.net/go/mautrix-appservice" + "maunium.net/go/mautrix/id" "maunium.net/go/mautrix-whatsapp/database" "maunium.net/go/mautrix-whatsapp/types" "maunium.net/go/mautrix-whatsapp/whatsapp-ext" ) -func (bridge *Bridge) ParsePuppetMXID(mxid types.MatrixUserID) (types.WhatsAppID, bool) { +func (bridge *Bridge) ParsePuppetMXID(mxid id.UserID) (types.WhatsAppID, bool) { userIDRegex, err := regexp.Compile(fmt.Sprintf("^@%s:%s$", bridge.Config.Bridge.FormatUsername("([0-9]+)"), bridge.Config.Homeserver.Domain)) @@ -49,7 +51,7 @@ func (bridge *Bridge) ParsePuppetMXID(mxid types.MatrixUserID) (types.WhatsAppID return jid, true } -func (bridge *Bridge) GetPuppetByMXID(mxid types.MatrixUserID) *Puppet { +func (bridge *Bridge) GetPuppetByMXID(mxid id.UserID) *Puppet { jid, ok := bridge.ParsePuppetMXID(mxid) if !ok { return nil @@ -78,7 +80,7 @@ func (bridge *Bridge) GetPuppetByJID(jid types.WhatsAppID) *Puppet { return puppet } -func (bridge *Bridge) GetPuppetByCustomMXID(mxid types.MatrixUserID) *Puppet { +func (bridge *Bridge) GetPuppetByCustomMXID(mxid id.UserID) *Puppet { bridge.puppetsLock.Lock() defer bridge.puppetsLock.Unlock() puppet, ok := bridge.puppetsByCustomMXID[mxid] @@ -129,7 +131,7 @@ func (bridge *Bridge) NewPuppet(dbPuppet *database.Puppet) *Puppet { bridge: bridge, log: bridge.Log.Sub(fmt.Sprintf("Puppet/%s", dbPuppet.JID)), - MXID: fmt.Sprintf("@%s:%s", + MXID: id.NewUserID( bridge.Config.Bridge.FormatUsername( strings.Replace( dbPuppet.JID, @@ -144,13 +146,13 @@ type Puppet struct { bridge *Bridge log log.Logger - typingIn types.MatrixRoomID + typingIn id.RoomID typingAt int64 - MXID types.MatrixUserID + MXID id.UserID customIntent *appservice.IntentAPI - customTypingIn map[string]bool + customTypingIn map[id.RoomID]bool customUser *User } @@ -192,11 +194,11 @@ func (puppet *Puppet) UpdateAvatar(source *User, avatar *whatsappExt.ProfilePicI } if len(avatar.URL) == 0 { - err := puppet.DefaultIntent().SetAvatarURL("") + err := puppet.DefaultIntent().SetAvatarURL(id.ContentURI{}) if err != nil { puppet.log.Warnln("Failed to remove avatar:", err) } - puppet.AvatarURL = "" + puppet.AvatarURL = id.ContentURI{} puppet.Avatar = avatar.Tag go puppet.updatePortalAvatar() return true diff --git a/types/types.go b/types/types.go index ab4052e..e58655a 100644 --- a/types/types.go +++ b/types/types.go @@ -21,12 +21,3 @@ type WhatsAppID = string // WhatsAppMessageID is the internal ID of a WhatsApp message. type WhatsAppMessageID = string - -// MatrixUserID is the ID of a Matrix user. -type MatrixUserID = string - -// MatrixRoomID is the internal room ID of a Matrix room. -type MatrixRoomID = string - -// MatrixEventID is the internal ID of a Matrix event. -type MatrixEventID = string diff --git a/user.go b/user.go index b55714f..857e50b 100644 --- a/user.go +++ b/user.go @@ -1,5 +1,5 @@ // mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. -// Copyright (C) 2019 Tulir Asokan +// Copyright (C) 2020 Tulir Asokan // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by @@ -32,8 +32,9 @@ import ( "github.com/Rhymen/go-whatsapp" waProto "github.com/Rhymen/go-whatsapp/binary/proto" - "maunium.net/go/mautrix" + "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/format" + "maunium.net/go/mautrix/id" "maunium.net/go/mautrix-whatsapp/database" "maunium.net/go/mautrix-whatsapp/types" @@ -65,7 +66,7 @@ type User struct { syncLock sync.Mutex } -func (bridge *Bridge) GetUserByMXID(userID types.MatrixUserID) *User { +func (bridge *Bridge) GetUserByMXID(userID id.UserID) *User { _, isPuppet := bridge.ParsePuppetMXID(userID) if isPuppet || userID == bridge.Bot.UserID { return nil @@ -104,7 +105,7 @@ func (bridge *Bridge) GetAllUsers() []*User { return output } -func (bridge *Bridge) loadDBUser(dbUser *database.User, mxid *types.MatrixUserID) *User { +func (bridge *Bridge) loadDBUser(dbUser *database.User, mxid *id.UserID) *User { if dbUser == nil { if mxid == nil { return nil @@ -160,7 +161,7 @@ func (bridge *Bridge) NewUser(dbUser *database.User) *User { return user } -func (user *User) SetManagementRoom(roomID types.MatrixRoomID) { +func (user *User) SetManagementRoom(roomID id.RoomID) { existingUser, ok := user.bridge.managementRooms[roomID] if ok { existingUser.ManagementRoom = "" @@ -194,9 +195,9 @@ func (user *User) Connect(evenIfNoSession bool) bool { conn, err := whatsapp.NewConn(timeout * time.Second) if err != nil { user.log.Errorln("Failed to connect to WhatsApp:", err) - msg := format.RenderMarkdown("\u26a0 Failed to connect to WhatsApp server. " + - "This indicates a network problem on the bridge server. See bridge logs for more info.") - _, _ = user.bridge.Bot.SendMessageEvent(user.ManagementRoom, mautrix.EventMessage, msg) + msg := format.RenderMarkdown("\u26a0 Failed to connect to WhatsApp server. "+ + "This indicates a network problem on the bridge server. See bridge logs for more info.", true, false) + _, _ = user.bridge.Bot.SendMessageEvent(user.ManagementRoom, event.EventMessage, msg) return false } user.Conn = whatsappExt.ExtendConn(conn) @@ -213,9 +214,9 @@ func (user *User) RestoreSession() bool { return true } else if err != nil { user.log.Errorln("Failed to restore session:", err) - msg := format.RenderMarkdown("\u26a0 Failed to connect to WhatsApp. Make sure WhatsApp " + - "on your phone is reachable and use `reconnect` to try connecting again.") - _, _ = user.bridge.Bot.SendMessageEvent(user.ManagementRoom, mautrix.EventMessage, msg) + msg := format.RenderMarkdown("\u26a0 Failed to connect to WhatsApp. Make sure WhatsApp "+ + "on your phone is reachable and use `reconnect` to try connecting again.", true, false) + _, _ = user.bridge.Bot.SendMessageEvent(user.ManagementRoom, event.EventMessage, msg) user.log.Debugln("Disconnecting due to failed session restore...") _, err := user.Conn.Disconnect() if err != nil { @@ -243,8 +244,8 @@ func (user *User) IsLoginInProgress() bool { return user.Conn != nil && user.Conn.IsLoginInProgress() } -func (user *User) loginQrChannel(ce *CommandEvent, qrChan <-chan string, eventIDChan chan<- string) { - var qrEventID string +func (user *User) loginQrChannel(ce *CommandEvent, qrChan <-chan string, eventIDChan chan<- id.EventID) { + var qrEventID id.EventID for code := range qrChan { if code == "stop" { return @@ -274,17 +275,17 @@ func (user *User) loginQrChannel(ce *CommandEvent, qrChan <-chan string, eventID qrEventID = sendResp.EventID eventIDChan <- qrEventID } else { - _, err = bot.SendMessageEvent(ce.RoomID, mautrix.EventMessage, &mautrix.Content{ - MsgType: mautrix.MsgImage, + _, err = bot.SendMessageEvent(ce.RoomID, event.EventMessage, &event.MessageEventContent{ + MsgType: event.MsgImage, Body: code, - URL: resp.ContentURI, - NewContent: &mautrix.Content{ - MsgType: mautrix.MsgImage, + URL: resp.ContentURI.CUString(), + NewContent: &event.MessageEventContent{ + MsgType: event.MsgImage, Body: code, - URL: resp.ContentURI, + URL: resp.ContentURI.CUString(), }, - RelatesTo: &mautrix.RelatesTo{ - Type: mautrix.RelReplace, + RelatesTo: &event.RelatesTo{ + Type: event.RelReplace, EventID: qrEventID, }, }) @@ -297,18 +298,18 @@ func (user *User) loginQrChannel(ce *CommandEvent, qrChan <-chan string, eventID func (user *User) Login(ce *CommandEvent) { qrChan := make(chan string, 3) - eventIDChan := make(chan string, 1) + eventIDChan := make(chan id.EventID, 1) go user.loginQrChannel(ce, qrChan, eventIDChan) session, err := user.Conn.LoginWithRetry(qrChan, user.bridge.Config.Bridge.LoginQRRegenCount) qrChan <- "stop" if err != nil { - var eventID string + var eventID id.EventID select { case eventID = <-eventIDChan: default: } - reply := mautrix.Content{ - MsgType: mautrix.MsgText, + reply := event.MessageEventContent{ + MsgType: event.MsgText, } if err == whatsapp.ErrAlreadyLoggedIn { reply.Body = "You're already logged in" @@ -323,12 +324,12 @@ func (user *User) Login(ce *CommandEvent) { msg := reply if eventID != "" { msg.NewContent = &reply - msg.RelatesTo = &mautrix.RelatesTo{ - Type: mautrix.RelReplace, + msg.RelatesTo = &event.RelatesTo{ + Type: event.RelReplace, EventID: eventID, } } - _, _ = ce.Bot.SendMessageEvent(ce.RoomID, mautrix.EventMessage, &msg) + _, _ = ce.Bot.SendMessageEvent(ce.RoomID, event.EventMessage, &msg) return } user.ConnectionErrors = 0 @@ -365,8 +366,11 @@ func (user *User) PostLogin() { } func (user *User) tryAutomaticDoublePuppeting() { - if len(user.bridge.Config.Bridge.LoginSharedSecret) == 0 || !strings.HasSuffix(user.MXID, user.bridge.Config.Homeserver.Domain) { - // Automatic login not enabled or user is on another homeserver + if len(user.bridge.Config.Bridge.LoginSharedSecret) == 0 { + // Automatic login not enabled + return + } else if _, homeserver, _ := user.MXID.Parse(); homeserver != user.bridge.Config.Homeserver.Domain { + // user is on another homeserver return } @@ -535,8 +539,8 @@ func (user *User) HandleError(err error) { func (user *User) tryReconnect(msg string) { if user.ConnectionErrors > user.bridge.Config.Bridge.MaxConnectionAttempts { - content := format.RenderMarkdown(fmt.Sprintf("%s. Use the `reconnect` command to reconnect.", msg)) - _, _ = user.bridge.Bot.SendMessageEvent(user.ManagementRoom, mautrix.EventMessage, content) + content := format.RenderMarkdown(fmt.Sprintf("%s. Use the `reconnect` command to reconnect.", msg), true, false) + _, _ = user.bridge.Bot.SendMessageEvent(user.ManagementRoom, event.EventMessage, content) return } if user.bridge.Config.Bridge.ReportConnectionRetry { @@ -591,8 +595,8 @@ func (user *User) tryReconnect(msg string) { "Use the `reconnect` command to try to reconnect.", msg, tries) } - content := format.RenderMarkdown(msg) - _, _ = user.bridge.Bot.SendMessageEvent(user.ManagementRoom, mautrix.EventMessage, content) + content := format.RenderMarkdown(msg, true, false) + _, _ = user.bridge.Bot.SendMessageEvent(user.ManagementRoom, event.EventMessage, content) } func (user *User) ShouldCallSynchronously() bool { @@ -766,7 +770,7 @@ func (user *User) HandleCommand(cmd whatsappExt.Command) { "Use the `reconnect` command to reconnect.", cmd.Kind) } user.cleanDisconnection = true - go user.bridge.Bot.SendMessageEvent(user.ManagementRoom, mautrix.EventMessage, format.RenderMarkdown(msg)) + go user.bridge.Bot.SendMessageEvent(user.ManagementRoom, event.EventMessage, format.RenderMarkdown(msg, true, false)) } } From edd91510f14ce696da4d379a6f532e95dde0d5fd Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 8 May 2020 23:53:30 +0300 Subject: [PATCH 02/22] Add olm to CI builds --- .editorconfig | 3 +++ .gitlab-ci.yml | 56 ++++++++++++++++++++++++++++++-------------------- Dockerfile | 8 +++++--- Dockerfile.ci | 3 ++- 4 files changed, 44 insertions(+), 26 deletions(-) diff --git a/.editorconfig b/.editorconfig index 21d312a..02798cd 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,3 +10,6 @@ insert_final_newline = true [*.{yaml,yml}] indent_style = space + +[.gitlab-ci.yml] +indent_size = 2 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 33b38da..b2080a7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,16 +3,15 @@ stages: - build docker - manifest -build: +.build: &build image: golang:1-alpine stage: build - tags: - - amd64 cache: paths: - .cache before_script: - - apk add git build-base + - echo "@edge_community http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories + - apk add git build-base olm-dev@edge_community - mkdir -p .cache - export GOPATH="$CI_PROJECT_DIR/.cache" script: @@ -22,31 +21,44 @@ build: - mautrix-whatsapp - example-config.yaml -build docker amd64: +.build-docker: &build-docker image: docker:stable stage: build docker + before_script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + script: + - docker pull $CI_REGISTRY_IMAGE:latest || true + - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-$DOCKER_ARCH . --file Dockerfile.ci + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-$DOCKER_ARCH + - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-$DOCKER_ARCH + +build amd64: + <<: *build tags: - amd64 - before_script: - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - script: - - docker pull $CI_REGISTRY_IMAGE:latest || true - - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 . --file Dockerfile.ci - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 - - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 -build docker arm64: - image: docker:stable - stage: build docker +build arm64: + <<: *build tags: - arm64 - before_script: - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - script: - - docker pull $CI_REGISTRY_IMAGE:latest || true - - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 . - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 - - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 + +build docker amd64: + <<: *build-docker + tags: + - amd64 + dependencies: + - build amd64 + variables: + DOCKER_ARCH: amd64 + +build docker arm64: + <<: *build-docker + tags: + - arm64 + dependencies: + - build arm64 + variables: + DOCKER_ARCH: arm64 manifest: stage: manifest diff --git a/Dockerfile b/Dockerfile index 869ab98..2ebffbc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ -FROM golang:1.12-alpine AS builder +FROM golang:1-alpine AS builder -RUN apk add --no-cache git ca-certificates build-base su-exec +RUN echo "@edge_community http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories +RUN apk add --no-cache git ca-certificates build-base su-exec olm-dev@edge_community WORKDIR /build COPY go.mod go.sum /build/ @@ -14,7 +15,8 @@ FROM alpine:latest ENV UID=1337 \ GID=1337 -RUN apk add --no-cache su-exec ca-certificates +RUN echo "@edge_community http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories +RUN apk add --no-cache su-exec ca-certificates olm@edge_community COPY --from=builder /usr/bin/mautrix-whatsapp /usr/bin/mautrix-whatsapp COPY --from=builder /build/example-config.yaml /opt/mautrix-whatsapp/example-config.yaml diff --git a/Dockerfile.ci b/Dockerfile.ci index f25dc14..c0147ae 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -3,7 +3,8 @@ FROM alpine:latest ENV UID=1337 \ GID=1337 -RUN apk add --no-cache su-exec ca-certificates +RUN echo "@edge_community http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories +RUN apk add --no-cache su-exec ca-certificates olm@edge_community ARG EXECUTABLE=./mautrix-whatsapp COPY $EXECUTABLE /usr/bin/mautrix-whatsapp From baae66ed0473c3a3f0cfba68c49e710759c84ab2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 9 May 2020 02:03:59 +0300 Subject: [PATCH 03/22] Add basic end-to-bridge encryption support Still missing persisting sync tokens and crypto state in DB --- commands.go | 42 ++++ config/bridge.go | 5 + crypto.go | 219 ++++++++++++++++++ database/portal.go | 14 +- database/statestore.go | 38 +++ .../2020-05-09-add-portal-encrypted-field.go | 12 + database/upgrades/upgrades.go | 2 +- example-config.yaml | 12 + main.go | 17 ++ matrix.go | 59 ++++- nocrypto.go | 26 +++ portal.go | 52 +++-- 12 files changed, 460 insertions(+), 38 deletions(-) create mode 100644 crypto.go create mode 100644 database/upgrades/2020-05-09-add-portal-encrypted-field.go create mode 100644 nocrypto.go diff --git a/commands.go b/commands.go index ec4fb9b..c1f3000 100644 --- a/commands.go +++ b/commands.go @@ -18,6 +18,7 @@ package main import ( "fmt" + "strconv" "strings" "github.com/Rhymen/go-whatsapp" @@ -118,6 +119,8 @@ func (handler *CommandHandler) CommandMux(ce *CommandEvent) { handler.CommandDeleteAllPortals(ce) case "dev-test": handler.CommandDevTest(ce) + case "set-pl": + handler.CommandSetPowerLevel(ce) case "login-matrix", "logout", "sync", "list", "open", "pm": if !ce.User.HasSession() { ce.Reply("You are not logged in. Use the `login` command to log into WhatsApp.") @@ -169,6 +172,45 @@ func (handler *CommandHandler) CommandDevTest(ce *CommandEvent) { } +func (handler *CommandHandler) CommandSetPowerLevel(ce *CommandEvent) { + portal := ce.Bridge.GetPortalByMXID(ce.RoomID) + if portal == nil { + ce.Reply("Not a portal room") + return + } + var level int + var userID id.UserID + var err error + if len(ce.Args) == 1 { + level, err = strconv.Atoi(ce.Args[0]) + if err != nil { + ce.Reply("Invalid power level \"%s\"", ce.Args[0]) + return + } + userID = ce.User.MXID + } else if len(ce.Args) == 2 { + userID = id.UserID(ce.Args[0]) + _, _, err := userID.Parse() + if err != nil { + ce.Reply("Invalid user ID \"%s\"", ce.Args[0]) + return + } + level, err = strconv.Atoi(ce.Args[1]) + if err != nil { + ce.Reply("Invalid power level \"%s\"", ce.Args[1]) + return + } + } else { + ce.Reply("**Usage:** `set-pl [user] `") + return + } + intent := portal.MainIntent() + _, err = intent.SetPowerLevel(ce.RoomID, userID, level) + if err != nil { + ce.Reply("Failed to set power levels: %v", err) + } +} + const cmdLoginHelp = `login - Authenticate this Bridge as WhatsApp Web Client` // CommandLogin handles login command diff --git a/config/bridge.go b/config/bridge.go index b7efad5..82cdd16 100644 --- a/config/bridge.go +++ b/config/bridge.go @@ -64,6 +64,11 @@ type BridgeConfig struct { CommandPrefix string `yaml:"command_prefix"` + Encryption struct { + Allow bool `yaml:"allow"` + Default bool `yaml:"default"` + } `yaml:"encryption"` + Permissions PermissionConfig `yaml:"permissions"` Relaybot RelaybotConfig `yaml:"relaybot"` diff --git a/crypto.go b/crypto.go new file mode 100644 index 0000000..17094c6 --- /dev/null +++ b/crypto.go @@ -0,0 +1,219 @@ +// mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. +// Copyright (C) 2020 Tulir Asokan +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// +build cgo + +package main + +import ( + "crypto/hmac" + "crypto/sha512" + "encoding/hex" + "time" + + "github.com/pkg/errors" + "maunium.net/go/maulogger/v2" + + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/crypto" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +var levelTrace = maulogger.Level{ + Name: "Trace", + Severity: -10, + Color: -1, +} + +type CryptoHelper struct { + bridge *Bridge + client *mautrix.Client + mach *crypto.OlmMachine + log maulogger.Logger +} + +func (bridge *Bridge) initCrypto() error { + if !bridge.Config.Bridge.Encryption.Allow { + bridge.Log.Debugln("Bridge built with end-to-bridge encryption, but disabled in config") + return nil + } else if bridge.Config.Bridge.LoginSharedSecret == "" { + bridge.Log.Warnln("End-to-bridge encryption enabled, but login_shared_secret not set") + return nil + } + bridge.Log.Debugln("Initializing end-to-bridge encryption...") + client, err := bridge.loginBot() + if err != nil { + return err + } + // TODO put this in the database + cryptoStore, err := crypto.NewGobStore("crypto.gob") + if err != nil { + return err + } + + log := bridge.Log.Sub("Crypto") + logger := &cryptoLogger{log} + stateStore := &cryptoStateStore{bridge} + helper := &CryptoHelper{ + bridge: bridge, + client: client, + log: log.Sub("Helper"), + mach: crypto.NewOlmMachine(client, logger, cryptoStore, stateStore), + } + + client.Logger = logger.int.Sub("Bot") + client.Syncer = &cryptoSyncer{helper.mach} + // TODO put this in the database too + client.Store = mautrix.NewInMemoryStore() + + err = helper.mach.Load() + if err != nil { + return err + } + + bridge.Crypto = helper + return nil +} + +func (helper *CryptoHelper) Start() { + helper.log.Debugln("Starting syncer for receiving to-device messages") + err := helper.client.Sync() + if err != nil { + helper.log.Errorln("Fatal error syncing:", err) + } +} + +func (helper *CryptoHelper) Stop() { + helper.client.StopSync() +} + +func (bridge *Bridge) loginBot() (*mautrix.Client, error) { + mac := hmac.New(sha512.New, []byte(bridge.Config.Bridge.LoginSharedSecret)) + mac.Write([]byte(bridge.AS.BotMXID())) + resp, err := bridge.AS.BotClient().Login(&mautrix.ReqLogin{ + Type: "m.login.password", + Identifier: mautrix.UserIdentifier{Type: "m.id.user", User: string(bridge.AS.BotMXID())}, + Password: hex.EncodeToString(mac.Sum(nil)), + DeviceID: "WhatsApp Bridge", + InitialDeviceDisplayName: "WhatsApp Bridge", + }) + if err != nil { + return nil, err + } + client, err := mautrix.NewClient(bridge.AS.HomeserverURL, bridge.AS.BotMXID(), resp.AccessToken) + if err != nil { + return nil, err + } + client.DeviceID = "WhatsApp Bridge" + return client, nil +} + +func (helper *CryptoHelper) Decrypt(evt *event.Event) (*event.Event, error) { + return helper.mach.DecryptMegolmEvent(evt) +} + +func (helper *CryptoHelper) Encrypt(roomID id.RoomID, evtType event.Type, content event.Content) (*event.EncryptedEventContent, error) { + encrypted, err := helper.mach.EncryptMegolmEvent(roomID, evtType, content) + if err != nil { + if err != crypto.SessionExpired && err != crypto.SessionNotShared && err != crypto.NoGroupSession { + return nil, err + } + helper.log.Debugfln("Got %v while encrypting event for %s, sharing group session and trying again...", err, roomID) + users, err := helper.bridge.StateStore.GetRoomMemberList(roomID) + if err != nil { + return nil, errors.Wrap(err, "failed to get room member list") + } + err = helper.mach.ShareGroupSession(roomID, users) + if err != nil { + return nil, errors.Wrap(err, "failed to share group session") + } + encrypted, err = helper.mach.EncryptMegolmEvent(roomID, evtType, content) + if err != nil { + return nil, errors.Wrap(err, "failed to encrypt event after re-sharing group session") + } + } + return encrypted, nil +} + +func (helper *CryptoHelper) HandleMemberEvent(evt *event.Event) { + helper.mach.HandleMemberEvent(evt) +} + +type cryptoSyncer struct { + *crypto.OlmMachine +} + +func (syncer *cryptoSyncer) ProcessResponse(resp *mautrix.RespSync, since string) error { + syncer.ProcessSyncResponse(resp, since) + return nil +} + +func (syncer *cryptoSyncer) OnFailedSync(_ *mautrix.RespSync, err error) (time.Duration, error) { + syncer.Log.Error("Error /syncing, waiting 10 seconds: %v", err) + return 10 * time.Second, nil +} + +func (syncer *cryptoSyncer) GetFilterJSON(_ id.UserID) *mautrix.Filter { + everything := []event.Type{{Type: "*"}} + return &mautrix.Filter{ + Presence: mautrix.FilterPart{NotTypes: everything}, + AccountData: mautrix.FilterPart{NotTypes: everything}, + Room: mautrix.RoomFilter{ + IncludeLeave: false, + Ephemeral: mautrix.FilterPart{NotTypes: everything}, + AccountData: mautrix.FilterPart{NotTypes: everything}, + State: mautrix.FilterPart{NotTypes: everything}, + Timeline: mautrix.FilterPart{NotTypes: everything}, + }, + } +} + +type cryptoLogger struct { + int maulogger.Logger +} + +func (c *cryptoLogger) Error(message string, args ...interface{}) { + c.int.Errorfln(message, args...) +} + +func (c *cryptoLogger) Warn(message string, args ...interface{}) { + c.int.Warnfln(message, args...) +} + +func (c *cryptoLogger) Debug(message string, args ...interface{}) { + c.int.Debugfln(message, args...) +} + +func (c *cryptoLogger) Trace(message string, args ...interface{}) { + c.int.Logfln(levelTrace, message, args...) +} + +type cryptoStateStore struct { + bridge *Bridge +} + +func (c *cryptoStateStore) IsEncrypted(id id.RoomID) bool { + portal := c.bridge.GetPortalByMXID(id) + if portal != nil { + return portal.Encrypted + } + return false +} + +func (c *cryptoStateStore) FindSharedRooms(id id.UserID) []id.RoomID { + return c.bridge.StateStore.FindSharedRooms(id) +} diff --git a/database/portal.go b/database/portal.go index 6164266..194d9f3 100644 --- a/database/portal.go +++ b/database/portal.go @@ -22,8 +22,9 @@ import ( log "maunium.net/go/maulogger/v2" - "maunium.net/go/mautrix-whatsapp/types" "maunium.net/go/mautrix/id" + + "maunium.net/go/mautrix-whatsapp/types" ) type PortalKey struct { @@ -114,11 +115,12 @@ type Portal struct { Topic string Avatar string AvatarURL id.ContentURI + Encrypted bool } func (portal *Portal) Scan(row Scannable) *Portal { var mxid, avatarURL sql.NullString - err := row.Scan(&portal.Key.JID, &portal.Key.Receiver, &mxid, &portal.Name, &portal.Topic, &portal.Avatar, &avatarURL) + err := row.Scan(&portal.Key.JID, &portal.Key.Receiver, &mxid, &portal.Name, &portal.Topic, &portal.Avatar, &avatarURL, &portal.Encrypted) if err != nil { if err != sql.ErrNoRows { portal.log.Errorln("Database scan failed:", err) @@ -138,8 +140,8 @@ func (portal *Portal) mxidPtr() *id.RoomID { } func (portal *Portal) Insert() { - _, err := portal.db.Exec("INSERT INTO portal VALUES ($1, $2, $3, $4, $5, $6, $7)", - portal.Key.JID, portal.Key.Receiver, portal.mxidPtr(), portal.Name, portal.Topic, portal.Avatar, portal.AvatarURL.String()) + _, err := portal.db.Exec("INSERT INTO portal (jid, receiver, mxid, name, topic, avatar, avatar_url, encrypted) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", + portal.Key.JID, portal.Key.Receiver, portal.mxidPtr(), portal.Name, portal.Topic, portal.Avatar, portal.AvatarURL.String(), portal.Encrypted) if err != nil { portal.log.Warnfln("Failed to insert %s: %v", portal.Key, err) } @@ -150,8 +152,8 @@ func (portal *Portal) Update() { if len(portal.MXID) > 0 { mxid = &portal.MXID } - _, err := portal.db.Exec("UPDATE portal SET mxid=$1, name=$2, topic=$3, avatar=$4, avatar_url=$5 WHERE jid=$6 AND receiver=$7", - mxid, portal.Name, portal.Topic, portal.Avatar, portal.AvatarURL.String(), portal.Key.JID, portal.Key.Receiver) + _, err := portal.db.Exec("UPDATE portal SET mxid=$1, name=$2, topic=$3, avatar=$4, avatar_url=$5, encrypted=$6 WHERE jid=$7 AND receiver=$8", + mxid, portal.Name, portal.Topic, portal.Avatar, portal.AvatarURL.String(), portal.Encrypted, portal.Key.JID, portal.Key.Receiver) if err != nil { portal.log.Warnfln("Failed to update %s: %v", portal.Key, err) } diff --git a/database/statestore.go b/database/statestore.go index ff2e41f..ac1f0f5 100644 --- a/database/statestore.go +++ b/database/statestore.go @@ -90,6 +90,24 @@ func (store *SQLStateStore) GetRoomMembers(roomID id.RoomID) map[id.UserID]*even return members } +func (store *SQLStateStore) GetRoomMemberList(roomID id.RoomID) (members []id.UserID, err error) { + var rows *sql.Rows + rows, err = store.db.Query("SELECT user_id FROM mx_user_profile WHERE room_id=$1", roomID) + if err != nil { + return + } + for rows.Next() { + var userID id.UserID + err := rows.Scan(&userID) + if err != nil { + store.log.Warnfln("Failed to scan member in %s: %v", roomID, err) + } else { + members = append(members, userID) + } + } + return +} + func (store *SQLStateStore) GetMembership(roomID id.RoomID, userID id.UserID) event.Membership { row := store.db.QueryRow("SELECT membership FROM mx_user_profile WHERE room_id=$1 AND user_id=$2", roomID, userID) membership := event.MembershipLeave @@ -118,6 +136,26 @@ func (store *SQLStateStore) TryGetMember(roomID id.RoomID, userID id.UserID) (*e return &member, err == nil } +func (store *SQLStateStore) FindSharedRooms(userID id.UserID) (rooms []id.RoomID) { + rows, err := store.db.Query(` + SELECT room_id FROM mx_user_profile WHERE user_id=$2 AND portal.encrypted=true + LEFT JOIN portal WHEN portal.mxid=mx_user_profile.room_id`, userID) + if err != nil { + store.log.Warnfln("Failed to query shared rooms with %s: %v", userID, err) + return + } + for rows.Next() { + var roomID id.RoomID + err := rows.Scan(&roomID) + if err != nil { + store.log.Warnfln("Failed to scan room ID: %v", err) + } else { + rooms = append(rooms, roomID) + } + } + return +} + func (store *SQLStateStore) IsInRoom(roomID id.RoomID, userID id.UserID) bool { return store.IsMembership(roomID, userID, "join") } diff --git a/database/upgrades/2020-05-09-add-portal-encrypted-field.go b/database/upgrades/2020-05-09-add-portal-encrypted-field.go new file mode 100644 index 0000000..ef0f963 --- /dev/null +++ b/database/upgrades/2020-05-09-add-portal-encrypted-field.go @@ -0,0 +1,12 @@ +package upgrades + +import ( + "database/sql" +) + +func init() { + upgrades[12] = upgrade{"Add encryption status to portal table", func(tx *sql.Tx, ctx context) error { + _, err := tx.Exec(`ALTER TABLE portal ADD COLUMN encrypted BOOLEAN NOT NULL DEFAULT false`) + return err + }} +} diff --git a/database/upgrades/upgrades.go b/database/upgrades/upgrades.go index dfaec11..3126cc7 100644 --- a/database/upgrades/upgrades.go +++ b/database/upgrades/upgrades.go @@ -28,7 +28,7 @@ type upgrade struct { fn upgradeFunc } -const NumberOfUpgrades = 12 +const NumberOfUpgrades = 13 var upgrades [NumberOfUpgrades]upgrade diff --git a/example-config.yaml b/example-config.yaml index 061f433..6b66025 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -138,6 +138,18 @@ bridge: # The prefix for commands. Only required in non-management rooms. command_prefix: "!wa" + # End-to-bridge encryption support options. This requires login_shared_secret to be configured + # in order to get a device for the bridge bot. + # + # Additionally, https://github.com/matrix-org/synapse/pull/5758 is required if using a normal + # application service. + encryption: + # Allow encryption, work in group chat rooms with e2ee enabled + allow: false + # Default to encryption, force-enable encryption in all portals the bridge creates + # This will cause the bridge bot to be in private chats for the encryption to work properly. + default: false + # Permissions for using the bridge. # Permitted values: # relaybot - Talk through the relaybot (if enabled), no access otherwise diff --git a/main.go b/main.go index cb9667c..5da6685 100644 --- a/main.go +++ b/main.go @@ -26,6 +26,7 @@ import ( flag "maunium.net/go/mauflag" log "maunium.net/go/maulogger/v2" + "maunium.net/go/mautrix/event" "maunium.net/go/mautrix" "maunium.net/go/mautrix-appservice" @@ -106,6 +107,7 @@ type Bridge struct { Bot *appservice.IntentAPI Formatter *Formatter Relaybot *User + Crypto Crypto usersByMXID map[id.UserID]*User usersByJID map[types.WhatsAppID]*User @@ -120,6 +122,14 @@ type Bridge struct { puppetsLock sync.Mutex } +type Crypto interface { + HandleMemberEvent(*event.Event) + Decrypt(*event.Event) (*event.Event, error) + Encrypt(id.RoomID, event.Type, event.Content) (*event.EncryptedEventContent, error) + Start() + Stop() +} + func NewBridge() *Bridge { bridge := &Bridge{ usersByMXID: make(map[id.UserID]*User), @@ -215,6 +225,11 @@ func (bridge *Bridge) Init() { bridge.Log.Debugln("Initializing Matrix event handler") bridge.MatrixHandler = NewMatrixHandler(bridge) bridge.Formatter = NewFormatter(bridge) + err = bridge.initCrypto() + if err != nil { + bridge.Log.Fatalln("Error initializing end-to-bridge encryption:", err) + os.Exit(19) + } } func (bridge *Bridge) Start() { @@ -235,6 +250,7 @@ func (bridge *Bridge) Start() { bridge.Log.Debugln("Starting event processor") go bridge.EventProcessor.Start() go bridge.UpdateBotProfile() + go bridge.Crypto.Start() go bridge.StartUsers() } @@ -299,6 +315,7 @@ func (bridge *Bridge) StartUsers() { } func (bridge *Bridge) Stop() { + bridge.Crypto.Stop() bridge.AS.Stop() bridge.EventProcessor.Stop() for _, user := range bridge.usersByJID { diff --git a/matrix.go b/matrix.go index 3f30be7..12e1114 100644 --- a/matrix.go +++ b/matrix.go @@ -21,7 +21,6 @@ import ( "strings" "maunium.net/go/maulogger/v2" - "maunium.net/go/mautrix-appservice" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/format" @@ -43,15 +42,30 @@ func NewMatrixHandler(bridge *Bridge) *MatrixHandler { cmd: NewCommandHandler(bridge), } bridge.EventProcessor.On(event.EventMessage, handler.HandleMessage) + bridge.EventProcessor.On(event.EventEncrypted, handler.HandleEncrypted) bridge.EventProcessor.On(event.EventSticker, handler.HandleMessage) bridge.EventProcessor.On(event.EventRedaction, handler.HandleRedaction) bridge.EventProcessor.On(event.StateMember, handler.HandleMembership) bridge.EventProcessor.On(event.StateRoomName, handler.HandleRoomMetadata) bridge.EventProcessor.On(event.StateRoomAvatar, handler.HandleRoomMetadata) bridge.EventProcessor.On(event.StateTopic, handler.HandleRoomMetadata) + bridge.EventProcessor.On(event.StateEncryption, handler.HandleEncryption) return handler } +func (mx *MatrixHandler) HandleEncryption(evt *event.Event) { + if evt.Content.AsEncryption().Algorithm != id.AlgorithmMegolmV1 { + return + } + portal := mx.bridge.GetPortalByMXID(evt.RoomID) + mx.log.Debugln(portal) + if portal != nil && !portal.Encrypted { + mx.log.Debugfln("%s enabled encryption in %s", evt.Sender, evt.RoomID) + portal.Encrypted = true + portal.Update() + } +} + func (mx *MatrixHandler) HandleBotInvite(evt *event.Event) { intent := mx.as.BotIntent() @@ -115,6 +129,10 @@ func (mx *MatrixHandler) HandleBotInvite(evt *event.Event) { } func (mx *MatrixHandler) HandleMembership(evt *event.Event) { + if mx.bridge.Crypto != nil { + mx.bridge.Crypto.HandleMemberEvent(evt) + } + content := evt.Content.AsMember() if content.Membership == event.MembershipInvite && id.UserID(evt.GetStateKey()) == mx.as.BotMXID() { mx.HandleBotInvite(evt) @@ -125,7 +143,7 @@ func (mx *MatrixHandler) HandleMembership(evt *event.Event) { return } - user := mx.bridge.GetUserByMXID(id.UserID(evt.Sender)) + user := mx.bridge.GetUserByMXID(evt.Sender) if user == nil || !user.Whitelisted || !user.IsConnected() { return } @@ -148,7 +166,7 @@ func (mx *MatrixHandler) HandleMembership(evt *event.Event) { } func (mx *MatrixHandler) HandleRoomMetadata(evt *event.Event) { - user := mx.bridge.GetUserByMXID(id.UserID(evt.Sender)) + user := mx.bridge.GetUserByMXID(evt.Sender) if user == nil || !user.Whitelisted || !user.IsConnected() { return } @@ -176,21 +194,40 @@ func (mx *MatrixHandler) HandleRoomMetadata(evt *event.Event) { } } -func (mx *MatrixHandler) HandleMessage(evt *event.Event) { +func (mx *MatrixHandler) shouldIgnoreEvent(evt *event.Event) bool { if _, isPuppet := mx.bridge.ParsePuppetMXID(evt.Sender); evt.Sender == mx.bridge.Bot.UserID || isPuppet { - return + return true } isCustomPuppet, ok := evt.Content.Raw["net.maunium.whatsapp.puppet"].(bool) if ok && isCustomPuppet && mx.bridge.GetPuppetByCustomMXID(evt.Sender) != nil { + return true + } + user := mx.bridge.GetUserByMXID(evt.Sender) + if !user.RelaybotWhitelisted { + return true + } + return false +} + +func (mx *MatrixHandler) HandleEncrypted(evt *event.Event) { + if mx.shouldIgnoreEvent(evt) || mx.bridge.Crypto == nil { + return + } + + decrypted, err := mx.bridge.Crypto.Decrypt(evt) + if err != nil { + mx.log.Warnln("Failed to decrypt %s: %v", evt.ID, err) + return + } + mx.bridge.EventProcessor.Dispatch(decrypted) +} + +func (mx *MatrixHandler) HandleMessage(evt *event.Event) { + if mx.shouldIgnoreEvent(evt) { return } user := mx.bridge.GetUserByMXID(evt.Sender) - - if !user.RelaybotWhitelisted { - return - } - content := evt.Content.AsMessage() if user.Whitelisted && content.MsgType == event.MsgText { commandPrefix := mx.bridge.Config.Bridge.CommandPrefix @@ -215,7 +252,7 @@ func (mx *MatrixHandler) HandleRedaction(evt *event.Event) { return } - user := mx.bridge.GetUserByMXID(id.UserID(evt.Sender)) + user := mx.bridge.GetUserByMXID(evt.Sender) if !user.Whitelisted { return diff --git a/nocrypto.go b/nocrypto.go new file mode 100644 index 0000000..0479daf --- /dev/null +++ b/nocrypto.go @@ -0,0 +1,26 @@ +// mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. +// Copyright (C) 2020 Tulir Asokan +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// +build !cgo + +package main + +func (bridge *Bridge) initCrypto() error { + if !bridge.Config.Bridge.Encryption.Allow { + bridge.Log.Warnln("Bridge built without end-to-bridge encryption, but encryption is enabled in config") + } + bridge.Log.Debugln("Bridge built without end-to-bridge encryption") +} diff --git a/portal.go b/portal.go index 1fb6a7a..87723fc 100644 --- a/portal.go +++ b/portal.go @@ -35,6 +35,7 @@ import ( "time" "github.com/chai2010/webp" + "github.com/pkg/errors" log "maunium.net/go/maulogger/v2" "github.com/Rhymen/go-whatsapp" @@ -908,6 +909,32 @@ func (portal *Portal) HandleFakeMessage(source *User, message FakeMessage) { portal.recentlyHandled[index] = message.ID } +func (portal *Portal) sendMainIntentMessage(content interface{}) (*mautrix.RespSendEvent, error) { + return portal.sendMessage(portal.MainIntent(), event.EventMessage, content, 0) +} + +func (portal *Portal) sendMessage(intent *appservice.IntentAPI, eventType event.Type, content interface{}, timestamp int64) (*mautrix.RespSendEvent, error) { + wrappedContent := event.Content{Parsed: content} + if timestamp != 0 && intent.IsCustomPuppet { + wrappedContent.Raw = map[string]interface{}{ + "net.maunium.whatsapp.puppet": intent.IsCustomPuppet, + } + } + if portal.Encrypted && portal.bridge.Crypto != nil { + encrypted, err := portal.bridge.Crypto.Encrypt(portal.MXID, eventType, wrappedContent) + if err != nil { + return nil, errors.Wrap(err, "failed to encrypt event") + } + eventType = event.EventEncrypted + wrappedContent.Parsed = encrypted + } + if timestamp == 0 { + return intent.SendMessageEvent(portal.MXID, eventType, &wrappedContent) + } else { + return intent.SendMassagedMessageEvent(portal.MXID, eventType, &wrappedContent, timestamp) + } +} + func (portal *Portal) HandleTextMessage(source *User, message whatsapp.TextMessage) { if !portal.startHandling(message.Info) { return @@ -927,12 +954,7 @@ func (portal *Portal) HandleTextMessage(source *User, message whatsapp.TextMessa portal.SetReply(content, message.ContextInfo) _, _ = intent.UserTyping(portal.MXID, false, 0) - resp, err := intent.SendMassagedMessageEvent(portal.MXID, event.EventMessage, &event.Content{ - Parsed: content, - Raw: map[string]interface{}{ - "net.maunium.whatsapp.puppet": intent.IsCustomPuppet, - }, - }, int64(message.Info.Timestamp*1000)) + resp, err := portal.sendMessage(intent, event.EventMessage, content, int64(message.Info.Timestamp*1000)) if err != nil { portal.log.Errorfln("Failed to handle message %s: %v", message.Info.Id, err) return @@ -1042,12 +1064,7 @@ func (portal *Portal) HandleMediaMessage(source *User, download func() ([]byte, if sendAsSticker { eventType = event.EventSticker } - resp, err := intent.SendMassagedMessageEvent(portal.MXID, eventType, &event.Content{ - Parsed: content, - Raw: map[string]interface{}{ - "net.maunium.whatsapp.puppet": intent.IsCustomPuppet, - }, - }, ts) + resp, err := portal.sendMessage(intent, eventType, content, ts) if err != nil { portal.log.Errorfln("Failed to handle message %s: %v", info.Id, err) return @@ -1061,12 +1078,7 @@ func (portal *Portal) HandleMediaMessage(source *User, download func() ([]byte, portal.bridge.Formatter.ParseWhatsApp(captionContent) - _, err := intent.SendMassagedMessageEvent(portal.MXID, event.EventMessage, &event.Content{ - Parsed: content, - Raw: map[string]interface{}{ - "net.maunium.whatsapp.puppet": intent.IsCustomPuppet, - }, - }, ts) + _, err := portal.sendMessage(intent, event.EventMessage, content, ts) if err != nil { portal.log.Warnfln("Failed to handle caption of message %s: %v", info.Id, err) } @@ -1178,7 +1190,7 @@ func (portal *Portal) sendMatrixConnectionError(sender *User, eventID id.EventID } msg := format.RenderMarkdown("\u26a0 You are not connected to WhatsApp, so your message was not bridged. " + reconnect, true, false) msg.MsgType = event.MsgNotice - _, err := portal.MainIntent().SendMessageEvent(portal.MXID, event.EventMessage, msg) + _, err := portal.sendMainIntentMessage(msg) if err != nil { portal.log.Errorln("Failed to send bridging failure message:", err) } @@ -1353,7 +1365,7 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event) { portal.log.Errorfln("Error handling Matrix event %s: %v", evt.ID, err) msg := format.RenderMarkdown(fmt.Sprintf("\u26a0 Your message may not have been bridged: %v", err), false, false) msg.MsgType = event.MsgNotice - _, err := portal.MainIntent().SendMessageEvent(portal.MXID, event.EventMessage, msg) + _, err := portal.sendMainIntentMessage(msg) if err != nil { portal.log.Errorln("Failed to send bridging failure message:", err) } From f89fcf72120caba68207cfbeb6be11eb2070f6cb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 9 May 2020 02:08:23 +0300 Subject: [PATCH 04/22] Make no-cgo build work without source changes --- go.mod | 1 + go.sum | 2 ++ nocrypto.go => no-cgo.go | 12 ++++++++++++ portal.go | 6 ++---- webp.go | 14 ++++++++++++++ 5 files changed, 31 insertions(+), 4 deletions(-) rename nocrypto.go => no-cgo.go (87%) create mode 100644 webp.go diff --git a/go.mod b/go.mod index 2b6cc1c..b360bb3 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/skip2/go-qrcode v0.0.0-20191027152451-9434209cb086 + golang.org/x/image v0.0.0-20200430140353-33d19683fad8 gopkg.in/yaml.v2 v2.2.8 maunium.net/go/mauflag v1.0.0 maunium.net/go/maulogger/v2 v2.1.1 diff --git a/go.sum b/go.sum index 3d63304..dd3f71e 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,8 @@ github.com/tulir/go-whatsapp v0.2.6 h1:d58cqz/iqcCDeT+uFjLso8oSgMTYqoxGhGhGOyyHB github.com/tulir/go-whatsapp v0.2.6/go.mod h1:gyw9zGup1/Y3ZQUueZaqz3iR/WX9a2Lth4aqEbXjkok= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8 h1:6WW6V3x1P/jokJBpRQYUJnMHRP6isStQwCozxnU7XQw= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/nocrypto.go b/no-cgo.go similarity index 87% rename from nocrypto.go rename to no-cgo.go index 0479daf..961bf5c 100644 --- a/nocrypto.go +++ b/no-cgo.go @@ -18,9 +18,21 @@ package main +import ( + "image" + "io" + + "golang.org/x/image/webp" +) + func (bridge *Bridge) initCrypto() error { if !bridge.Config.Bridge.Encryption.Allow { bridge.Log.Warnln("Bridge built without end-to-bridge encryption, but encryption is enabled in config") } bridge.Log.Debugln("Bridge built without end-to-bridge encryption") + return nil +} + +func decodeWebp(r io.Reader) (image.Image, error) { + return webp.Decode(r) } diff --git a/portal.go b/portal.go index 87723fc..9c52fd9 100644 --- a/portal.go +++ b/portal.go @@ -34,7 +34,6 @@ import ( "sync" "time" - "github.com/chai2010/webp" "github.com/pkg/errors" log "maunium.net/go/maulogger/v2" @@ -989,7 +988,7 @@ func (portal *Portal) HandleMediaMessage(source *User, download func() ([]byte, // synapse doesn't handle webp well, so we convert it. This can be dropped once https://github.com/matrix-org/synapse/issues/4382 is fixed if mimeType == "image/webp" { - img, err := webp.Decode(bytes.NewReader(data)) + img, err := decodeWebp(bytes.NewReader(data)) if err != nil { portal.log.Errorfln("Failed to decode media for %s: %v", err) return @@ -1188,7 +1187,7 @@ func (portal *Portal) sendMatrixConnectionError(sender *User, eventID id.EventID if sender.IsLoginInProgress() { reconnect = "You have a login attempt in progress, please wait." } - msg := format.RenderMarkdown("\u26a0 You are not connected to WhatsApp, so your message was not bridged. " + reconnect, true, false) + msg := format.RenderMarkdown("\u26a0 You are not connected to WhatsApp, so your message was not bridged. "+reconnect, true, false) msg.MsgType = event.MsgNotice _, err := portal.sendMainIntentMessage(msg) if err != nil { @@ -1229,7 +1228,6 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event) { } portal.log.Debugfln("Received event %s", evt.ID) - ts := uint64(evt.Timestamp / 1000) status := waProto.WebMessageInfo_ERROR fromMe := true diff --git a/webp.go b/webp.go new file mode 100644 index 0000000..79b5be2 --- /dev/null +++ b/webp.go @@ -0,0 +1,14 @@ +// +build cgo + +package main + +import ( + "image" + "io" + + "github.com/chai2010/webp" +) + +func decodeWebp(r io.Reader) (image.Image, error) { + return webp.Decode(r) +} From 0a77e25cda1b0d8a3a079f3840e8734dd4c01585 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 9 May 2020 02:11:16 +0300 Subject: [PATCH 05/22] Add static build to CI --- .gitlab-ci.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b2080a7..58d93eb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,7 @@ stages: - .cache before_script: - echo "@edge_community http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories - - apk add git build-base olm-dev@edge_community + - apk add build-base olm-dev@edge_community - mkdir -p .cache - export GOPATH="$CI_PROJECT_DIR/.cache" script: @@ -32,6 +32,24 @@ stages: - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-$DOCKER_ARCH - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-$DOCKER_ARCH +build static amd64: + image: golang:1-alpine + stage: build + tags: + - amd64 + cache: + paths: + - .cache + before_script: + - mkdir -p .cache + - export GOPATH="$CI_PROJECT_DIR/.cache" + script: + - CGO_ENABLED=0 go build -o mautrix-whatsapp + artifacts: + paths: + - mautrix-whatsapp + - example-config.yaml + build amd64: <<: *build tags: From e7458139c48c2bc1c617d49c572ca6b08c355fc8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 9 May 2020 02:14:12 +0300 Subject: [PATCH 06/22] Update deps --- go.mod | 4 ++-- go.sum | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index b360bb3..d324625 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,8 @@ require ( gopkg.in/yaml.v2 v2.2.8 maunium.net/go/mauflag v1.0.0 maunium.net/go/maulogger/v2 v2.1.1 - maunium.net/go/mautrix v0.3.6 - maunium.net/go/mautrix-appservice v0.2.0 + maunium.net/go/mautrix v0.3.7 + maunium.net/go/mautrix-appservice v0.2.1 ) replace github.com/Rhymen/go-whatsapp => github.com/tulir/go-whatsapp v0.2.6 diff --git a/go.sum b/go.sum index dd3f71e..956cfc1 100644 --- a/go.sum +++ b/go.sum @@ -82,7 +82,11 @@ maunium.net/go/mautrix v0.1.0-beta.2 h1:RxYTqTzW6iXu83gf8ucqGwYx8JLa+a17LWjiPkVV maunium.net/go/mautrix v0.1.0-beta.2/go.mod h1:YFMU9DBeXH7cqx7sJLg0DkVxwNPbih8QbpUTYf/IjMM= maunium.net/go/mautrix v0.3.6 h1:bXUo8WFdv7sUpvr7jgJ6TVMEQgVHtw1z1T3eUcLpPCA= maunium.net/go/mautrix v0.3.6/go.mod h1:SkGZzch8CvU2qKtNpYxtzZ0sQxfVEJ3IsVVLSUBUx9Y= +maunium.net/go/mautrix v0.3.7 h1:N0czrZeAwjvBrw2a/B2G6U3EwIYaWpt7OuSslGp8DRc= +maunium.net/go/mautrix v0.3.7/go.mod h1:SkGZzch8CvU2qKtNpYxtzZ0sQxfVEJ3IsVVLSUBUx9Y= maunium.net/go/mautrix-appservice v0.1.0-alpha.6 h1:dNE+RykOC0UhSyRNbMHXEk3BzSOp3dj8aQwKuNMELWM= maunium.net/go/mautrix-appservice v0.1.0-alpha.6/go.mod h1:Dfiwiuicvn8s2VKrBDrZ9eCjlKUMbuCi91TE6xeEHRM= maunium.net/go/mautrix-appservice v0.2.0 h1:HmEpBSdGK7/8/xqOhxNP6viSQPkgjFVTfMI33moz51A= maunium.net/go/mautrix-appservice v0.2.0/go.mod h1:55u7GKZBfxIs6tfAQTpvLvKLUjaJvll5HLcmx5Set1A= +maunium.net/go/mautrix-appservice v0.2.1 h1:G3gSem2CktOhjelNCk6HbzR2VsD0YzRX4luoen7atfE= +maunium.net/go/mautrix-appservice v0.2.1/go.mod h1:gLatDUjKqfv8s7WyEmT+k1fd1FvwVBts2wwhzShFNqQ= From 6e50a7c3803b074dc5062257ec478ca1683b25b2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 9 May 2020 14:31:06 +0300 Subject: [PATCH 07/22] Switch appservice module location --- commands.go | 2 +- config/config.go | 2 +- config/registration.go | 2 +- custompuppet.go | 6 +++--- database/statestore.go | 2 +- go.mod | 3 +-- go.sum | 8 ++------ main.go | 4 ++-- matrix.go | 3 ++- portal.go | 2 +- puppet.go | 2 +- 11 files changed, 16 insertions(+), 20 deletions(-) diff --git a/commands.go b/commands.go index c1f3000..79d6b7b 100644 --- a/commands.go +++ b/commands.go @@ -26,7 +26,7 @@ import ( "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix" - "maunium.net/go/mautrix-appservice" + "maunium.net/go/mautrix/appservice" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/format" "maunium.net/go/mautrix/id" diff --git a/config/config.go b/config/config.go index e8a4f0a..d076e01 100644 --- a/config/config.go +++ b/config/config.go @@ -21,7 +21,7 @@ import ( "gopkg.in/yaml.v2" - "maunium.net/go/mautrix-appservice" + "maunium.net/go/mautrix/appservice" ) type Config struct { diff --git a/config/registration.go b/config/registration.go index fd79f17..0524200 100644 --- a/config/registration.go +++ b/config/registration.go @@ -20,7 +20,7 @@ import ( "fmt" "regexp" - "maunium.net/go/mautrix-appservice" + "maunium.net/go/mautrix/appservice" ) func (config *Config) NewRegistration() (*appservice.Registration, error) { diff --git a/custompuppet.go b/custompuppet.go index 1f2c520..8a50383 100644 --- a/custompuppet.go +++ b/custompuppet.go @@ -27,7 +27,7 @@ import ( "github.com/Rhymen/go-whatsapp" "maunium.net/go/mautrix" - appservice "maunium.net/go/mautrix-appservice" + "maunium.net/go/mautrix/appservice" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" ) @@ -270,6 +270,6 @@ func (puppet *Puppet) GetFilterJSON(_ id.UserID) *mautrix.Filter { func (puppet *Puppet) SaveFilterID(_ id.UserID, _ string) {} func (puppet *Puppet) SaveNextBatch(_ id.UserID, nbt string) { puppet.NextBatch = nbt; puppet.Update() } func (puppet *Puppet) SaveRoom(room *mautrix.Room) {} -func (puppet *Puppet) LoadFilterID(_ id.UserID) string { return "" } -func (puppet *Puppet) LoadNextBatch(_ id.UserID) string { return puppet.NextBatch } +func (puppet *Puppet) LoadFilterID(_ id.UserID) string { return "" } +func (puppet *Puppet) LoadNextBatch(_ id.UserID) string { return puppet.NextBatch } func (puppet *Puppet) LoadRoom(roomID id.RoomID) *mautrix.Room { return nil } diff --git a/database/statestore.go b/database/statestore.go index ac1f0f5..178d4c1 100644 --- a/database/statestore.go +++ b/database/statestore.go @@ -24,7 +24,7 @@ import ( log "maunium.net/go/maulogger/v2" - "maunium.net/go/mautrix-appservice" + "maunium.net/go/mautrix/appservice" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" ) diff --git a/go.mod b/go.mod index d324625..4534d58 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,7 @@ require ( gopkg.in/yaml.v2 v2.2.8 maunium.net/go/mauflag v1.0.0 maunium.net/go/maulogger/v2 v2.1.1 - maunium.net/go/mautrix v0.3.7 - maunium.net/go/mautrix-appservice v0.2.1 + maunium.net/go/mautrix v0.4.0 ) replace github.com/Rhymen/go-whatsapp => github.com/tulir/go-whatsapp v0.2.6 diff --git a/go.sum b/go.sum index 956cfc1..cba311b 100644 --- a/go.sum +++ b/go.sum @@ -84,9 +84,5 @@ maunium.net/go/mautrix v0.3.6 h1:bXUo8WFdv7sUpvr7jgJ6TVMEQgVHtw1z1T3eUcLpPCA= maunium.net/go/mautrix v0.3.6/go.mod h1:SkGZzch8CvU2qKtNpYxtzZ0sQxfVEJ3IsVVLSUBUx9Y= maunium.net/go/mautrix v0.3.7 h1:N0czrZeAwjvBrw2a/B2G6U3EwIYaWpt7OuSslGp8DRc= maunium.net/go/mautrix v0.3.7/go.mod h1:SkGZzch8CvU2qKtNpYxtzZ0sQxfVEJ3IsVVLSUBUx9Y= -maunium.net/go/mautrix-appservice v0.1.0-alpha.6 h1:dNE+RykOC0UhSyRNbMHXEk3BzSOp3dj8aQwKuNMELWM= -maunium.net/go/mautrix-appservice v0.1.0-alpha.6/go.mod h1:Dfiwiuicvn8s2VKrBDrZ9eCjlKUMbuCi91TE6xeEHRM= -maunium.net/go/mautrix-appservice v0.2.0 h1:HmEpBSdGK7/8/xqOhxNP6viSQPkgjFVTfMI33moz51A= -maunium.net/go/mautrix-appservice v0.2.0/go.mod h1:55u7GKZBfxIs6tfAQTpvLvKLUjaJvll5HLcmx5Set1A= -maunium.net/go/mautrix-appservice v0.2.1 h1:G3gSem2CktOhjelNCk6HbzR2VsD0YzRX4luoen7atfE= -maunium.net/go/mautrix-appservice v0.2.1/go.mod h1:gLatDUjKqfv8s7WyEmT+k1fd1FvwVBts2wwhzShFNqQ= +maunium.net/go/mautrix v0.4.0 h1:IYfmxCoxR/6UMi92IncsSZeKQbZm8Xa35XIRX814KJ4= +maunium.net/go/mautrix v0.4.0/go.mod h1:8Y+NqmROJyWYvvP4yPfX9tLM59VCfgE/kcQ0SeX68ho= diff --git a/main.go b/main.go index 5da6685..ddcd4b3 100644 --- a/main.go +++ b/main.go @@ -26,10 +26,10 @@ import ( flag "maunium.net/go/mauflag" log "maunium.net/go/maulogger/v2" - "maunium.net/go/mautrix/event" "maunium.net/go/mautrix" - "maunium.net/go/mautrix-appservice" + "maunium.net/go/mautrix/appservice" + "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" "maunium.net/go/mautrix-whatsapp/config" diff --git a/matrix.go b/matrix.go index 12e1114..d064f49 100644 --- a/matrix.go +++ b/matrix.go @@ -21,7 +21,8 @@ import ( "strings" "maunium.net/go/maulogger/v2" - "maunium.net/go/mautrix-appservice" + + "maunium.net/go/mautrix/appservice" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/format" "maunium.net/go/mautrix/id" diff --git a/portal.go b/portal.go index 9c52fd9..ffac2c8 100644 --- a/portal.go +++ b/portal.go @@ -41,7 +41,7 @@ import ( waProto "github.com/Rhymen/go-whatsapp/binary/proto" "maunium.net/go/mautrix" - "maunium.net/go/mautrix-appservice" + "maunium.net/go/mautrix/appservice" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/format" "maunium.net/go/mautrix/id" diff --git a/puppet.go b/puppet.go index 6b56a50..4831c3d 100644 --- a/puppet.go +++ b/puppet.go @@ -26,7 +26,7 @@ import ( log "maunium.net/go/maulogger/v2" - "maunium.net/go/mautrix-appservice" + "maunium.net/go/mautrix/appservice" "maunium.net/go/mautrix/id" "maunium.net/go/mautrix-whatsapp/database" From dfc5722a8036954482d3ac0afa7c51e62d63995e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 9 May 2020 20:07:21 +0300 Subject: [PATCH 08/22] Move crypto store to main database --- crypto.go | 126 +++--- database/cryptostore.go | 393 +++++++++++++++++++ database/statestore.go | 28 +- database/upgrades/2020-05-09-crypto-store.go | 74 ++++ database/upgrades/upgrades.go | 2 +- go.mod | 2 +- go.sum | 2 + main.go | 14 +- 8 files changed, 561 insertions(+), 80 deletions(-) create mode 100644 database/cryptostore.go create mode 100644 database/upgrades/2020-05-09-crypto-store.go diff --git a/crypto.go b/crypto.go index 17094c6..286b600 100644 --- a/crypto.go +++ b/crypto.go @@ -28,6 +28,7 @@ import ( "maunium.net/go/maulogger/v2" "maunium.net/go/mautrix" + "maunium.net/go/mautrix-whatsapp/database" "maunium.net/go/mautrix/crypto" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" @@ -40,13 +41,15 @@ var levelTrace = maulogger.Level{ } type CryptoHelper struct { - bridge *Bridge - client *mautrix.Client - mach *crypto.OlmMachine - log maulogger.Logger + bridge *Bridge + client *mautrix.Client + mach *crypto.OlmMachine + store *database.SQLCryptoStore + log maulogger.Logger + baseLog maulogger.Logger } -func (bridge *Bridge) initCrypto() error { +func NewCryptoHelper(bridge *Bridge) *CryptoHelper { if !bridge.Config.Bridge.Encryption.Allow { bridge.Log.Debugln("Bridge built with end-to-bridge encryption, but disabled in config") return nil @@ -54,39 +57,60 @@ func (bridge *Bridge) initCrypto() error { bridge.Log.Warnln("End-to-bridge encryption enabled, but login_shared_secret not set") return nil } - bridge.Log.Debugln("Initializing end-to-bridge encryption...") - client, err := bridge.loginBot() - if err != nil { - return err + baseLog := bridge.Log.Sub("Crypto") + return &CryptoHelper{ + bridge: bridge, + log: baseLog.Sub("Helper"), + baseLog: baseLog, } - // TODO put this in the database - cryptoStore, err := crypto.NewGobStore("crypto.gob") +} + +func (helper *CryptoHelper) Init() error { + helper.log.Debugln("Initializing end-to-bridge encryption...") + var err error + helper.client, err = helper.loginBot() if err != nil { return err } - log := bridge.Log.Sub("Crypto") - logger := &cryptoLogger{log} - stateStore := &cryptoStateStore{bridge} - helper := &CryptoHelper{ - bridge: bridge, - client: client, - log: log.Sub("Helper"), - mach: crypto.NewOlmMachine(client, logger, cryptoStore, stateStore), + helper.log.Debugln("Logged in as bridge bot with device ID", helper.client.DeviceID) + logger := &cryptoLogger{helper.baseLog} + stateStore := &cryptoStateStore{helper.bridge} + helper.store = database.NewSQLCryptoStore(helper.bridge.DB, helper.client.DeviceID) + helper.store.UserID = helper.client.UserID + helper.store.GhostIDFormat = helper.bridge.Config.Bridge.FormatUsername("%") + helper.mach = crypto.NewOlmMachine(helper.client, logger, helper.store, stateStore) + + helper.client.Logger = logger.int.Sub("Bot") + helper.client.Syncer = &cryptoSyncer{helper.mach} + helper.client.Store = &cryptoClientStore{helper.store} + + return helper.mach.Load() +} + +func (helper *CryptoHelper) loginBot() (*mautrix.Client, error) { + deviceID := helper.bridge.DB.FindDeviceID() + if len(deviceID) > 0 { + helper.log.Debugln("Found existing device ID for bot in database:", deviceID) } - - client.Logger = logger.int.Sub("Bot") - client.Syncer = &cryptoSyncer{helper.mach} - // TODO put this in the database too - client.Store = mautrix.NewInMemoryStore() - - err = helper.mach.Load() + mac := hmac.New(sha512.New, []byte(helper.bridge.Config.Bridge.LoginSharedSecret)) + mac.Write([]byte(helper.bridge.AS.BotMXID())) + resp, err := helper.bridge.AS.BotClient().Login(&mautrix.ReqLogin{ + Type: "m.login.password", + Identifier: mautrix.UserIdentifier{Type: "m.id.user", User: string(helper.bridge.AS.BotMXID())}, + Password: hex.EncodeToString(mac.Sum(nil)), + DeviceID: deviceID, + InitialDeviceDisplayName: "WhatsApp Bridge", + }) if err != nil { - return err + return nil, err } - - bridge.Crypto = helper - return nil + client, err := mautrix.NewClient(helper.bridge.AS.HomeserverURL, helper.bridge.AS.BotMXID(), resp.AccessToken) + if err != nil { + return nil, err + } + client.DeviceID = resp.DeviceID + return client, nil } func (helper *CryptoHelper) Start() { @@ -101,27 +125,6 @@ func (helper *CryptoHelper) Stop() { helper.client.StopSync() } -func (bridge *Bridge) loginBot() (*mautrix.Client, error) { - mac := hmac.New(sha512.New, []byte(bridge.Config.Bridge.LoginSharedSecret)) - mac.Write([]byte(bridge.AS.BotMXID())) - resp, err := bridge.AS.BotClient().Login(&mautrix.ReqLogin{ - Type: "m.login.password", - Identifier: mautrix.UserIdentifier{Type: "m.id.user", User: string(bridge.AS.BotMXID())}, - Password: hex.EncodeToString(mac.Sum(nil)), - DeviceID: "WhatsApp Bridge", - InitialDeviceDisplayName: "WhatsApp Bridge", - }) - if err != nil { - return nil, err - } - client, err := mautrix.NewClient(bridge.AS.HomeserverURL, bridge.AS.BotMXID(), resp.AccessToken) - if err != nil { - return nil, err - } - client.DeviceID = "WhatsApp Bridge" - return client, nil -} - func (helper *CryptoHelper) Decrypt(evt *event.Event) (*event.Event, error) { return helper.mach.DecryptMegolmEvent(evt) } @@ -133,7 +136,7 @@ func (helper *CryptoHelper) Encrypt(roomID id.RoomID, evtType event.Type, conten return nil, err } helper.log.Debugfln("Got %v while encrypting event for %s, sharing group session and trying again...", err, roomID) - users, err := helper.bridge.StateStore.GetRoomMemberList(roomID) + users, err := helper.store.GetRoomMembers(roomID) if err != nil { return nil, errors.Wrap(err, "failed to get room member list") } @@ -202,6 +205,25 @@ func (c *cryptoLogger) Trace(message string, args ...interface{}) { c.int.Logfln(levelTrace, message, args...) } +type cryptoClientStore struct { + int *database.SQLCryptoStore +} + +func (c cryptoClientStore) SaveFilterID(_ id.UserID, _ string) {} +func (c cryptoClientStore) LoadFilterID(_ id.UserID) string { return "" } +func (c cryptoClientStore) SaveRoom(_ *mautrix.Room) {} +func (c cryptoClientStore) LoadRoom(_ id.RoomID) *mautrix.Room { return nil } + +func (c cryptoClientStore) SaveNextBatch(_ id.UserID, nextBatchToken string) { + c.int.PutNextBatch(nextBatchToken) +} + +func (c cryptoClientStore) LoadNextBatch(_ id.UserID) string { + return c.int.GetNextBatch() +} + +var _ mautrix.Storer = (*cryptoClientStore)(nil) + type cryptoStateStore struct { bridge *Bridge } diff --git a/database/cryptostore.go b/database/cryptostore.go new file mode 100644 index 0000000..8b36216 --- /dev/null +++ b/database/cryptostore.go @@ -0,0 +1,393 @@ +// mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. +// Copyright (C) 2020 Tulir Asokan +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package database + +import ( + "database/sql" + "fmt" + "strings" + "sync" + + "github.com/lib/pq" + "github.com/pkg/errors" + log "maunium.net/go/maulogger/v2" + + "maunium.net/go/mautrix/crypto" + "maunium.net/go/mautrix/crypto/olm" + "maunium.net/go/mautrix/id" +) + +type SQLCryptoStore struct { + db *Database + log log.Logger + + UserID id.UserID + DeviceID id.DeviceID + SyncToken string + PickleKey []byte + Account *crypto.OlmAccount + + GhostIDFormat string + + OGSLock sync.RWMutex + OutGroupSessions map[id.RoomID]*crypto.OutboundGroupSession +} + +var _ crypto.Store = (*SQLCryptoStore)(nil) + +func NewSQLCryptoStore(db *Database, deviceID id.DeviceID) *SQLCryptoStore { + return &SQLCryptoStore{ + db: db, + log: db.log.Sub("CryptoStore"), + PickleKey: []byte("maunium.net/go/mautrix-whatsapp"), + DeviceID: deviceID, + + OutGroupSessions: make(map[id.RoomID]*crypto.OutboundGroupSession), + } +} + +func (db *Database) FindDeviceID() (deviceID id.DeviceID) { + err := db.QueryRow("SELECT device_id FROM crypto_account LIMIT 1").Scan(&deviceID) + if err != nil && err != sql.ErrNoRows { + db.log.Warnln("Failed to scan device ID:", err) + } + return +} + +func (store *SQLCryptoStore) GetRoomMembers(roomID id.RoomID) (members []id.UserID, err error) { + var rows *sql.Rows + rows, err = store.db.Query(` + SELECT user_id FROM mx_user_profile + WHERE room_id=$1 + AND (membership='join' OR membership='invite') + AND user_id<>$2 + AND user_id NOT LIKE $3 + `, roomID, store.UserID, store.GhostIDFormat) + if err != nil { + return + } + for rows.Next() { + var userID id.UserID + err := rows.Scan(&userID) + if err != nil { + store.log.Warnfln("Failed to scan member in %s: %v", roomID, err) + } else { + members = append(members, userID) + } + } + return +} + +func (store *SQLCryptoStore) Flush() error { + return nil +} + +func (store *SQLCryptoStore) PutNextBatch(nextBatch string) { + store.SyncToken = nextBatch + _, err := store.db.Exec(`UPDATE crypto_account SET sync_token=$1 WHERE device_id=$2`, store.SyncToken, store.DeviceID) + if err != nil { + store.log.Warnln("Failed to store sync token:", err) + } +} + +func (store *SQLCryptoStore) GetNextBatch() string { + if store.SyncToken == "" { + err := store.db. + QueryRow("SELECT sync_token FROM crypto_account WHERE device_id=$1", store.DeviceID). + Scan(&store.SyncToken) + if err != nil && err != sql.ErrNoRows { + store.log.Warnln("Failed to scan sync token:", err) + } + } + return store.SyncToken +} + +func (store *SQLCryptoStore) PutAccount(account *crypto.OlmAccount) error { + store.Account = account + bytes := account.Internal.Pickle(store.PickleKey) + var err error + if store.db.dialect == "postgres" { + _, err = store.db.Exec(` + INSERT INTO crypto_account (device_id, shared, sync_token, account) VALUES ($1, $2, $3, $4) + ON CONFLICT (device_id) DO UPDATE SET shared=$2, sync_token=$3, account=$4`, + store.DeviceID, account.Shared, store.SyncToken, bytes) + } else if store.db.dialect == "sqlite3" { + _, err = store.db.Exec("INSERT OR REPLACE INTO crypto_account (deivce_id, shared, sync_token, account) VALUES ($1, $2, $3, $4)", + store.DeviceID, account.Shared, store.SyncToken, bytes) + } else { + err = fmt.Errorf("unsupported dialect %s", store.db.dialect) + } + if err != nil { + store.log.Warnln("Failed to store account:", err) + } + return nil +} + +func (store *SQLCryptoStore) GetAccount() (*crypto.OlmAccount, error) { + if store.Account == nil { + row := store.db.QueryRow("SELECT shared, sync_token, account FROM crypto_account WHERE device_id=$1", store.DeviceID) + acc := &crypto.OlmAccount{Internal: *olm.NewBlankAccount()} + var accountBytes []byte + err := row.Scan(&acc.Shared, &store.SyncToken, &accountBytes) + if err == sql.ErrNoRows { + return nil, nil + } else if err != nil { + return nil, err + } + err = acc.Internal.Unpickle(accountBytes, store.PickleKey) + if err != nil { + return nil, err + } + store.Account = acc + } + return store.Account, nil +} + +func (store *SQLCryptoStore) HasSession(key id.SenderKey) bool { + // TODO this may need to be changed if olm sessions start expiring + var sessionID id.SessionID + err := store.db.QueryRow("SELECT session_id FROM crypto_olm_session WHERE sender_key=$1 LIMIT 1", key).Scan(&sessionID) + if err == sql.ErrNoRows { + return false + } + return len(sessionID) > 0 +} + +func (store *SQLCryptoStore) GetSessions(key id.SenderKey) (crypto.OlmSessionList, error) { + rows, err := store.db.Query("SELECT session, created_at, last_used FROM crypto_olm_session WHERE sender_key=$1 ORDER BY session_id", key) + if err != nil { + return nil, err + } + list := crypto.OlmSessionList{} + for rows.Next() { + sess := crypto.OlmSession{Internal: *olm.NewBlankSession()} + var sessionBytes []byte + err := rows.Scan(&sessionBytes, &sess.CreationTime, &sess.UseTime) + if err != nil { + return nil, err + } + err = sess.Internal.Unpickle(sessionBytes, store.PickleKey) + if err != nil { + return nil, err + } + list = append(list, &sess) + } + return list, nil +} + +func (store *SQLCryptoStore) GetLatestSession(key id.SenderKey) (*crypto.OlmSession, error) { + row := store.db.QueryRow("SELECT session, created_at, last_used FROM crypto_olm_session WHERE sender_key=$1 ORDER BY session_id DESC LIMIT 1", key) + sess := crypto.OlmSession{Internal: *olm.NewBlankSession()} + var sessionBytes []byte + err := row.Scan(&sessionBytes, &sess.CreationTime, &sess.UseTime) + if err == sql.ErrNoRows { + return nil, nil + } else if err != nil { + return nil, err + } + return &sess, sess.Internal.Unpickle(sessionBytes, store.PickleKey) +} + +func (store *SQLCryptoStore) AddSession(key id.SenderKey, session *crypto.OlmSession) error { + sessionBytes := session.Internal.Pickle(store.PickleKey) + _, err := store.db.Exec("INSERT INTO crypto_olm_session (session_id, sender_key, session, created_at, last_used) VALUES ($1, $2, $3, $4, $5)", + session.ID(), key, sessionBytes, session.CreationTime, session.UseTime) + return err +} + +func (store *SQLCryptoStore) PutGroupSession(roomID id.RoomID, senderKey id.SenderKey, sessionID id.SessionID, session *crypto.InboundGroupSession) error { + sessionBytes := session.Internal.Pickle(store.PickleKey) + forwardingChains := strings.Join(session.ForwardingChains, ",") + _, err := store.db.Exec("INSERT INTO crypto_megolm_inbound_session (session_id, sender_key, signing_key, room_id, session, forwarding_chains) VALUES ($1, $2, $3, $4, $5, $6)", + sessionID, senderKey, session.SigningKey, roomID, sessionBytes, forwardingChains) + return err +} + +func (store *SQLCryptoStore) GetGroupSession(roomID id.RoomID, senderKey id.SenderKey, sessionID id.SessionID) (*crypto.InboundGroupSession, error) { + var signingKey id.Ed25519 + var sessionBytes []byte + var forwardingChains string + err := store.db.QueryRow(` + SELECT signing_key, session, forwarding_chains + FROM crypto_megolm_inbound_session + WHERE room_id=$1 AND sender_key=$2 AND session_id=$3`, + roomID, senderKey, sessionID, + ).Scan(&signingKey, &sessionBytes, &forwardingChains) + if err == sql.ErrNoRows { + return nil, nil + } else if err != nil { + return nil, err + } + igs := olm.NewBlankInboundGroupSession() + err = igs.Unpickle(sessionBytes, store.PickleKey) + if err != nil { + return nil, err + } + return &crypto.InboundGroupSession{ + Internal: *igs, + SigningKey: signingKey, + SenderKey: senderKey, + RoomID: roomID, + ForwardingChains: strings.Split(forwardingChains, ","), + }, nil +} + +func (store *SQLCryptoStore) PutOutboundGroupSession(roomID id.RoomID, session *crypto.OutboundGroupSession) error { + store.OGSLock.Lock() + store.OutGroupSessions[roomID] = session + store.OGSLock.Unlock() + return nil +} + +func (store *SQLCryptoStore) GetOutboundGroupSession(roomID id.RoomID) (*crypto.OutboundGroupSession, error) { + store.OGSLock.RLock() + defer store.OGSLock.RUnlock() + return store.OutGroupSessions[roomID], nil +} + +func (store *SQLCryptoStore) PopOutboundGroupSession(roomID id.RoomID) error { + store.OGSLock.Lock() + delete(store.OutGroupSessions, roomID) + store.OGSLock.Unlock() + return nil +} + +func (store *SQLCryptoStore) ValidateMessageIndex(senderKey id.SenderKey, sessionID id.SessionID, eventID id.EventID, index uint, timestamp int64) bool { + var resultEventID id.EventID + var resultTimestamp int64 + err := store.db.QueryRow( + "SELECT event_id, timestamp FROM crypto_message_index WHERE sender_key=$1 AND session_id=$2 AND index=$3", + senderKey, sessionID, index, + ).Scan(&resultEventID, &resultTimestamp) + if err == sql.ErrNoRows { + _, err := store.db.Exec("INSERT INTO crypto_message_index (sender_key, session_id, index, event_id, timestamp) VALUES ($1, $2, $3, $4, $5)", + senderKey, sessionID, index, eventID, timestamp) + if err != nil { + store.log.Warnln("Failed to store message index:", err) + } + return true + } else if err != nil { + store.log.Warnln("Failed to scan message index:", err) + return true + } + if resultEventID != eventID || resultTimestamp != timestamp { + return false + } + return true +} + +func (store *SQLCryptoStore) GetDevices(userID id.UserID) (map[id.DeviceID]*crypto.DeviceIdentity, error) { + var ignore id.UserID + err := store.db.QueryRow("SELECT user_id FROM crypto_tracked_user WHERE user_id=$1", userID).Scan(&ignore) + if err == sql.ErrNoRows { + return nil, nil + } else if err != nil { + return nil, err + } + + rows, err := store.db.Query("SELECT device_id, identity_key, signing_key, trust, deleted, name FROM crypto_device WHERE user_id=$1", userID) + if err != nil { + return nil, err + } + data := make(map[id.DeviceID]*crypto.DeviceIdentity) + for rows.Next() { + var identity crypto.DeviceIdentity + err := rows.Scan(&identity.DeviceID, &identity.IdentityKey, &identity.SigningKey, &identity.Trust, &identity.Deleted, &identity.Name) + if err != nil { + return nil, err + } + identity.UserID = userID + data[identity.DeviceID] = &identity + } + return data, nil +} + +func (store *SQLCryptoStore) PutDevices(userID id.UserID, devices map[id.DeviceID]*crypto.DeviceIdentity) error { + tx, err := store.db.Begin() + if err != nil { + return err + } + + if store.db.dialect == "postgres" { + _, err = tx.Exec("INSERT INTO crypto_tracked_user (user_id) VALUES ($1) ON CONFLICT (user_id) DO NOTHING", userID) + } else if store.db.dialect == "sqlite3" { + _, err = tx.Exec("INSERT OR IGNORE INTO crypto_tracked_users (user_id) VALUES ($1)", userID) + } else { + err = fmt.Errorf("unsupported dialect %s", store.db.dialect) + } + if err != nil { + return errors.Wrap(err, "failed to add user to tracked users list") + } + + _, err = tx.Exec("DELETE FROM crypto_device WHERE user_id=$1", userID) + if err != nil { + _ = tx.Rollback() + return errors.Wrap(err, "failed to delete old devices") + } + if len(devices) == 0 { + err = tx.Commit() + if err != nil { + return errors.Wrap(err, "failed to commit changes (no devices added)") + } + return nil + } + // TODO do this in batches to avoid too large db queries + values := make([]interface{}, 1, len(devices)*6+1) + values[0] = userID + valueStrings := make([]string, 0, len(devices)) + i := 2 + for deviceID, identity := range devices { + values = append(values, deviceID, identity.IdentityKey, identity.SigningKey, identity.Trust, identity.Deleted, identity.Name) + valueStrings = append(valueStrings, fmt.Sprintf("($1, $%d, $%d, $%d, $%d, $%d, $%d)", i, i+1, i+2, i+3, i+4, i+5)) + i += 6 + } + valueString := strings.Join(valueStrings, ",") + _, err = tx.Exec("INSERT INTO crypto_device (user_id, device_id, identity_key, signing_key, trust, deleted, name) VALUES "+valueString, values...) + if err != nil { + _ = tx.Rollback() + return errors.Wrap(err, "failed to insert new devices") + } + err = tx.Commit() + if err != nil { + return errors.Wrap(err, "failed to commit changes") + } + return nil +} + +func (store *SQLCryptoStore) FilterTrackedUsers(users []id.UserID) []id.UserID { + var rows *sql.Rows + var err error + if store.db.dialect == "postgres" { + rows, err = store.db.Query("SELECT user_id FROM crypto_tracked_user WHERE user_id = ANY($1)", pq.Array(users)) + } else { + rows, err = store.db.Query("SELECT user_id FROM crypto_tracked_user WHERE user_id IN ($1)", users) + } + if err != nil { + store.log.Warnln("Failed to filter tracked users:", err) + return users + } + var ptr int + for rows.Next() { + err = rows.Scan(&users[ptr]) + if err != nil { + store.log.Warnln("Failed to tracked user ID:", err) + } else { + ptr++ + } + } + return users[:ptr] +} diff --git a/database/statestore.go b/database/statestore.go index 178d4c1..e76d60f 100644 --- a/database/statestore.go +++ b/database/statestore.go @@ -39,11 +39,13 @@ type SQLStateStore struct { typingLock sync.RWMutex } +var _ appservice.StateStore = (*SQLStateStore)(nil) + func NewSQLStateStore(db *Database) *SQLStateStore { return &SQLStateStore{ TypingStateStore: appservice.NewTypingStateStore(), db: db, - log: log.Sub("StateStore"), + log: db.log.Sub("StateStore"), } } @@ -90,24 +92,6 @@ func (store *SQLStateStore) GetRoomMembers(roomID id.RoomID) map[id.UserID]*even return members } -func (store *SQLStateStore) GetRoomMemberList(roomID id.RoomID) (members []id.UserID, err error) { - var rows *sql.Rows - rows, err = store.db.Query("SELECT user_id FROM mx_user_profile WHERE room_id=$1", roomID) - if err != nil { - return - } - for rows.Next() { - var userID id.UserID - err := rows.Scan(&userID) - if err != nil { - store.log.Warnfln("Failed to scan member in %s: %v", roomID, err) - } else { - members = append(members, userID) - } - } - return -} - func (store *SQLStateStore) GetMembership(roomID id.RoomID, userID id.UserID) event.Membership { row := store.db.QueryRow("SELECT membership FROM mx_user_profile WHERE room_id=$1 AND user_id=$2", roomID, userID) membership := event.MembershipLeave @@ -138,8 +122,10 @@ func (store *SQLStateStore) TryGetMember(roomID id.RoomID, userID id.UserID) (*e func (store *SQLStateStore) FindSharedRooms(userID id.UserID) (rooms []id.RoomID) { rows, err := store.db.Query(` - SELECT room_id FROM mx_user_profile WHERE user_id=$2 AND portal.encrypted=true - LEFT JOIN portal WHEN portal.mxid=mx_user_profile.room_id`, userID) + SELECT room_id FROM mx_user_profile + LEFT JOIN portal ON portal.mxid=mx_user_profile.room_id + WHERE user_id=$1 AND portal.encrypted=true + `, userID) if err != nil { store.log.Warnfln("Failed to query shared rooms with %s: %v", userID, err) return diff --git a/database/upgrades/2020-05-09-crypto-store.go b/database/upgrades/2020-05-09-crypto-store.go new file mode 100644 index 0000000..529ff9c --- /dev/null +++ b/database/upgrades/2020-05-09-crypto-store.go @@ -0,0 +1,74 @@ +package upgrades + +import ( + "database/sql" +) + +func init() { + upgrades[13] = upgrade{"Add crypto store to database", func(tx *sql.Tx, ctx context) error { + // TODO use DATETIME instead of timestamp and BLOB instead of bytea for sqlite + _, err := tx.Exec(`CREATE TABLE crypto_account ( + device_id VARCHAR(255) PRIMARY KEY, + shared BOOLEAN NOT NULL, + sync_token TEXT NOT NULL, + account bytea NOT NULL + )`) + if err != nil { + return err + } + _, err = tx.Exec(`CREATE TABLE crypto_message_index ( + sender_key CHAR(43), + session_id VARCHAR(255), + index INTEGER, + event_id VARCHAR(255) NOT NULL, + timestamp BIGINT NOT NULL, + + PRIMARY KEY (sender_key, session_id, index) + )`) + if err != nil { + return err + } + _, err = tx.Exec(`CREATE TABLE crypto_tracked_user ( + user_id VARCHAR(255) PRIMARY KEY + )`) + if err != nil { + return err + } + _, err = tx.Exec(`CREATE TABLE crypto_device ( + user_id VARCHAR(255), + device_id VARCHAR(255), + identity_key CHAR(43) NOT NULL, + signing_key CHAR(43) NOT NULL, + trust SMALLINT NOT NULL, + deleted BOOLEAN NOT NULL, + name VARCHAR(255) NOT NULL, + + PRIMARY KEY (user_id, device_id) + )`) + if err != nil { + return err + } + _, err = tx.Exec(`CREATE TABLE crypto_olm_session ( + session_id CHAR(43) PRIMARY KEY, + sender_key VARCHAR(255) NOT NULL, + session bytea NOT NULL, + created_at timestamp NOT NULL, + last_used timestamp NOT NULL + )`) + if err != nil { + return err + } + _, err = tx.Exec(`CREATE TABLE crypto_megolm_inbound_session ( + session_id CHAR(43) PRIMARY KEY, + sender_key CHAR(43) NOT NULL, + signing_key CHAR(43) NOT NULL, + room_id VARCHAR(255) NOT NULL, + session bytea NOT NULL, + forwarding_chains bytea NOT NULL + )`) + if err != nil { + return err + } + return nil + }} +} diff --git a/database/upgrades/upgrades.go b/database/upgrades/upgrades.go index 3126cc7..ec8e6e7 100644 --- a/database/upgrades/upgrades.go +++ b/database/upgrades/upgrades.go @@ -28,7 +28,7 @@ type upgrade struct { fn upgradeFunc } -const NumberOfUpgrades = 13 +const NumberOfUpgrades = 14 var upgrades [NumberOfUpgrades]upgrade diff --git a/go.mod b/go.mod index 4534d58..38dc65e 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( gopkg.in/yaml.v2 v2.2.8 maunium.net/go/mauflag v1.0.0 maunium.net/go/maulogger/v2 v2.1.1 - maunium.net/go/mautrix v0.4.0 + maunium.net/go/mautrix v0.4.1 ) replace github.com/Rhymen/go-whatsapp => github.com/tulir/go-whatsapp v0.2.6 diff --git a/go.sum b/go.sum index cba311b..a109e80 100644 --- a/go.sum +++ b/go.sum @@ -86,3 +86,5 @@ maunium.net/go/mautrix v0.3.7 h1:N0czrZeAwjvBrw2a/B2G6U3EwIYaWpt7OuSslGp8DRc= maunium.net/go/mautrix v0.3.7/go.mod h1:SkGZzch8CvU2qKtNpYxtzZ0sQxfVEJ3IsVVLSUBUx9Y= maunium.net/go/mautrix v0.4.0 h1:IYfmxCoxR/6UMi92IncsSZeKQbZm8Xa35XIRX814KJ4= maunium.net/go/mautrix v0.4.0/go.mod h1:8Y+NqmROJyWYvvP4yPfX9tLM59VCfgE/kcQ0SeX68ho= +maunium.net/go/mautrix v0.4.1 h1:i2lJNT+TE4AAL3cVKUN4jKVRkujCE/oS8aIsj8+7iNE= +maunium.net/go/mautrix v0.4.1/go.mod h1:8Y+NqmROJyWYvvP4yPfX9tLM59VCfgE/kcQ0SeX68ho= diff --git a/main.go b/main.go index ddcd4b3..13cb13e 100644 --- a/main.go +++ b/main.go @@ -126,6 +126,7 @@ type Crypto interface { HandleMemberEvent(*event.Event) Decrypt(*event.Event) (*event.Event, error) Encrypt(id.RoomID, event.Type, event.Content) (*event.EncryptedEventContent, error) + Init() error Start() Stop() } @@ -225,11 +226,7 @@ func (bridge *Bridge) Init() { bridge.Log.Debugln("Initializing Matrix event handler") bridge.MatrixHandler = NewMatrixHandler(bridge) bridge.Formatter = NewFormatter(bridge) - err = bridge.initCrypto() - if err != nil { - bridge.Log.Fatalln("Error initializing end-to-bridge encryption:", err) - os.Exit(19) - } + bridge.Crypto = NewCryptoHelper(bridge) } func (bridge *Bridge) Start() { @@ -238,6 +235,13 @@ func (bridge *Bridge) Start() { bridge.Log.Fatalln("Failed to initialize database:", err) os.Exit(15) } + if bridge.Crypto != nil { + err := bridge.Crypto.Init() + if err != nil { + bridge.Log.Fatalln("Error initializing end-to-bridge encryption:", err) + os.Exit(19) + } + } if bridge.Provisioning != nil { bridge.Log.Debugln("Initializing provisioning API") bridge.Provisioning.Init() From ea239074921c5c708a9fb52a186587d457ad8226 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 9 May 2020 20:23:30 +0300 Subject: [PATCH 09/22] Fix some bugs with db crypto store --- database/cryptostore.go | 16 ++++++++---- database/migrate.go | 26 +++++++++++++++++++- database/upgrades/2020-05-09-crypto-store.go | 4 +-- matrix.go | 2 +- 4 files changed, 39 insertions(+), 9 deletions(-) diff --git a/database/cryptostore.go b/database/cryptostore.go index 8b36216..0f780b2 100644 --- a/database/cryptostore.go +++ b/database/cryptostore.go @@ -126,7 +126,7 @@ func (store *SQLCryptoStore) PutAccount(account *crypto.OlmAccount) error { ON CONFLICT (device_id) DO UPDATE SET shared=$2, sync_token=$3, account=$4`, store.DeviceID, account.Shared, store.SyncToken, bytes) } else if store.db.dialect == "sqlite3" { - _, err = store.db.Exec("INSERT OR REPLACE INTO crypto_account (deivce_id, shared, sync_token, account) VALUES ($1, $2, $3, $4)", + _, err = store.db.Exec("INSERT OR REPLACE INTO crypto_account (device_id, shared, sync_token, account) VALUES ($1, $2, $3, $4)", store.DeviceID, account.Shared, store.SyncToken, bytes) } else { err = fmt.Errorf("unsupported dialect %s", store.db.dialect) @@ -270,11 +270,11 @@ func (store *SQLCryptoStore) ValidateMessageIndex(senderKey id.SenderKey, sessio var resultEventID id.EventID var resultTimestamp int64 err := store.db.QueryRow( - "SELECT event_id, timestamp FROM crypto_message_index WHERE sender_key=$1 AND session_id=$2 AND index=$3", + `SELECT event_id, timestamp FROM crypto_message_index WHERE sender_key=$1 AND session_id=$2 AND "index"=$3`, senderKey, sessionID, index, ).Scan(&resultEventID, &resultTimestamp) if err == sql.ErrNoRows { - _, err := store.db.Exec("INSERT INTO crypto_message_index (sender_key, session_id, index, event_id, timestamp) VALUES ($1, $2, $3, $4, $5)", + _, err := store.db.Exec(`INSERT INTO crypto_message_index (sender_key, session_id, "index", event_id, timestamp) VALUES ($1, $2, $3, $4, $5)`, senderKey, sessionID, index, eventID, timestamp) if err != nil { store.log.Warnln("Failed to store message index:", err) @@ -325,7 +325,7 @@ func (store *SQLCryptoStore) PutDevices(userID id.UserID, devices map[id.DeviceI if store.db.dialect == "postgres" { _, err = tx.Exec("INSERT INTO crypto_tracked_user (user_id) VALUES ($1) ON CONFLICT (user_id) DO NOTHING", userID) } else if store.db.dialect == "sqlite3" { - _, err = tx.Exec("INSERT OR IGNORE INTO crypto_tracked_users (user_id) VALUES ($1)", userID) + _, err = tx.Exec("INSERT OR IGNORE INTO crypto_tracked_user (user_id) VALUES ($1)", userID) } else { err = fmt.Errorf("unsupported dialect %s", store.db.dialect) } @@ -374,7 +374,13 @@ func (store *SQLCryptoStore) FilterTrackedUsers(users []id.UserID) []id.UserID { if store.db.dialect == "postgres" { rows, err = store.db.Query("SELECT user_id FROM crypto_tracked_user WHERE user_id = ANY($1)", pq.Array(users)) } else { - rows, err = store.db.Query("SELECT user_id FROM crypto_tracked_user WHERE user_id IN ($1)", users) + queryString := make([]string, len(users)) + params := make([]interface{}, len(users)) + for i, user := range users { + queryString[i] = fmt.Sprintf("$%d", i+1) + params[i] = user + } + rows, err = store.db.Query("SELECT user_id FROM crypto_tracked_user WHERE user_id IN (" + strings.Join(queryString, ",") + ")", params...) } if err != nil { store.log.Warnln("Failed to filter tracked users:", err) diff --git a/database/migrate.go b/database/migrate.go index b3cf4e0..9d30871 100644 --- a/database/migrate.go +++ b/database/migrate.go @@ -89,7 +89,7 @@ func migrateTable(old *Database, new *Database, table string, columns ...string) } func Migrate(old *Database, new *Database) { - err := migrateTable(old, new, "portal", "jid", "receiver", "mxid", "name", "topic", "avatar", "avatar_url") + err := migrateTable(old, new, "portal", "jid", "receiver", "mxid", "name", "topic", "avatar", "avatar_url", "encrypted") if err != nil { panic(err) } @@ -121,4 +121,28 @@ func Migrate(old *Database, new *Database) { if err != nil { panic(err) } + err = migrateTable(old, new, "crypto_account", "device_id", "shared", "sync_token", "account") + if err != nil { + panic(err) + } + err = migrateTable(old, new, "crypto_message_index", "sender_key", "session_id", `"index"`, "event_id", "timestamp") + if err != nil { + panic(err) + } + err = migrateTable(old, new, "crypto_tracked_user", "user_id") + if err != nil { + panic(err) + } + err = migrateTable(old, new, "crypto_device", "user_id", "device_id", "identity_key", "signing_key", "trust", "deleted", "name") + if err != nil { + panic(err) + } + err = migrateTable(old, new, "crypto_olm_session", "session_id", "sender_key", "session", "created_at", "last_used") + if err != nil { + panic(err) + } + err = migrateTable(old, new, "crypto_megolm_inbound_session", "session_id", "sender_key", "signing_key", "room_id", "session", "forwarding_chains") + if err != nil { + panic(err) + } } diff --git a/database/upgrades/2020-05-09-crypto-store.go b/database/upgrades/2020-05-09-crypto-store.go index 529ff9c..ea454dc 100644 --- a/database/upgrades/2020-05-09-crypto-store.go +++ b/database/upgrades/2020-05-09-crypto-store.go @@ -19,11 +19,11 @@ func init() { _, err = tx.Exec(`CREATE TABLE crypto_message_index ( sender_key CHAR(43), session_id VARCHAR(255), - index INTEGER, + "index" INTEGER, event_id VARCHAR(255) NOT NULL, timestamp BIGINT NOT NULL, - PRIMARY KEY (sender_key, session_id, index) + PRIMARY KEY (sender_key, session_id, "index") )`) if err != nil { return err diff --git a/matrix.go b/matrix.go index d064f49..3e524f4 100644 --- a/matrix.go +++ b/matrix.go @@ -217,7 +217,7 @@ func (mx *MatrixHandler) HandleEncrypted(evt *event.Event) { decrypted, err := mx.bridge.Crypto.Decrypt(evt) if err != nil { - mx.log.Warnln("Failed to decrypt %s: %v", evt.ID, err) + mx.log.Warnfln("Failed to decrypt %s: %v", evt.ID, err) return } mx.bridge.EventProcessor.Dispatch(decrypted) From 168a6ff93f8cd92f166a222efa5d3103fc610f2b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 9 May 2020 20:25:48 +0300 Subject: [PATCH 10/22] Fix static build --- crypto.go | 2 +- database/cryptostore.go | 2 ++ no-cgo.go | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crypto.go b/crypto.go index 286b600..5e1f0d9 100644 --- a/crypto.go +++ b/crypto.go @@ -49,7 +49,7 @@ type CryptoHelper struct { baseLog maulogger.Logger } -func NewCryptoHelper(bridge *Bridge) *CryptoHelper { +func NewCryptoHelper(bridge *Bridge) Crypto { if !bridge.Config.Bridge.Encryption.Allow { bridge.Log.Debugln("Bridge built with end-to-bridge encryption, but disabled in config") return nil diff --git a/database/cryptostore.go b/database/cryptostore.go index 0f780b2..f044002 100644 --- a/database/cryptostore.go +++ b/database/cryptostore.go @@ -14,6 +14,8 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +// +build cgo + package database import ( diff --git a/no-cgo.go b/no-cgo.go index 961bf5c..88b359a 100644 --- a/no-cgo.go +++ b/no-cgo.go @@ -25,7 +25,7 @@ import ( "golang.org/x/image/webp" ) -func (bridge *Bridge) initCrypto() error { +func NewCryptoHelper(bridge *Bridge) Crypto { if !bridge.Config.Bridge.Encryption.Allow { bridge.Log.Warnln("Bridge built without end-to-bridge encryption, but encryption is enabled in config") } From db5c1a3f615af21eeacc411cf987c700557ceaa9 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 12 May 2020 20:30:38 +0300 Subject: [PATCH 11/22] Make sure user gets invited to portal when using open command --- commands.go | 1 + 1 file changed, 1 insertion(+) diff --git a/commands.go b/commands.go index 79d6b7b..3894f0f 100644 --- a/commands.go +++ b/commands.go @@ -559,6 +559,7 @@ func (handler *CommandHandler) CommandOpen(ce *CommandEvent) { portal.Sync(user, contact) ce.Reply("Portal room created.") } + _, _ = portal.MainIntent().InviteUser(portal.MXID, &mautrix.ReqInviteUser{UserID: user.MXID}) } const cmdPMHelp = `pm [--force] <_international phone number_> - Open a private chat with the given phone number.` From 1c3de877db234951104059ebb51c8b21941d536f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 12 May 2020 22:25:55 +0300 Subject: [PATCH 12/22] Maybe support e2be by default and fix some bugs --- crypto.go | 3 +- database/cryptostore.go | 7 +++++ database/upgrades/2020-05-09-crypto-store.go | 12 ++++---- example-config.yaml | 1 + go.mod | 2 +- go.sum | 4 +++ matrix.go | 4 +++ portal.go | 30 ++++++++++++++++++-- 8 files changed, 52 insertions(+), 11 deletions(-) diff --git a/crypto.go b/crypto.go index 5e1f0d9..6067b99 100644 --- a/crypto.go +++ b/crypto.go @@ -22,6 +22,7 @@ import ( "crypto/hmac" "crypto/sha512" "encoding/hex" + "fmt" "time" "github.com/pkg/errors" @@ -78,7 +79,7 @@ func (helper *CryptoHelper) Init() error { stateStore := &cryptoStateStore{helper.bridge} helper.store = database.NewSQLCryptoStore(helper.bridge.DB, helper.client.DeviceID) helper.store.UserID = helper.client.UserID - helper.store.GhostIDFormat = helper.bridge.Config.Bridge.FormatUsername("%") + helper.store.GhostIDFormat = fmt.Sprintf("@%s:%s", helper.bridge.Config.Bridge.FormatUsername("%"), helper.bridge.AS.HomeserverDomain) helper.mach = crypto.NewOlmMachine(helper.client, logger, helper.store, stateStore) helper.client.Logger = logger.int.Sub("Bot") diff --git a/database/cryptostore.go b/database/cryptostore.go index f044002..5cca49b 100644 --- a/database/cryptostore.go +++ b/database/cryptostore.go @@ -211,6 +211,13 @@ func (store *SQLCryptoStore) AddSession(key id.SenderKey, session *crypto.OlmSes return err } +func (store *SQLCryptoStore) UpdateSession(key id.SenderKey, session *crypto.OlmSession) error { + sessionBytes := session.Internal.Pickle(store.PickleKey) + _, err := store.db.Exec("UPDATE crypto_olm_session SET session=$1, last_used=$2 WHERE session_id=$3", + sessionBytes, session.UseTime, session.ID()) + return err +} + func (store *SQLCryptoStore) PutGroupSession(roomID id.RoomID, senderKey id.SenderKey, sessionID id.SessionID, session *crypto.InboundGroupSession) error { sessionBytes := session.Internal.Pickle(store.PickleKey) forwardingChains := strings.Join(session.ForwardingChains, ",") diff --git a/database/upgrades/2020-05-09-crypto-store.go b/database/upgrades/2020-05-09-crypto-store.go index ea454dc..fd479b0 100644 --- a/database/upgrades/2020-05-09-crypto-store.go +++ b/database/upgrades/2020-05-09-crypto-store.go @@ -18,7 +18,7 @@ func init() { } _, err = tx.Exec(`CREATE TABLE crypto_message_index ( sender_key CHAR(43), - session_id VARCHAR(255), + session_id CHAR(43), "index" INTEGER, event_id VARCHAR(255) NOT NULL, timestamp BIGINT NOT NULL, @@ -49,11 +49,11 @@ func init() { return err } _, err = tx.Exec(`CREATE TABLE crypto_olm_session ( - session_id CHAR(43) PRIMARY KEY, - sender_key VARCHAR(255) NOT NULL, - session bytea NOT NULL, - created_at timestamp NOT NULL, - last_used timestamp NOT NULL + session_id CHAR(43) PRIMARY KEY, + sender_key CHAR(43) NOT NULL, + session bytea NOT NULL, + created_at timestamp NOT NULL, + last_used timestamp NOT NULL )`) if err != nil { return err diff --git a/example-config.yaml b/example-config.yaml index 6b66025..dfe2c25 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -148,6 +148,7 @@ bridge: allow: false # Default to encryption, force-enable encryption in all portals the bridge creates # This will cause the bridge bot to be in private chats for the encryption to work properly. + # It is recommended to also set private_chat_portal_meta to true when using this. default: false # Permissions for using the bridge. diff --git a/go.mod b/go.mod index 38dc65e..f8d6064 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( gopkg.in/yaml.v2 v2.2.8 maunium.net/go/mauflag v1.0.0 maunium.net/go/maulogger/v2 v2.1.1 - maunium.net/go/mautrix v0.4.1 + maunium.net/go/mautrix v0.4.3 ) replace github.com/Rhymen/go-whatsapp => github.com/tulir/go-whatsapp v0.2.6 diff --git a/go.sum b/go.sum index a109e80..9dbe6fa 100644 --- a/go.sum +++ b/go.sum @@ -88,3 +88,7 @@ maunium.net/go/mautrix v0.4.0 h1:IYfmxCoxR/6UMi92IncsSZeKQbZm8Xa35XIRX814KJ4= maunium.net/go/mautrix v0.4.0/go.mod h1:8Y+NqmROJyWYvvP4yPfX9tLM59VCfgE/kcQ0SeX68ho= maunium.net/go/mautrix v0.4.1 h1:i2lJNT+TE4AAL3cVKUN4jKVRkujCE/oS8aIsj8+7iNE= maunium.net/go/mautrix v0.4.1/go.mod h1:8Y+NqmROJyWYvvP4yPfX9tLM59VCfgE/kcQ0SeX68ho= +maunium.net/go/mautrix v0.4.2 h1:GBU++Z7o/fLPcEsNMkNOUsnDknwV/MGPQ0BN4ikK6tw= +maunium.net/go/mautrix v0.4.2/go.mod h1:8Y+NqmROJyWYvvP4yPfX9tLM59VCfgE/kcQ0SeX68ho= +maunium.net/go/mautrix v0.4.3 h1:fVoJy992TjBEvuK5NeO9fpBh+9JuSFsxaEdGjFp/7h4= +maunium.net/go/mautrix v0.4.3/go.mod h1:8Y+NqmROJyWYvvP4yPfX9tLM59VCfgE/kcQ0SeX68ho= diff --git a/matrix.go b/matrix.go index 3e524f4..42779e2 100644 --- a/matrix.go +++ b/matrix.go @@ -130,6 +130,10 @@ func (mx *MatrixHandler) HandleBotInvite(evt *event.Event) { } func (mx *MatrixHandler) HandleMembership(evt *event.Event) { + if _, isPuppet := mx.bridge.ParsePuppetMXID(evt.Sender); evt.Sender == mx.bridge.Bot.UserID || isPuppet { + return + } + if mx.bridge.Crypto != nil { mx.bridge.Crypto.HandleMemberEvent(evt) } diff --git a/portal.go b/portal.go index ffac2c8..04b6c96 100644 --- a/portal.go +++ b/portal.go @@ -725,7 +725,6 @@ func (portal *Portal) CreateMatrixRoom(user *User) error { portal.log.Infoln("Creating Matrix room. Info source:", user.MXID) var metadata *whatsappExt.GroupInfo - isPrivateChat := false if portal.IsPrivateChat() { puppet := portal.bridge.GetPuppetByJID(portal.Key.JID) if portal.bridge.Config.Bridge.PrivateChatPortalMeta { @@ -736,7 +735,6 @@ func (portal *Portal) CreateMatrixRoom(user *User) error { portal.Name = "" } portal.Topic = "WhatsApp private chat" - isPrivateChat = true } else if portal.IsStatusBroadcastRoom() { portal.Name = "WhatsApp Status Broadcast" portal.Topic = "WhatsApp status updates from your contacts" @@ -770,13 +768,26 @@ func (portal *Portal) CreateMatrixRoom(user *User) error { invite = portal.bridge.Config.Bridge.Relaybot.InviteUsers } + if portal.bridge.Config.Bridge.Encryption.Default { + initialState = append(initialState, &event.Event{ + Type: event.StateEncryption, + Content: event.Content{ + Parsed: event.EncryptionEventContent{Algorithm: id.AlgorithmMegolmV1}, + }, + }) + portal.Encrypted = true + if portal.IsPrivateChat() { + invite = append(invite, portal.bridge.Bot.UserID) + } + } + resp, err := intent.CreateRoom(&mautrix.ReqCreateRoom{ Visibility: "private", Name: portal.Name, Topic: portal.Topic, Invite: invite, Preset: "private_chat", - IsDirect: isPrivateChat, + IsDirect: portal.IsPrivateChat(), InitialState: initialState, }) if err != nil { @@ -784,6 +795,12 @@ func (portal *Portal) CreateMatrixRoom(user *User) error { } portal.MXID = resp.RoomID portal.Update() + + // We set the memberships beforehand to make sure the encryption key exchange in initial backfill knows the users are here. + for _, user := range invite { + portal.bridge.StateStore.SetMembership(portal.MXID, user, event.MembershipInvite) + } + if metadata != nil { portal.SyncParticipants(metadata) } else { @@ -796,6 +813,13 @@ func (portal *Portal) CreateMatrixRoom(user *User) error { if portal.IsPrivateChat() { puppet := user.bridge.GetPuppetByJID(portal.Key.JID) user.addPuppetToCommunity(puppet) + + if portal.bridge.Config.Bridge.Encryption.Default { + err = portal.bridge.Bot.EnsureJoined(portal.MXID) + if err != nil { + portal.log.Errorln("Failed to join created portal with bridge bot for e2be:", err) + } + } } err = portal.FillInitialHistory(user) if err != nil { From c9adb3aba3d48c02b0db2141e579406b08af00db Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 12 May 2020 23:16:33 +0300 Subject: [PATCH 13/22] Store outbound group sessions in database --- database/cryptostore.go | 56 ++++++++++++------- ...2020-05-12-outbound-group-session-store.go | 26 +++++++++ database/upgrades/upgrades.go | 2 +- go.mod | 2 +- go.sum | 2 + 5 files changed, 66 insertions(+), 22 deletions(-) create mode 100644 database/upgrades/2020-05-12-outbound-group-session-store.go diff --git a/database/cryptostore.go b/database/cryptostore.go index 5cca49b..dbfca25 100644 --- a/database/cryptostore.go +++ b/database/cryptostore.go @@ -22,7 +22,6 @@ import ( "database/sql" "fmt" "strings" - "sync" "github.com/lib/pq" "github.com/pkg/errors" @@ -44,9 +43,6 @@ type SQLCryptoStore struct { Account *crypto.OlmAccount GhostIDFormat string - - OGSLock sync.RWMutex - OutGroupSessions map[id.RoomID]*crypto.OutboundGroupSession } var _ crypto.Store = (*SQLCryptoStore)(nil) @@ -57,8 +53,6 @@ func NewSQLCryptoStore(db *Database, deviceID id.DeviceID) *SQLCryptoStore { log: db.log.Sub("CryptoStore"), PickleKey: []byte("maunium.net/go/mautrix-whatsapp"), DeviceID: deviceID, - - OutGroupSessions: make(map[id.RoomID]*crypto.OutboundGroupSession), } } @@ -255,24 +249,46 @@ func (store *SQLCryptoStore) GetGroupSession(roomID id.RoomID, senderKey id.Send }, nil } -func (store *SQLCryptoStore) PutOutboundGroupSession(roomID id.RoomID, session *crypto.OutboundGroupSession) error { - store.OGSLock.Lock() - store.OutGroupSessions[roomID] = session - store.OGSLock.Unlock() - return nil +func (store *SQLCryptoStore) AddOutboundGroupSession(session *crypto.OutboundGroupSession) error { + sessionBytes := session.Internal.Pickle(store.PickleKey) + _, err := store.db.Exec("INSERT INTO crypto_megolm_outbound_session (room_id, session_id, session, shared, max_messages, message_count, max_age, created_at, last_used) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + session.RoomID, session.ID(), sessionBytes, session.Shared, session.MaxMessages, session.MessageCount, session.MaxAge, session.CreationTime, session.UseTime) + return err +} + +func (store *SQLCryptoStore) UpdateOutboundGroupSession(session *crypto.OutboundGroupSession) error { + sessionBytes := session.Internal.Pickle(store.PickleKey) + _, err := store.db.Exec("UPDATE crypto_megolm_outbound_session SET session=$1, message_count=$2, last_used=$3 WHERE room_id=$4 AND session_id=$5", + sessionBytes, session.MessageCount, session.UseTime, session.RoomID, session.ID()) + return err } func (store *SQLCryptoStore) GetOutboundGroupSession(roomID id.RoomID) (*crypto.OutboundGroupSession, error) { - store.OGSLock.RLock() - defer store.OGSLock.RUnlock() - return store.OutGroupSessions[roomID], nil + var ogs crypto.OutboundGroupSession + var sessionBytes []byte + err := store.db.QueryRow(` + SELECT session, shared, max_messages, message_count, max_age, created_at, last_used + FROM crypto_megolm_outbound_session WHERE room_id=$1`, + roomID, + ).Scan(&sessionBytes, &ogs.Shared, &ogs.MaxMessages, &ogs.MessageCount, &ogs.MaxAge, &ogs.CreationTime, &ogs.UseTime) + if err == sql.ErrNoRows { + return nil, nil + } else if err != nil { + return nil, err + } + intOGS := olm.NewBlankOutboundGroupSession() + err = intOGS.Unpickle(sessionBytes, store.PickleKey) + if err != nil { + return nil, err + } + ogs.Internal = *intOGS + ogs.RoomID = roomID + return &ogs, nil } -func (store *SQLCryptoStore) PopOutboundGroupSession(roomID id.RoomID) error { - store.OGSLock.Lock() - delete(store.OutGroupSessions, roomID) - store.OGSLock.Unlock() - return nil +func (store *SQLCryptoStore) RemoveOutboundGroupSession(roomID id.RoomID) error { + _, err := store.db.Exec("DELETE FROM crypto_megolm_outbound_session WHERE room_id=$1", roomID) + return err } func (store *SQLCryptoStore) ValidateMessageIndex(senderKey id.SenderKey, sessionID id.SessionID, eventID id.EventID, index uint, timestamp int64) bool { @@ -389,7 +405,7 @@ func (store *SQLCryptoStore) FilterTrackedUsers(users []id.UserID) []id.UserID { queryString[i] = fmt.Sprintf("$%d", i+1) params[i] = user } - rows, err = store.db.Query("SELECT user_id FROM crypto_tracked_user WHERE user_id IN (" + strings.Join(queryString, ",") + ")", params...) + rows, err = store.db.Query("SELECT user_id FROM crypto_tracked_user WHERE user_id IN ("+strings.Join(queryString, ",")+")", params...) } if err != nil { store.log.Warnln("Failed to filter tracked users:", err) diff --git a/database/upgrades/2020-05-12-outbound-group-session-store.go b/database/upgrades/2020-05-12-outbound-group-session-store.go new file mode 100644 index 0000000..2d635ef --- /dev/null +++ b/database/upgrades/2020-05-12-outbound-group-session-store.go @@ -0,0 +1,26 @@ +package upgrades + +import ( + "database/sql" +) + +func init() { + upgrades[14] = upgrade{"Add outbound group sessions to database", func(tx *sql.Tx, ctx context) error { + // TODO use DATETIME instead of timestamp and BLOB instead of bytea for sqlite + _, err := tx.Exec(`CREATE TABLE crypto_megolm_outbound_session ( + room_id VARCHAR(255) PRIMARY KEY, + session_id CHAR(43) NOT NULL UNIQUE, + session bytea NOT NULL, + shared BOOLEAN NOT NULL, + max_messages INTEGER NOT NULL, + message_count INTEGER NOT NULL, + max_age BIGINT NOT NULL, + created_at timestamp NOT NULL, + last_used timestamp NOT NULL + )`) + if err != nil { + return err + } + return nil + }} +} diff --git a/database/upgrades/upgrades.go b/database/upgrades/upgrades.go index ec8e6e7..22c8384 100644 --- a/database/upgrades/upgrades.go +++ b/database/upgrades/upgrades.go @@ -28,7 +28,7 @@ type upgrade struct { fn upgradeFunc } -const NumberOfUpgrades = 14 +const NumberOfUpgrades = 15 var upgrades [NumberOfUpgrades]upgrade diff --git a/go.mod b/go.mod index f8d6064..574c106 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( gopkg.in/yaml.v2 v2.2.8 maunium.net/go/mauflag v1.0.0 maunium.net/go/maulogger/v2 v2.1.1 - maunium.net/go/mautrix v0.4.3 + maunium.net/go/mautrix v0.4.4 ) replace github.com/Rhymen/go-whatsapp => github.com/tulir/go-whatsapp v0.2.6 diff --git a/go.sum b/go.sum index 9dbe6fa..4562111 100644 --- a/go.sum +++ b/go.sum @@ -92,3 +92,5 @@ maunium.net/go/mautrix v0.4.2 h1:GBU++Z7o/fLPcEsNMkNOUsnDknwV/MGPQ0BN4ikK6tw= maunium.net/go/mautrix v0.4.2/go.mod h1:8Y+NqmROJyWYvvP4yPfX9tLM59VCfgE/kcQ0SeX68ho= maunium.net/go/mautrix v0.4.3 h1:fVoJy992TjBEvuK5NeO9fpBh+9JuSFsxaEdGjFp/7h4= maunium.net/go/mautrix v0.4.3/go.mod h1:8Y+NqmROJyWYvvP4yPfX9tLM59VCfgE/kcQ0SeX68ho= +maunium.net/go/mautrix v0.4.4 h1:C5yYDzUdRtJj/9Vot5YBPQUsWmn19sTySew7f4ACLhM= +maunium.net/go/mautrix v0.4.4/go.mod h1:8Y+NqmROJyWYvvP4yPfX9tLM59VCfgE/kcQ0SeX68ho= From fc6f8df4d38a4874a443a7b8738eafe35c560aed Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 12 May 2020 23:33:59 +0300 Subject: [PATCH 14/22] Fix replacing outbound group session in db --- database/cryptostore.go | 22 ++++++++++++++++++---- go.mod | 2 +- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/database/cryptostore.go b/database/cryptostore.go index dbfca25..59cecd3 100644 --- a/database/cryptostore.go +++ b/database/cryptostore.go @@ -249,11 +249,25 @@ func (store *SQLCryptoStore) GetGroupSession(roomID id.RoomID, senderKey id.Send }, nil } -func (store *SQLCryptoStore) AddOutboundGroupSession(session *crypto.OutboundGroupSession) error { +func (store *SQLCryptoStore) AddOutboundGroupSession(session *crypto.OutboundGroupSession) (err error) { sessionBytes := session.Internal.Pickle(store.PickleKey) - _, err := store.db.Exec("INSERT INTO crypto_megolm_outbound_session (room_id, session_id, session, shared, max_messages, message_count, max_age, created_at, last_used) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", - session.RoomID, session.ID(), sessionBytes, session.Shared, session.MaxMessages, session.MessageCount, session.MaxAge, session.CreationTime, session.UseTime) - return err + if store.db.dialect == "postgres" { + _, err = store.db.Exec(` + INSERT INTO crypto_megolm_outbound_session ( + room_id, session_id, session, shared, max_messages, message_count, max_age, created_at, last_used + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (room_id) DO UPDATE SET session_id=$2, session=$3, shared=$4, max_messages=$5, message_count=$6, max_age=$7, created_at=$8, last_used=$9`, + session.RoomID, session.ID(), sessionBytes, session.Shared, session.MaxMessages, session.MessageCount, session.MaxAge, session.CreationTime, session.UseTime) + } else if store.db.dialect == "sqlite" { + _, err = store.db.Exec(` + INSERT OR REPLACE INTO crypto_megolm_outbound_session ( + room_id, session_id, session, shared, max_messages, message_count, max_age, created_at, last_used + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, + session.RoomID, session.ID(), sessionBytes, session.Shared, session.MaxMessages, session.MessageCount, session.MaxAge, session.CreationTime, session.UseTime) + } else { + err = fmt.Errorf("unsupported dialect %s", store.db.dialect) + } + return } func (store *SQLCryptoStore) UpdateOutboundGroupSession(session *crypto.OutboundGroupSession) error { diff --git a/go.mod b/go.mod index 574c106..4a5cfae 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( gopkg.in/yaml.v2 v2.2.8 maunium.net/go/mauflag v1.0.0 maunium.net/go/maulogger/v2 v2.1.1 - maunium.net/go/mautrix v0.4.4 + maunium.net/go/mautrix v0.4.5 ) replace github.com/Rhymen/go-whatsapp => github.com/tulir/go-whatsapp v0.2.6 From 562754bc777121865db268f1c63d0dc2aaf89eaf Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 12 May 2020 23:34:51 +0300 Subject: [PATCH 15/22] Update go.sum --- go.sum | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.sum b/go.sum index 4562111..f9b7366 100644 --- a/go.sum +++ b/go.sum @@ -94,3 +94,5 @@ maunium.net/go/mautrix v0.4.3 h1:fVoJy992TjBEvuK5NeO9fpBh+9JuSFsxaEdGjFp/7h4= maunium.net/go/mautrix v0.4.3/go.mod h1:8Y+NqmROJyWYvvP4yPfX9tLM59VCfgE/kcQ0SeX68ho= maunium.net/go/mautrix v0.4.4 h1:C5yYDzUdRtJj/9Vot5YBPQUsWmn19sTySew7f4ACLhM= maunium.net/go/mautrix v0.4.4/go.mod h1:8Y+NqmROJyWYvvP4yPfX9tLM59VCfgE/kcQ0SeX68ho= +maunium.net/go/mautrix v0.4.5 h1:cQhlPURW0TGjlqEoac+4+J/aS5/Rg8x1b+fiFZZz6LI= +maunium.net/go/mautrix v0.4.5/go.mod h1:8Y+NqmROJyWYvvP4yPfX9tLM59VCfgE/kcQ0SeX68ho= From a770263d86f65dc6c81572d402ae8a2c31a78c60 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 12 May 2020 23:38:01 +0300 Subject: [PATCH 16/22] Encrypt media bridging error notices --- portal.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/portal.go b/portal.go index 04b6c96..ccc1572 100644 --- a/portal.go +++ b/portal.go @@ -1001,7 +1001,10 @@ func (portal *Portal) HandleMediaMessage(source *User, download func() ([]byte, return } else if err != nil { portal.log.Errorfln("Failed to download media for %s: %v", info.Id, err) - resp, err := portal.MainIntent().SendNotice(portal.MXID, "Failed to bridge media") + resp, err := portal.sendMainIntentMessage(event.MessageEventContent{ + MsgType: event.MsgNotice, + Body: "Failed to bridge media", + }) if err != nil { portal.log.Errorfln("Failed to send media download error message for %s: %v", info.Id, err) } else { From e08676079afd2b2726edc1250890f4257749b390 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 20 May 2020 16:43:55 +0300 Subject: [PATCH 17/22] Fix bridging encrypted media from Matrix --- portal.go | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/portal.go b/portal.go index ccc1572..f453973 100644 --- a/portal.go +++ b/portal.go @@ -1002,8 +1002,8 @@ func (portal *Portal) HandleMediaMessage(source *User, download func() ([]byte, } else if err != nil { portal.log.Errorfln("Failed to download media for %s: %v", info.Id, err) resp, err := portal.sendMainIntentMessage(event.MessageEventContent{ - MsgType: event.MsgNotice, - Body: "Failed to bridge media", + MsgType: event.MsgNotice, + Body: "Failed to bridge media", }) if err != nil { portal.log.Errorfln("Failed to send media download error message for %s: %v", info.Id, err) @@ -1157,25 +1157,38 @@ func (portal *Portal) downloadThumbnail(content *event.MessageEventContent, id i return buf.Bytes() } -func (portal *Portal) preprocessMatrixMedia(sender *User, relaybotFormatted bool, content *event.MessageEventContent, id id.EventID, mediaType whatsapp.MediaType) *MediaUpload { +func (portal *Portal) preprocessMatrixMedia(sender *User, relaybotFormatted bool, content *event.MessageEventContent, eventID id.EventID, mediaType whatsapp.MediaType) *MediaUpload { var caption string if relaybotFormatted { caption = portal.bridge.Formatter.ParseMatrix(content.FormattedBody) } - mxc, err := content.URL.Parse() + var file *event.EncryptedFileInfo + rawMXC := content.URL + if content.File != nil { + file = content.File + rawMXC = file.URL + } + mxc, err := rawMXC.Parse() if err != nil { - portal.log.Errorln("Malformed content URL in %s: %v", id, err) + portal.log.Errorln("Malformed content URL in %s: %v", eventID, err) } data, err := portal.MainIntent().DownloadBytes(mxc) if err != nil { - portal.log.Errorfln("Failed to download media in %s: %v", id, err) + portal.log.Errorfln("Failed to download media in %s: %v", eventID, err) return nil } + if file != nil { + data, err = file.Decrypt(data) + if err != nil { + portal.log.Errorfln("Failed to decrypt media in %s: %v", eventID, err) + return nil + } + } url, mediaKey, fileEncSHA256, fileSHA256, fileLength, err := sender.Conn.Upload(bytes.NewReader(data), mediaType) if err != nil { - portal.log.Errorfln("Failed to upload media in %s: %v", id, err) + portal.log.Errorfln("Failed to upload media in %s: %v", eventID, err) return nil } @@ -1186,7 +1199,7 @@ func (portal *Portal) preprocessMatrixMedia(sender *User, relaybotFormatted bool FileEncSHA256: fileEncSHA256, FileSHA256: fileSHA256, FileLength: fileLength, - Thumbnail: portal.downloadThumbnail(content, id), + Thumbnail: portal.downloadThumbnail(content, eventID), } } From 9002bf62ed0be68462588236625ba32b5c9aa2a0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 21 May 2020 19:49:01 +0300 Subject: [PATCH 18/22] Fix needing to reconnect after initial login --- commands.go | 10 ++++++++++ provisioning.go | 4 ++++ user.go | 15 +++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/commands.go b/commands.go index 3894f0f..f9b06cb 100644 --- a/commands.go +++ b/commands.go @@ -230,6 +230,13 @@ func (handler *CommandHandler) CommandLogout(ce *CommandEvent) { ce.Reply("You're not logged in.") return } + puppet := handler.bridge.GetPuppetByJID(ce.User.JID) + if puppet.CustomMXID != "" { + err := puppet.SwitchCustomMXID("", "") + if err != nil { + ce.User.log.Warnln("Failed to logout-matrix while logging out of WhatsApp:", err) + } + } err := ce.User.Conn.Logout() if err != nil { ce.User.log.Warnln("Error while logging out:", err) @@ -242,6 +249,9 @@ func (handler *CommandHandler) CommandLogout(ce *CommandEvent) { } ce.User.Conn.RemoveHandlers() ce.User.Conn = nil + ce.User.removeFromJIDMap() + // TODO this causes a foreign key violation, which should be fixed + //ce.User.JID = "" ce.User.SetSession(nil) ce.Reply("Logged out successfully.") } diff --git a/provisioning.go b/provisioning.go index a08560d..287d08d 100644 --- a/provisioning.go +++ b/provisioning.go @@ -292,6 +292,9 @@ func (prov *ProvisioningAPI) Logout(w http.ResponseWriter, r *http.Request) { } user.Conn.RemoveHandlers() user.Conn = nil + user.removeFromJIDMap() + // TODO this causes a foreign key violation, which should be fixed + //ce.User.JID = "" user.SetSession(nil) jsonResponse(w, http.StatusOK, Response{true, "Logged out successfully."}) } @@ -351,6 +354,7 @@ func (prov *ProvisioningAPI) Login(w http.ResponseWriter, r *http.Request) { } user.ConnectionErrors = 0 user.JID = strings.Replace(user.Conn.Info.Wid, whatsappExt.OldUserSuffix, whatsappExt.NewUserSuffix, 1) + user.addToJIDMap() user.SetSession(&session) _ = c.WriteJSON(map[string]interface{}{ "success": true, diff --git a/user.go b/user.go index 857e50b..d948374 100644 --- a/user.go +++ b/user.go @@ -90,6 +90,18 @@ func (bridge *Bridge) GetUserByJID(userID types.WhatsAppID) *User { return user } +func (user *User) addToJIDMap() { + user.bridge.usersLock.Lock() + user.bridge.usersByJID[user.JID] = user + user.bridge.usersLock.Unlock() +} + +func (user *User) removeFromJIDMap() { + user.bridge.usersLock.Lock() + delete(user.bridge.usersByJID, user.JID) + user.bridge.usersLock.Unlock() +} + func (bridge *Bridge) GetAllUsers() []*User { bridge.usersLock.Lock() defer bridge.usersLock.Unlock() @@ -332,8 +344,11 @@ func (user *User) Login(ce *CommandEvent) { _, _ = ce.Bot.SendMessageEvent(ce.RoomID, event.EventMessage, &msg) return } + // TODO there's a bit of duplication between this and the provisioning API login method + // Also between the two logout methods (commands.go and provisioning.go) user.ConnectionErrors = 0 user.JID = strings.Replace(user.Conn.Info.Wid, whatsappExt.OldUserSuffix, whatsappExt.NewUserSuffix, 1) + user.addToJIDMap() user.SetSession(&session) ce.Reply("Successfully logged in, synchronizing chats...") user.PostLogin() From c0fc06b410cd927120e520641918156ccdd7d2d9 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 21 May 2020 20:00:00 +0300 Subject: [PATCH 19/22] Allow using custom puppet for backfilling --- puppet.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/puppet.go b/puppet.go index 4831c3d..9e496a4 100644 --- a/puppet.go +++ b/puppet.go @@ -161,7 +161,9 @@ func (puppet *Puppet) PhoneNumber() string { } func (puppet *Puppet) IntentFor(portal *Portal) *appservice.IntentAPI { - if (!portal.IsPrivateChat() && puppet.customIntent == nil) || portal.backfilling || portal.Key.JID == puppet.JID { + if (!portal.IsPrivateChat() && puppet.customIntent == nil) || + (portal.backfilling && portal.bridge.Config.Bridge.InviteOwnPuppetForBackfilling) || + portal.Key.JID == puppet.JID { return puppet.DefaultIntent() } return puppet.customIntent From e4bc254a65f57587dc0ee991fdf1e3033ac37ca1 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 21 May 2020 20:14:43 +0300 Subject: [PATCH 20/22] Change incoming call notices back to m.text --- portal.go | 9 ++++++++- user.go | 7 +++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/portal.go b/portal.go index f453973..c6239c3 100644 --- a/portal.go +++ b/portal.go @@ -919,7 +919,14 @@ func (portal *Portal) HandleFakeMessage(source *User, message FakeMessage) { return } - _, err := portal.MainIntent().SendNotice(portal.MXID, message.Text) + content := event.MessageEventContent{ + MsgType: event.MsgNotice, + Body: message.Text, + } + if message.Alert { + content.MsgType = event.MsgText + } + _, err := portal.sendMainIntentMessage(content) if err != nil { portal.log.Errorfln("Failed to handle fake message %s: %v", message.ID, err) return diff --git a/user.go b/user.go index d948374..c808ded 100644 --- a/user.go +++ b/user.go @@ -675,8 +675,9 @@ func (user *User) HandleMessageRevoke(message whatsappExt.MessageRevocation) { } type FakeMessage struct { - Text string - ID string + Text string + ID string + Alert bool } func (user *User) HandleCallInfo(info whatsappExt.CallInfo) { @@ -692,11 +693,13 @@ func (user *User) HandleCallInfo(info whatsappExt.CallInfo) { return } data.Text = "Incoming call" + data.Alert = true case whatsappExt.CallOfferVideo: if !user.bridge.Config.Bridge.CallNotices.Start { return } data.Text = "Incoming video call" + data.Alert = true case whatsappExt.CallTerminate: if !user.bridge.Config.Bridge.CallNotices.End { return From 64af0209b75bf0793446cf9d32fba8b140ba9150 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 21 May 2020 20:50:54 +0300 Subject: [PATCH 21/22] Suggest delete-session when trying to logout without connection. Fixes #98 --- commands.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/commands.go b/commands.go index f9b06cb..b7d51c3 100644 --- a/commands.go +++ b/commands.go @@ -121,7 +121,9 @@ func (handler *CommandHandler) CommandMux(ce *CommandEvent) { handler.CommandDevTest(ce) case "set-pl": handler.CommandSetPowerLevel(ce) - case "login-matrix", "logout", "sync", "list", "open", "pm": + case "logout": + handler.CommandLogout(ce) + case "login-matrix", "sync", "list", "open", "pm": if !ce.User.HasSession() { ce.Reply("You are not logged in. Use the `login` command to log into WhatsApp.") return @@ -133,8 +135,6 @@ func (handler *CommandHandler) CommandMux(ce *CommandEvent) { switch ce.Command { case "login-matrix": handler.CommandLoginMatrix(ce) - case "logout": - handler.CommandLogout(ce) case "sync": handler.CommandSync(ce) case "list": @@ -229,6 +229,9 @@ func (handler *CommandHandler) CommandLogout(ce *CommandEvent) { if ce.User.Session == nil { ce.Reply("You're not logged in.") return + } else if !ce.User.IsConnected() { + ce.Reply("You are not connected to WhatsApp. Use the `reconnect` command to reconnect, or `delete-session` to forget all login information.") + return } puppet := handler.bridge.GetPuppetByJID(ce.User.JID) if puppet.CustomMXID != "" { From ed978bcb9c353a7e73a6c1e6ae04b6facc8b9b54 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 21 May 2020 20:56:41 +0300 Subject: [PATCH 22/22] Don't use different types for SQLite in DB schema --- database/cryptostore.go | 2 +- database/migrate.go | 4 +++ .../upgrades/2018-09-01-initial-schema.go | 25 +++++++------------ database/upgrades/2020-05-09-crypto-store.go | 1 - ...2020-05-12-outbound-group-session-store.go | 1 - 5 files changed, 14 insertions(+), 19 deletions(-) diff --git a/database/cryptostore.go b/database/cryptostore.go index 59cecd3..7e04bc3 100644 --- a/database/cryptostore.go +++ b/database/cryptostore.go @@ -258,7 +258,7 @@ func (store *SQLCryptoStore) AddOutboundGroupSession(session *crypto.OutboundGro ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (room_id) DO UPDATE SET session_id=$2, session=$3, shared=$4, max_messages=$5, message_count=$6, max_age=$7, created_at=$8, last_used=$9`, session.RoomID, session.ID(), sessionBytes, session.Shared, session.MaxMessages, session.MessageCount, session.MaxAge, session.CreationTime, session.UseTime) - } else if store.db.dialect == "sqlite" { + } else if store.db.dialect == "sqlite3" { _, err = store.db.Exec(` INSERT OR REPLACE INTO crypto_megolm_outbound_session ( room_id, session_id, session, shared, max_messages, message_count, max_age, created_at, last_used diff --git a/database/migrate.go b/database/migrate.go index 9d30871..a13c6ca 100644 --- a/database/migrate.go +++ b/database/migrate.go @@ -145,4 +145,8 @@ func Migrate(old *Database, new *Database) { if err != nil { panic(err) } + err = migrateTable(old, new, "crypto_megolm_outbound_session", "room_id", "session_id", "session", "shared", "max_messages", "message_count", "max_age", "created_at", "last_used") + if err != nil { + panic(err) + } } diff --git a/database/upgrades/2018-09-01-initial-schema.go b/database/upgrades/2018-09-01-initial-schema.go index b1b8709..f142726 100644 --- a/database/upgrades/2018-09-01-initial-schema.go +++ b/database/upgrades/2018-09-01-initial-schema.go @@ -2,26 +2,19 @@ package upgrades import ( "database/sql" - "fmt" ) func init() { upgrades[0] = upgrade{"Initial schema", func(tx *sql.Tx, ctx context) error { - var byteType string - if ctx.dialect == SQLite { - byteType = "BLOB" - } else { - byteType = "bytea" - } _, err := tx.Exec(`CREATE TABLE IF NOT EXISTS portal ( jid VARCHAR(255), receiver VARCHAR(255), mxid VARCHAR(255) UNIQUE, - + name VARCHAR(255) NOT NULL, topic VARCHAR(255) NOT NULL, avatar VARCHAR(255) NOT NULL, - + PRIMARY KEY (jid, receiver) )`) if err != nil { @@ -38,7 +31,7 @@ func init() { return err } - _, err = tx.Exec(fmt.Sprintf(`CREATE TABLE IF NOT EXISTS "user" ( + _, err = tx.Exec(`CREATE TABLE IF NOT EXISTS "user" ( mxid VARCHAR(255) PRIMARY KEY, jid VARCHAR(255) UNIQUE, @@ -47,24 +40,24 @@ func init() { client_id VARCHAR(255), client_token VARCHAR(255), server_token VARCHAR(255), - enc_key %[1]s, - mac_key %[1]s - )`, byteType)) + enc_key bytea, + mac_key bytea + )`) if err != nil { return err } - _, err = tx.Exec(fmt.Sprintf(`CREATE TABLE IF NOT EXISTS message ( + _, err = tx.Exec(`CREATE TABLE IF NOT EXISTS message ( chat_jid VARCHAR(255), chat_receiver VARCHAR(255), jid VARCHAR(255), mxid VARCHAR(255) NOT NULL UNIQUE, sender VARCHAR(255) NOT NULL, - content %[1]s NOT NULL, + content bytea NOT NULL, PRIMARY KEY (chat_jid, chat_receiver, jid), FOREIGN KEY (chat_jid, chat_receiver) REFERENCES portal(jid, receiver) ON DELETE CASCADE - )`, byteType)) + )`) if err != nil { return err } diff --git a/database/upgrades/2020-05-09-crypto-store.go b/database/upgrades/2020-05-09-crypto-store.go index fd479b0..8be6cd8 100644 --- a/database/upgrades/2020-05-09-crypto-store.go +++ b/database/upgrades/2020-05-09-crypto-store.go @@ -6,7 +6,6 @@ import ( func init() { upgrades[13] = upgrade{"Add crypto store to database", func(tx *sql.Tx, ctx context) error { - // TODO use DATETIME instead of timestamp and BLOB instead of bytea for sqlite _, err := tx.Exec(`CREATE TABLE crypto_account ( device_id VARCHAR(255) PRIMARY KEY, shared BOOLEAN NOT NULL, diff --git a/database/upgrades/2020-05-12-outbound-group-session-store.go b/database/upgrades/2020-05-12-outbound-group-session-store.go index 2d635ef..0f108a6 100644 --- a/database/upgrades/2020-05-12-outbound-group-session-store.go +++ b/database/upgrades/2020-05-12-outbound-group-session-store.go @@ -6,7 +6,6 @@ import ( func init() { upgrades[14] = upgrade{"Add outbound group sessions to database", func(tx *sql.Tx, ctx context) error { - // TODO use DATETIME instead of timestamp and BLOB instead of bytea for sqlite _, err := tx.Exec(`CREATE TABLE crypto_megolm_outbound_session ( room_id VARCHAR(255) PRIMARY KEY, session_id CHAR(43) NOT NULL UNIQUE,