Re-break everything and fix Matrix->WhatsApp replies

Gopkg.lock

@@ -101,7 +101,7 @@
   branch = "master"
   name = "golang.org/x/sys"
   packages = ["unix"]
-  revision = "49385e6e15226593f68b26af201feec29d5bba22"
+  revision = "fa5fdf94c78965f1aa8423f0cc50b8b8d728b05a"
 
 [[projects]]
   name = "gopkg.in/russross/blackfriday.v2"
@@ -140,7 +140,7 @@
   branch = "master"
   name = "maunium.net/go/mautrix-appservice"
   packages = ["."]
-  revision = "fb756247f82716de7698b8200f28f16b4fd04a6b"
+  revision = "4e24d1dd7bd9d89f946ec56cb4350ce777d17bfe"
 
 [solve-meta]
   analyzer-name = "dep"

config/bridge.go

@@ -60,13 +60,24 @@ type UsernameTemplateArgs struct {
 	UserID   string
 }
 
-func (bc BridgeConfig) FormatDisplayname(contact whatsapp.Contact) string {
+func (bc BridgeConfig) FormatDisplayname(contact whatsapp.Contact) (string, int8) {
 	var buf bytes.Buffer
 	if index := strings.IndexRune(contact.Jid, '@'); index > 0 {
 		contact.Jid = "+" + contact.Jid[:index]
 	}
 	bc.displaynameTemplate.Execute(&buf, contact)
-	return buf.String()
+	var quality int8
+	switch {
+	case len(contact.Notify) > 0:
+		quality = 3
+	case len(contact.Name) > 0 || len(contact.Short) > 0:
+		quality = 2
+	case len(contact.Jid) > 0:
+		quality = 1
+	default:
+		quality = 0
+	}
+	return buf.String(), quality
 }
 
 func (bc BridgeConfig) FormatUsername(userID types.WhatsAppID) string {
@@ -76,7 +87,7 @@ func (bc BridgeConfig) FormatUsername(userID types.WhatsAppID) string {
 }
 
 func (bc BridgeConfig) MarshalYAML() (interface{}, error) {
-	bc.DisplaynameTemplate = bc.FormatDisplayname(whatsapp.Contact{
+	bc.DisplaynameTemplate, _ = bc.FormatDisplayname(whatsapp.Contact{
 		Jid:    "{{.Jid}}",
 		Notify: "{{.Notify}}",
 		Name:   "{{.Name}}",

config/registration.go

@@ -24,7 +24,7 @@ import (
 )
 
 func (config *Config) NewRegistration() (*appservice.Registration, error) {
-	registration := appservice.CreateRegistration("mautrix-whatsapp")
+	registration := appservice.CreateRegistration()
 
 	err := config.copyToRegistration(registration)
 	if err != nil {
@@ -37,7 +37,7 @@ func (config *Config) NewRegistration() (*appservice.Registration, error) {
 }
 
 func (config *Config) GetRegistration() (*appservice.Registration, error) {
-	registration := appservice.CreateRegistration("mautrix-whatsapp")
+	registration := appservice.CreateRegistration()
 
 	err := config.copyToRegistration(registration)
 	if err != nil {

database/message.go

@@ -17,8 +17,11 @@
 package database
 
 import (
+	"bytes"
 	"database/sql"
+	"encoding/json"
 
+	waProto "github.com/Rhymen/go-whatsapp/binary/proto"
 	log "maunium.net/go/maulogger"
 	"maunium.net/go/mautrix-whatsapp/types"
 )
@@ -32,8 +35,10 @@ func (mq *MessageQuery) CreateTable() error {
 	_, err := mq.db.Exec(`CREATE TABLE IF NOT EXISTS message (
 		chat_jid      VARCHAR(25),
 		chat_receiver VARCHAR(25),
-		jid  VARCHAR(255),
+		jid           VARCHAR(255),
-		mxid VARCHAR(255) NOT NULL UNIQUE,
+		mxid          VARCHAR(255) NOT NULL UNIQUE,
+		sender        VARCHAR(25)  NOT NULL,
+		content       BLOB         NOT NULL,
 
 		PRIMARY KEY (chat_jid, chat_receiver, jid),
 		FOREIGN KEY (chat_jid, chat_receiver) REFERENCES portal(jid, receiver)
@@ -80,34 +85,54 @@ type Message struct {
 	db  *Database
 	log log.Logger
 
-	Chat PortalKey
+	Chat    PortalKey
 	JID     types.WhatsAppMessageID
 	MXID    types.MatrixEventID
+	Sender  types.WhatsAppID
+	Content *waProto.Message
 }
 
 func (msg *Message) Scan(row Scannable) *Message {
-	err := row.Scan(&msg.Chat.JID, &msg.Chat.Receiver, &msg.JID, &msg.MXID)
+	var content []byte
+	err := row.Scan(&msg.Chat.JID, &msg.Chat.Receiver, &msg.JID, &msg.MXID, &msg.Sender, &content)
 	if err != nil {
 		if err != sql.ErrNoRows {
 			msg.log.Errorln("Database scan failed:", err)
 		}
 		return nil
 	}
+
+	msg.parseBinaryContent(content)
+
 	return msg
 }
 
-func (msg *Message) Insert() error {
+func (msg *Message) parseBinaryContent(content []byte) {
-	_, err := msg.db.Exec("INSERT INTO message VALUES (?, ?, ?, ?)", msg.Chat.JID, msg.Chat.Receiver, msg.JID, msg.MXID)
+	msg.Content = &waProto.Message{}
+	reader := bytes.NewReader(content)
+	// dec := gob.NewDecoder(reader)
+	dec := json.NewDecoder(reader)
+	err := dec.Decode(msg.Content)
 	if err != nil {
-		msg.log.Warnfln("Failed to insert %s: %v", msg.Chat, msg.JID, err)
+		msg.log.Warnln("Failed to decode message content:", err)
 	}
-	return err
 }
 
-func (msg *Message) Update() error {
+func (msg *Message) binaryContent() []byte {
-	_, err := msg.db.Exec("UPDATE portal SET mxid=? WHERE chat_jid=? AND chat_receiver=? AND jid=?", msg.MXID, msg.Chat.JID, msg.Chat.Receiver, msg.JID)
+	var buf bytes.Buffer
+	//enc := gob.NewEncoder(&buf)
+	enc := json.NewEncoder(&buf)
+	err := enc.Encode(msg.Content)
 	if err != nil {
-		msg.log.Warnfln("Failed to update %s: %v", msg.Chat, msg.JID, err)
+		msg.log.Warnln("Failed to encode message content:", err)
+	}
+	return buf.Bytes()
+}
+
+func (msg *Message) Insert() error {
+	_, err := msg.db.Exec("INSERT INTO message VALUES (?, ?, ?, ?, ?, ?)", msg.Chat.JID, msg.Chat.Receiver, msg.JID, msg.MXID, msg.Sender, msg.binaryContent())
+	if err != nil {
+		msg.log.Warnfln("Failed to insert %s@%s: %v", msg.Chat, msg.JID, err)
 	}
 	return err
 }

database/puppet.go

@@ -30,9 +30,10 @@ type PuppetQuery struct {
 
 func (pq *PuppetQuery) CreateTable() error {
 	_, err := pq.db.Exec(`CREATE TABLE IF NOT EXISTS puppet (
-		jid         VARCHAR(25) PRIMARY KEY,
+		jid          VARCHAR(25) PRIMARY KEY,
-		displayname VARCHAR(255),
+		avatar       VARCHAR(255),
-		avatar      VARCHAR(255)
+		displayname  VARCHAR(255),
+		name_quality TINYINT
 	)`)
 	return err
 }
@@ -69,8 +70,9 @@ type Puppet struct {
 	log log.Logger
 
 	JID         types.WhatsAppID
-	Displayname string
 	Avatar      string
+	Displayname string
+	NameQuality int8
 }
 
 func (puppet *Puppet) Scan(row Scannable) *Puppet {
@@ -88,19 +90,19 @@ func (puppet *Puppet) Scan(row Scannable) *Puppet {
 }
 
 func (puppet *Puppet) Insert() error {
-	_, err := puppet.db.Exec("INSERT INTO puppet VALUES (?, ?, ?)",
+	_, err := puppet.db.Exec("INSERT INTO puppet VALUES (?, ?, ?, ?)",
-		puppet.JID, puppet.Displayname, puppet.Avatar)
+		puppet.JID, puppet.Avatar, puppet.Displayname, puppet.NameQuality)
 	if err != nil {
-		puppet.log.Errorfln("Failed to insert %s: %v", puppet.JID, err)
+		puppet.log.Warnfln("Failed to insert %s: %v", puppet.JID, err)
 	}
 	return err
 }
 
 func (puppet *Puppet) Update() error {
-	_, err := puppet.db.Exec("UPDATE puppet SET displayname=?, avatar=? WHERE jid=?",
+	_, err := puppet.db.Exec("UPDATE puppet SET displayname=?, name_quality=?, avatar=? WHERE jid=?",
-		puppet.Displayname, puppet.Avatar, puppet.JID)
+		puppet.Displayname, puppet.NameQuality, puppet.Avatar, puppet.JID)
 	if err != nil {
-		puppet.log.Errorfln("Failed to update %s->%s: %v", puppet.JID, err)
+		puppet.log.Warnfln("Failed to update %s->%s: %v", puppet.JID, err)
 	}
 	return err
 }

database/user.go

@@ -131,6 +131,14 @@ func stripSuffix(jid types.WhatsAppID) string {
 	return jid[:index]
 }
 
+func (user *User) jidPtr() *string {
+	if len(user.JID) > 0 {
+		str := stripSuffix(user.JID)
+		return &str
+	}
+	return nil
+}
+
 func (user *User) sessionUnptr() (sess whatsapp.Session) {
 	if user.Session != nil {
 		sess = *user.Session
@@ -140,17 +148,23 @@ func (user *User) sessionUnptr() (sess whatsapp.Session) {
 
 func (user *User) Insert() error {
 	sess := user.sessionUnptr()
-	_, err := user.db.Exec("INSERT INTO user VALUES (?, ?, ?, ?, ?, ?, ?, ?)", user.MXID, stripSuffix(user.JID),
+	_, err := user.db.Exec("INSERT INTO user VALUES (?, ?, ?, ?, ?, ?, ?, ?)", user.MXID, user.jidPtr(),
 		user.ManagementRoom,
 		sess.ClientId, sess.ClientToken, sess.ServerToken, sess.EncKey, sess.MacKey)
+	if err != nil {
+		user.log.Warnfln("Failed to insert %s: %v", user.MXID, err)
+	}
 	return err
 }
 
 func (user *User) Update() error {
 	sess := user.sessionUnptr()
 	_, err := user.db.Exec("UPDATE user SET jid=?, management_room=?, client_id=?, client_token=?, server_token=?, enc_key=?, mac_key=? WHERE mxid=?",
-		stripSuffix(user.JID), user.ManagementRoom,
+		user.jidPtr(), user.ManagementRoom,
 		sess.ClientId, sess.ClientToken, sess.ServerToken, sess.EncKey, sess.MacKey,
 		user.MXID)
+	if err != nil {
+		user.log.Warnfln("Failed to update %s: %v", user.MXID, err)
+	}
 	return err
 }

matrix.go

@@ -139,6 +139,10 @@ func (mx *MatrixHandler) HandleRoomMetadata(evt *gomatrix.Event) {
 }
 
 func (mx *MatrixHandler) HandleMessage(evt *gomatrix.Event) {
+	if _, isPuppet := mx.bridge.ParsePuppetMXID(evt.Sender); evt.Sender == mx.bridge.Bot.UserID || isPuppet {
+		return
+	}
+
 	roomID := types.MatrixRoomID(evt.RoomID)
 	user := mx.bridge.GetUserByMXID(types.MatrixUserID(evt.Sender))
 

portal.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/gob"
 	"encoding/hex"
 	"fmt"
 	"image"
@@ -121,6 +122,8 @@ type Portal struct {
 	recentlyHandled      [20]types.WhatsAppMessageID
 	recentlyHandledLock  sync.Mutex
 	recentlyHandledIndex uint8
+
+	isPrivate *bool
 }
 
 func (portal *Portal) getMessageLock(messageID types.WhatsAppMessageID) sync.Mutex {
@@ -157,18 +160,33 @@ func (portal *Portal) isDuplicate(id types.WhatsAppMessageID) bool {
 	return false
 }
 
-func (portal *Portal) markHandled(jid types.WhatsAppMessageID, mxid types.MatrixEventID) {
+func init() {
+	gob.Register(&waProto.Message{})
+}
+
+func (portal *Portal) markHandled(source *User, message *waProto.WebMessageInfo, mxid types.MatrixEventID) {
 	msg := portal.bridge.DB.Message.New()
 	msg.Chat = portal.Key
-	msg.JID = jid
+	msg.JID = message.GetKey().GetId()
 	msg.MXID = mxid
+	if message.GetKey().GetFromMe() {
+		msg.Sender = source.JID
+	} else if portal.IsPrivateChat() {
+		msg.Sender = portal.Key.JID
+	} else {
+		msg.Sender = message.GetKey().GetParticipant()
+		if len(msg.Sender) == 0 {
+			msg.Sender = message.GetParticipant()
+		}
+	}
+	msg.Content = message.Message
 	msg.Insert()
 
 	portal.recentlyHandledLock.Lock()
 	index := portal.recentlyHandledIndex
 	portal.recentlyHandledIndex = (portal.recentlyHandledIndex + 1) % 20
 	portal.recentlyHandledLock.Unlock()
-	portal.recentlyHandled[index] = jid
+	portal.recentlyHandled[index] = msg.JID
 }
 
 func (portal *Portal) startHandling(id types.WhatsAppMessageID) (*sync.Mutex, bool) {
@@ -184,8 +202,9 @@ func (portal *Portal) startHandling(id types.WhatsAppMessageID) (*sync.Mutex, bo
 	return &lock, true
 }
 
-func (portal *Portal) finishHandling(id types.WhatsAppMessageID, mxid types.MatrixEventID) {
+func (portal *Portal) finishHandling(source *User, message *waProto.WebMessageInfo, mxid types.MatrixEventID) {
-	portal.markHandled(id, mxid)
+	portal.markHandled(source, message, mxid)
+	id := message.GetKey().GetId()
 	portal.deleteMessageLock(id)
 	portal.log.Debugln("Handled message", id, "->", mxid)
 }
@@ -450,7 +469,11 @@ func (portal *Portal) CreateMatrixRoom(invite []string) error {
 }
 
 func (portal *Portal) IsPrivateChat() bool {
-	return strings.HasSuffix(portal.Key.JID, whatsappExt.NewUserSuffix)
+	if portal.isPrivate == nil {
+		val := strings.HasSuffix(portal.Key.JID, whatsappExt.NewUserSuffix)
+		portal.isPrivate = &val
+	}
+	return *portal.isPrivate
 }
 
 func (portal *Portal) MainIntent() *appservice.IntentAPI {
@@ -527,7 +550,7 @@ func (portal *Portal) HandleTextMessage(source *User, message whatsapp.TextMessa
 		portal.log.Errorfln("Failed to handle message %s: %v", message.Info.Id, err)
 		return
 	}
-	portal.finishHandling(message.Info.Id, resp.EventID)
+	portal.finishHandling(source, message.Info.Source, resp.EventID)
 }
 
 func (portal *Portal) HandleMediaMessage(source *User, download func() ([]byte, error), thumbnail []byte, info whatsapp.MessageInfo, mimeType, caption string) {
@@ -628,7 +651,7 @@ func (portal *Portal) HandleMediaMessage(source *User, download func() ([]byte,
 		// TODO store caption mxid?
 	}
 
-	portal.finishHandling(info.Id, resp.EventID)
+	portal.finishHandling(source, info.Source, resp.EventID)
 }
 
 func makeMessageID() *string {
@@ -716,31 +739,6 @@ type MediaUpload struct {
 	Thumbnail     []byte
 }
 
-func (portal *Portal) GetMessage(user *User, jid types.WhatsAppMessageID) *waProto.WebMessageInfo {
-	node, err := user.Conn.LoadMessagesBefore(portal.Key.JID, jid, 1)
-	if err != nil {
-		return nil
-	}
-	msgs, ok := node.Content.([]interface{})
-	if !ok {
-		return nil
-	}
-	msg, ok := msgs[0].(*waProto.WebMessageInfo)
-	if !ok {
-		return nil
-	}
-	node, err = user.Conn.LoadMessagesAfter(portal.Key.JID, msg.GetKey().GetId(), 1)
-	if err != nil {
-		return nil
-	}
-	msgs, ok = node.Content.([]interface{})
-	if !ok {
-		return nil
-	}
-	msg, _ = msgs[0].(*waProto.WebMessageInfo)
-	return msg
-}
-
 func (portal *Portal) HandleMatrixMessage(sender *User, evt *gomatrix.Event) {
 	if portal.IsPrivateChat() && sender.JID != portal.Key.Receiver {
 		return
@@ -764,17 +762,10 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *gomatrix.Event) {
 	if len(replyToID) > 0 {
 		evt.Content.RemoveReplyFallback()
 		msg := portal.bridge.DB.Message.GetByMXID(replyToID)
-		if msg != nil {
+		if msg != nil && msg.Content != nil {
-			origMsg := portal.GetMessage(sender, msg.JID)
+			ctxInfo.StanzaId = &msg.JID
-			if origMsg != nil {
+			ctxInfo.Participant = &msg.Sender
-				ctxInfo.StanzaId = &msg.JID
+			ctxInfo.QuotedMessage = []*waProto.Message{msg.Content}
-				replyMsgSender := origMsg.GetParticipant()
-				if origMsg.GetKey().GetFromMe() {
-					replyMsgSender = sender.JID
-				}
-				ctxInfo.Participant = &replyMsgSender
-				ctxInfo.QuotedMessage = []*waProto.Message{origMsg.Message}
-			}
 		}
 	}
 	var err error
@@ -863,7 +854,7 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *gomatrix.Event) {
 		portal.log.Debugln("Unhandled Matrix event:", evt)
 		return
 	}
-	portal.markHandled(info.GetKey().GetId(), evt.ID)
+	portal.markHandled(sender, info, evt.ID)
 	err = sender.Conn.Send(info)
 	if err != nil {
 		portal.log.Errorfln("Error handling Matrix event %s: %v", evt.ID, err)

puppet.go

@@ -43,7 +43,7 @@ func (bridge *Bridge) ParsePuppetMXID(mxid types.MatrixUserID) (types.WhatsAppID
 		return "", false
 	}
 
-	jid := types.WhatsAppID(match[2] + whatsappExt.NewUserSuffix)
+	jid := types.WhatsAppID(match[1] + whatsappExt.NewUserSuffix)
 	return jid, true
 }
 
@@ -168,11 +168,12 @@ func (puppet *Puppet) Sync(source *User, contact whatsapp.Contact) {
 	if contact.Jid == source.JID {
 		contact.Notify = source.Conn.Info.Pushname
 	}
-	newName := puppet.bridge.Config.Bridge.FormatDisplayname(contact)
+	newName, quality := puppet.bridge.Config.Bridge.FormatDisplayname(contact)
-	if puppet.Displayname != newName {
+	if puppet.Displayname != newName && quality >= puppet.NameQuality {
 		err := puppet.Intent().SetDisplayName(newName)
 		if err == nil {
 			puppet.Displayname = newName
+			puppet.NameQuality = quality
 			puppet.Update()
 		} else {
 			puppet.log.Warnln("Failed to set display name:", err)

vendor/maunium.net/go/mautrix-appservice/generator.go

@@ -44,14 +44,16 @@ func GenerateRegistration(asName, botName string, reserveRooms, reserveUsers boo
 	boldCyan.Println("Generating appservice config and registration.")
 	reader := bufio.NewReader(os.Stdin)
 
+	registration := CreateRegistration()
+	config := Create()
+	registration.RateLimited = false
+
 	name, err := readString(reader, "Enter name for appservice", asName)
 	if err != nil {
 		fmt.Println("Failed to read user Input:", err)
 		return
 	}
-	registration := CreateRegistration(name)
+	registration.ID = name
-	config := Create()
-	registration.RateLimited = false
 
 	registration.SenderLocalpart, err = readString(reader, "Enter bot username", botName)
 	if err != nil {

vendor/maunium.net/go/mautrix-appservice/http.go

@@ -1,11 +1,11 @@
 package appservice
 
 import (
+	"context"
 	"encoding/json"
+	"github.com/gorilla/mux"
 	"io/ioutil"
 	"net/http"
-	"github.com/gorilla/mux"
-	"context"
 	"time"
 )
 
@@ -106,7 +106,6 @@ func (as *AppService) PutTransaction(w http.ResponseWriter, r *http.Request) {
 	}
 
 	for _, event := range eventList.Events {
-		as.Log.Debugln("Received event", event.ID)
 		as.UpdateState(event)
 		as.Events <- event
 	}

vendor/maunium.net/go/mautrix-appservice/protocol.go

@@ -2,8 +2,8 @@ package appservice
 
 import (
 	"encoding/json"
-	"net/http"
 	"maunium.net/go/gomatrix"
+	"net/http"
 )
 
 // EventList contains a list of events.

vendor/maunium.net/go/mautrix-appservice/registration.go

@@ -21,7 +21,7 @@ type Registration struct {
 }
 
 // CreateRegistration creates a Registration with random appservice and homeserver tokens.
-func CreateRegistration(name string) *Registration {
+func CreateRegistration() *Registration {
 	return &Registration{
 		AppToken:    RandomString(64),
 		ServerToken: RandomString(64),