Add basic newsletter support. Fixes #655

commands.go

@@ -1011,10 +1011,10 @@ func fnOpen(ce *WrappedCommandEvent) {
 	ce.Log.Debugln("Importing", jid, "for", ce.User.MXID)
 	portal := ce.User.GetPortalByJID(info.JID)
 	if len(portal.MXID) > 0 {
-		portal.UpdateMatrixRoom(ce.User, info)
+		portal.UpdateMatrixRoom(ce.User, info, nil)
 		ce.Reply("Portal room synced.")
 	} else {
-		err = portal.CreateMatrixRoom(ce.User, info, true, true)
+		err = portal.CreateMatrixRoom(ce.User, info, nil, true, true)
 		if err != nil {
 			ce.Reply("Failed to create room: %v", err)
 		} else {

database/portal.go

@@ -34,7 +34,7 @@ type PortalKey struct {
 }
 
 func NewPortalKey(jid, receiver types.JID) PortalKey {
-	if jid.Server == types.GroupServer {
+	if jid.Server == types.GroupServer || jid.Server == types.NewsletterServer {
 		receiver = jid
 	} else if jid.Server == types.LegacyUserServer {
 		jid.Server = types.DefaultUserServer

go.mod

@@ -11,9 +11,9 @@ require (
 	github.com/rs/zerolog v1.30.0
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/tidwall/gjson v1.16.0
-	go.mau.fi/util v0.1.0
+	go.mau.fi/util v0.1.1-0.20231013112707-e938021823cc
 	go.mau.fi/webp v0.1.0
-	go.mau.fi/whatsmeow v0.0.0-20230916142552-a743fdc23bf1
+	go.mau.fi/whatsmeow v0.0.0-20231013150720-028a685d137c
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
 	golang.org/x/image v0.12.0
 	golang.org/x/net v0.15.0

go.sum

@@ -66,12 +66,12 @@ github.com/yuin/goldmark v1.5.6 h1:COmQAWTCcGetChm3Ig7G/t8AFAN00t+o8Mt4cf7JpwA=
 github.com/yuin/goldmark v1.5.6/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c=
 go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I=
-go.mau.fi/util v0.1.0 h1:BwIFWIOEeO7lsiI2eWKFkWTfc5yQmoe+0FYyOFVyaoE=
+go.mau.fi/util v0.1.1-0.20231013112707-e938021823cc h1:/ZY5g+McWqVSA6fK8ROBOyJFb5hCBBQKMcm2oRFzc9c=
-go.mau.fi/util v0.1.0/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84=
+go.mau.fi/util v0.1.1-0.20231013112707-e938021823cc/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84=
 go.mau.fi/webp v0.1.0 h1:BHObH/DcFntT9KYun5pDr0Ot4eUZO8k2C7eP7vF4ueA=
 go.mau.fi/webp v0.1.0/go.mod h1:e42Z+VMFrUMS9cpEwGRIor+lQWO8oUAyPyMtcL+NMt8=
-go.mau.fi/whatsmeow v0.0.0-20230916142552-a743fdc23bf1 h1:tfVqib0PAAgMJrZu/Ko25J436e91HKgZepwdhgPmeHM=
+go.mau.fi/whatsmeow v0.0.0-20231013150720-028a685d137c h1:fpNierRxUnaQfYUH0v45bBq8JK6rAt2EkTZtlIYdBDs=
-go.mau.fi/whatsmeow v0.0.0-20230916142552-a743fdc23bf1/go.mod h1:1xFS2b5zqsg53ApsYB4FDtko7xG7r+gVgBjh9k+9/GE=
+go.mau.fi/whatsmeow v0.0.0-20231013150720-028a685d137c/go.mod h1:rczIT6OzqI4FQIujJe8X/rfxi0pQVFpjQIN2+9vD3Gg=
 go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto=
 go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

historysync.go

@@ -154,7 +154,7 @@ func (user *User) backfillAll() {
 					Msg("Chat already has a room, deleting messages from database")
 				user.bridge.DB.HistorySync.DeleteConversation(user.MXID, portal.Key.JID.String())
 			} else if limit < 0 || i < limit {
-				err = portal.CreateMatrixRoom(user, nil, true, true)
+				err = portal.CreateMatrixRoom(user, nil, nil, true, true)
 				if err != nil {
 					user.zlog.Err(err).Msg("Failed to create Matrix room for backfill")
 				}
@@ -316,7 +316,7 @@ func (user *User) backfillInChunks(req *database.Backfill, conv *database.Histor
 
 	if len(portal.MXID) == 0 {
 		user.log.Debugln("Creating portal for", portal.Key.JID, "as part of history sync handling")
-		err := portal.CreateMatrixRoom(user, nil, true, false)
+		err := portal.CreateMatrixRoom(user, nil, nil, true, false)
 		if err != nil {
 			user.log.Errorfln("Failed to create room for %s during backfill: %v", portal.Key.JID, err)
 			return

portal.go

@@ -344,7 +344,7 @@ func (portal *Portal) handleWhatsAppMessageLoopItem(msg PortalMessage) {
 			return
 		}
 		portal.log.Debugln("Creating Matrix room from incoming message")
-		err := portal.CreateMatrixRoom(msg.source, nil, false, true)
+		err := portal.CreateMatrixRoom(msg.source, nil, nil, false, true)
 		if err != nil {
 			portal.log.Errorln("Failed to create portal room:", err)
 			return
@@ -1077,6 +1077,9 @@ func (portal *Portal) getMessagePuppet(user *User, info *types.MessageInfo) (pup
 }
 
 func (portal *Portal) getMessageIntent(user *User, info *types.MessageInfo, msgType string) *appservice.IntentAPI {
+	if portal.IsNewsletter() && info.Sender == info.Chat {
+		return portal.MainIntent()
+	}
 	puppet := portal.getMessagePuppet(user, info)
 	if puppet == nil {
 		return nil
@@ -1161,6 +1164,9 @@ func (portal *Portal) syncParticipant(source *User, participant types.GroupParti
 }
 
 func (portal *Portal) SyncParticipants(source *User, metadata *types.GroupInfo) ([]id.UserID, *event.PowerLevelsEventContent) {
+	if portal.IsNewsletter() {
+		return nil, nil
+	}
 	changed := false
 	var levels *event.PowerLevelsEventContent
 	var err error
@@ -1239,6 +1245,18 @@ func reuploadAvatar(intent *appservice.IntentAPI, url string) (id.ContentURI, er
 	return resp.ContentURI, nil
 }
 
+func (user *User) reuploadAvatarDirectPath(intent *appservice.IntentAPI, directPath string) (id.ContentURI, error) {
+	data, err := user.Client.DownloadMediaWithPath(directPath, nil, nil, nil, 0, "", "")
+	if err != nil {
+		return id.ContentURI{}, fmt.Errorf("failed to download avatar: %w", err)
+	}
+	resp, err := intent.UploadBytes(data, http.DetectContentType(data))
+	if err != nil {
+		return id.ContentURI{}, fmt.Errorf("failed to upload avatar to Matrix: %w", err)
+	}
+	return resp.ContentURI, nil
+}
+
 func (user *User) updateAvatar(jid types.JID, isCommunity bool, avatarID *string, avatarURL *id.ContentURI, avatarSet *bool, log log.Logger, intent *appservice.IntentAPI) bool {
 	currentID := ""
 	if *avatarSet && *avatarID != "remove" && *avatarID != "unauthorized" {
@@ -1273,14 +1291,23 @@ func (user *User) updateAvatar(jid types.JID, isCommunity bool, avatarID *string
 	}
 	if avatar.ID == *avatarID && *avatarSet {
 		return false
-	} else if len(avatar.URL) == 0 {
+	} else if len(avatar.URL) == 0 && len(avatar.DirectPath) == 0 {
 		log.Warnln("Didn't get URL in response to avatar query")
 		return false
 	} else if avatar.ID != *avatarID || avatarURL.IsEmpty() {
-		url, err := reuploadAvatar(intent, avatar.URL)
+		var url id.ContentURI
-		if err != nil {
+		if len(avatar.URL) > 0 {
-			log.Warnln("Failed to reupload avatar:", err)
+			url, err = reuploadAvatar(intent, avatar.URL)
-			return false
+			if err != nil {
+				log.Warnln("Failed to reupload avatar:", err)
+				return false
+			}
+		} else {
+			url, err = user.reuploadAvatarDirectPath(intent, avatar.DirectPath)
+			if err != nil {
+				log.Warnln("Failed to reupload avatar:", err)
+				return false
+			}
 		}
 		*avatarURL = url
 	}
@@ -1290,10 +1317,61 @@ func (user *User) updateAvatar(jid types.JID, isCommunity bool, avatarID *string
 	return true
 }
 
+func (portal *Portal) UpdateNewsletterAvatar(user *User, meta *types.NewsletterMetadata) bool {
+	portal.avatarLock.Lock()
+	defer portal.avatarLock.Unlock()
+	var picID string
+	picture := meta.ThreadMeta.Picture
+	if picture == nil {
+		picID = meta.ThreadMeta.Preview.ID
+	} else {
+		picID = picture.ID
+	}
+	if picID == "" {
+		picID = "remove"
+	}
+	if portal.Avatar != picID || !portal.AvatarSet {
+		if picID == "remove" {
+			portal.AvatarURL = id.ContentURI{}
+		} else if portal.Avatar != picID || portal.AvatarURL.IsEmpty() {
+			var err error
+			if picture == nil {
+				meta, err = user.Client.GetNewsletterInfo(portal.Key.JID)
+				if err != nil {
+					portal.log.Warnln("Failed to fetch full res avatar info for newsletter:", err)
+					return false
+				}
+				picture = meta.ThreadMeta.Picture
+				if picture == nil {
+					portal.log.Warnln("Didn't get full res avatar info for newsletter")
+					return false
+				}
+				picID = picture.ID
+			}
+			portal.AvatarURL, err = user.reuploadAvatarDirectPath(portal.MainIntent(), picture.DirectPath)
+			if err != nil {
+				portal.log.Warnln("Failed to reupload newsletter avatar:", err)
+				return false
+			}
+		}
+		portal.Avatar = picID
+		portal.AvatarSet = false
+		return portal.setRoomAvatar(true, types.EmptyJID, true)
+	}
+	return false
+}
+
 func (portal *Portal) UpdateAvatar(user *User, setBy types.JID, updateInfo bool) bool {
+	if portal.IsNewsletter() {
+		return false
+	}
 	portal.avatarLock.Lock()
 	defer portal.avatarLock.Unlock()
 	changed := user.updateAvatar(portal.Key.JID, portal.IsParent, &portal.Avatar, &portal.AvatarURL, &portal.AvatarSet, portal.log, portal.MainIntent())
+	return portal.setRoomAvatar(changed, setBy, updateInfo)
+}
+
+func (portal *Portal) setRoomAvatar(changed bool, setBy types.JID, updateInfo bool) bool {
 	if !changed || portal.Avatar == "unauthorized" {
 		if changed || updateInfo {
 			portal.Update(nil)
@@ -1391,6 +1469,21 @@ func (portal *Portal) UpdateTopic(topic string, setBy types.JID, updateInfo bool
 	return false
 }
 
+func newsletterToGroupInfo(meta *types.NewsletterMetadata) *types.GroupInfo {
+	var out types.GroupInfo
+	out.JID = meta.ID
+	out.Name = meta.ThreadMeta.Name.Text
+	out.NameSetAt = meta.ThreadMeta.Name.UpdateTime.Time
+	out.Topic = meta.ThreadMeta.Description.Text
+	out.TopicSetAt = meta.ThreadMeta.Description.UpdateTime.Time
+	out.TopicID = meta.ThreadMeta.Description.ID
+	out.GroupCreated = meta.ThreadMeta.CreationTime.Time
+	out.IsAnnounce = true
+	out.IsLocked = true
+	out.IsIncognito = true
+	return &out
+}
+
 func (portal *Portal) UpdateParentGroup(source *User, parent types.JID, updateInfo bool) bool {
 	portal.parentGroupUpdateLock.Lock()
 	defer portal.parentGroupUpdateLock.Unlock()
@@ -1435,6 +1528,14 @@ func (portal *Portal) UpdateMetadata(user *User, groupInfo *types.GroupInfo) boo
 		//update = portal.UpdateTopic(BroadcastTopic, "", nil, false) || update
 		return update
 	}
+	if groupInfo == nil && portal.IsNewsletter() {
+		newsletterInfo, err := user.Client.GetNewsletterInfo(portal.Key.JID)
+		if err != nil {
+			portal.zlog.Err(err).Msg("Failed to get newsletter info")
+			return false
+		}
+		groupInfo = newsletterToGroupInfo(newsletterInfo)
+	}
 	if groupInfo == nil {
 		var err error
 		groupInfo, err = user.Client.GetGroupInfo(portal.Key.JID)
@@ -1471,7 +1572,7 @@ func (portal *Portal) ensureUserInvited(user *User) bool {
 	return user.ensureInvited(portal.MainIntent(), portal.MXID, portal.IsPrivateChat())
 }
 
-func (portal *Portal) UpdateMatrixRoom(user *User, groupInfo *types.GroupInfo) bool {
+func (portal *Portal) UpdateMatrixRoom(user *User, groupInfo *types.GroupInfo, newsletterMetadata *types.NewsletterMetadata) bool {
 	if len(portal.MXID) == 0 {
 		return false
 	}
@@ -1480,10 +1581,16 @@ func (portal *Portal) UpdateMatrixRoom(user *User, groupInfo *types.GroupInfo) b
 	portal.ensureUserInvited(user)
 	go portal.addToPersonalSpace(user)
 
+	if groupInfo == nil && newsletterMetadata != nil {
+		groupInfo = newsletterToGroupInfo(newsletterMetadata)
+	}
+
 	update := false
 	update = portal.UpdateMetadata(user, groupInfo) || update
-	if !portal.IsPrivateChat() && !portal.IsBroadcastList() {
+	if !portal.IsPrivateChat() && !portal.IsBroadcastList() && !portal.IsNewsletter() {
 		update = portal.UpdateAvatar(user, types.EmptyJID, false) || update
+	} else if newsletterMetadata != nil {
+		update = portal.UpdateNewsletterAvatar(user, newsletterMetadata) || update
 	}
 	if update || portal.LastSync.Add(24*time.Hour).Before(time.Now()) {
 		portal.LastSync = time.Now()
@@ -1691,7 +1798,7 @@ func (portal *Portal) GetEncryptionEventContent() (evt *event.EncryptionEventCon
 	return
 }
 
-func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, isFullInfo, backfill bool) error {
+func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, newsletterMetadata *types.NewsletterMetadata, isFullInfo, backfill bool) error {
 	portal.roomCreateLock.Lock()
 	defer portal.roomCreateLock.Unlock()
 	if len(portal.MXID) > 0 {
@@ -1738,7 +1845,18 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, i
 		portal.log.Debugln("Broadcast list is not yet supported, not creating room after all")
 		return fmt.Errorf("broadcast list bridging is currently not supported")
 	} else {
-		if groupInfo == nil || !isFullInfo {
+		if portal.IsNewsletter() {
+			if newsletterMetadata == nil {
+				var err error
+				newsletterMetadata, err = user.Client.GetNewsletterInfo(portal.Key.JID)
+				if err != nil {
+					return err
+				}
+			}
+			if groupInfo == nil {
+				groupInfo = newsletterToGroupInfo(newsletterMetadata)
+			}
+		} else if groupInfo == nil || !isFullInfo {
 			foundInfo, err := user.Client.GetGroupInfo(portal.Key.JID)
 
 			// Ensure that the user is actually a participant in the conversation
@@ -1764,7 +1882,11 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, i
 				portal.ExpirationTime = groupInfo.DisappearingTimer
 			}
 		}
-		portal.UpdateAvatar(user, types.EmptyJID, false)
+		if portal.IsNewsletter() {
+			portal.UpdateNewsletterAvatar(user, newsletterMetadata)
+		} else {
+			portal.UpdateAvatar(user, types.EmptyJID, false)
+		}
 	}
 
 	powerLevels := portal.GetBasePowerLevels()
@@ -1843,7 +1965,7 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, i
 	autoJoinInvites := portal.bridge.SpecVersions.Supports(mautrix.BeeperFeatureAutojoinInvites)
 	if autoJoinInvites {
 		portal.log.Debugfln("Hungryserv mode: adding all group members in create request")
-		if groupInfo != nil {
+		if groupInfo != nil && !portal.IsNewsletter() {
 			// TODO non-hungryserv could also include all members in invites, and then send joins manually?
 			participants, powerLevels := portal.SyncParticipants(user, groupInfo)
 			invite = append(invite, participants...)
@@ -1911,7 +2033,7 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, i
 	go portal.updateCommunitySpace(user, true, true)
 	go portal.addToPersonalSpace(user)
 
-	if groupInfo != nil && !autoJoinInvites {
+	if !portal.IsNewsletter() && groupInfo != nil && !autoJoinInvites {
 		portal.SyncParticipants(user, groupInfo)
 	}
 	//if broadcastMetadata != nil {
@@ -1989,7 +2111,7 @@ func (portal *Portal) updateCommunitySpace(user *User, add, updateInfo bool) boo
 			return false
 		}
 		portal.log.Debugfln("Creating portal for parent group %v", space.Key.JID)
-		err := space.CreateMatrixRoom(user, nil, false, false)
+		err := space.CreateMatrixRoom(user, nil, nil, false, false)
 		if err != nil {
 			portal.log.Debugfln("Failed to create portal for parent group: %v", err)
 			return false
@@ -2039,6 +2161,10 @@ func (portal *Portal) IsBroadcastList() bool {
 	return portal.Key.JID.Server == types.BroadcastServer
 }
 
+func (portal *Portal) IsNewsletter() bool {
+	return portal.Key.JID.Server == types.NewsletterServer
+}
+
 func (portal *Portal) IsStatusBroadcastList() bool {
 	return portal.Key.JID == types.StatusBroadcastJID
 }

provisioning.go

@@ -451,7 +451,7 @@ func (prov *ProvisioningAPI) OpenGroup(w http.ResponseWriter, r *http.Request) {
 		portal := user.GetPortalByJID(info.JID)
 		status := http.StatusOK
 		if len(portal.MXID) == 0 {
-			err = portal.CreateMatrixRoom(user, info, true, true)
+			err = portal.CreateMatrixRoom(user, info, nil, true, true)
 			if err != nil {
 				jsonResponse(w, http.StatusInternalServerError, Error{
 					Error: fmt.Sprintf("Failed to create portal: %v", err),
@@ -532,7 +532,7 @@ func (prov *ProvisioningAPI) JoinGroup(w http.ResponseWriter, r *http.Request) {
 		status := http.StatusOK
 		if len(portal.MXID) == 0 {
 			time.Sleep(500 * time.Millisecond) // Wait for incoming group info to create the portal automatically
-			err = portal.CreateMatrixRoom(user, info, true, true)
+			err = portal.CreateMatrixRoom(user, info, nil, true, true)
 			if err != nil {
 				jsonResponse(w, http.StatusInternalServerError, Error{
 					Error: fmt.Sprintf("Failed to create portal: %v", err),

user.go

@@ -327,7 +327,7 @@ func (user *User) doPuppetResync() {
 			user.log.Warnfln("Failed to get group info for %s to do background sync: %v", portal.Key.JID, err)
 		} else {
 			user.log.Debugfln("Doing background sync for %s", portal.Key.JID)
-			portal.UpdateMatrixRoom(user, groupInfo)
+			portal.UpdateMatrixRoom(user, groupInfo, nil)
 		}
 	}
 	if len(puppetJIDs) == 0 {
@@ -850,6 +850,10 @@ func (user *User) HandleEvent(event interface{}) {
 	case *events.JoinedGroup:
 		user.groupListCache = nil
 		go user.handleGroupCreate(v)
+	case *events.NewsletterJoin:
+		go user.handleNewsletterJoin(v)
+	case *events.NewsletterLeave:
+		go user.handleNewsletterLeave(v)
 	case *events.Picture:
 		go user.handlePictureUpdate(v)
 	case *events.Receipt:
@@ -1184,13 +1188,13 @@ func (user *User) ResyncGroups(createPortals bool) error {
 		portal := user.GetPortalByJID(group.JID)
 		if len(portal.MXID) == 0 {
 			if createPortals {
-				err = portal.CreateMatrixRoom(user, group, true, true)
+				err = portal.CreateMatrixRoom(user, group, nil, true, true)
 				if err != nil {
 					return fmt.Errorf("failed to create room for %s: %w", group.JID, err)
 				}
 			}
 		} else {
-			portal.UpdateMatrixRoom(user, group)
+			portal.UpdateMatrixRoom(user, group, nil)
 		}
 	}
 	return nil
@@ -1301,12 +1305,12 @@ func (user *User) handleGroupCreate(evt *events.JoinedGroup) {
 			user.log.Debugfln("Ignoring group create event with key %s", evt.CreateKey)
 			return
 		}
-		err := portal.CreateMatrixRoom(user, &evt.GroupInfo, true, true)
+		err := portal.CreateMatrixRoom(user, &evt.GroupInfo, nil, true, true)
 		if err != nil {
 			user.log.Errorln("Failed to create Matrix room after join notification: %v", err)
 		}
 	} else {
-		portal.UpdateMatrixRoom(user, &evt.GroupInfo)
+		portal.UpdateMatrixRoom(user, &evt.GroupInfo, nil)
 	}
 }
 
@@ -1376,6 +1380,25 @@ func (user *User) handleGroupUpdate(evt *events.GroupInfo) {
 	}
 }
 
+func (user *User) handleNewsletterJoin(evt *events.NewsletterJoin) {
+	portal := user.GetPortalByJID(evt.ID)
+	if portal.MXID == "" {
+		err := portal.CreateMatrixRoom(user, nil, &evt.NewsletterMetadata, true, false)
+		if err != nil {
+			user.zlog.Err(err).Msg("Failed to create room on newsletter join event")
+		}
+	} else {
+		portal.UpdateMatrixRoom(user, nil, &evt.NewsletterMetadata)
+	}
+}
+
+func (user *User) handleNewsletterLeave(evt *events.NewsletterLeave) {
+	portal := user.GetPortalByJID(evt.ID)
+	if portal.MXID != "" {
+		portal.HandleWhatsAppKick(user, user.JID, []types.JID{user.JID})
+	}
+}
+
 func (user *User) handlePictureUpdate(evt *events.Picture) {
 	if evt.JID.Server == types.DefaultUserServer {
 		puppet := user.bridge.GetPuppetByJID(evt.JID)
@@ -1405,7 +1428,7 @@ func (user *User) StartPM(jid types.JID, reason string) (*Portal, *Puppet, bool,
 			return portal, puppet, false, nil
 		}
 	}
-	err := portal.CreateMatrixRoom(user, nil, false, true)
+	err := portal.CreateMatrixRoom(user, nil, nil, false, true)
 	return portal, puppet, true, err
 }