Merge pull request #435 from mautrix/adamvy/link-preview

Add support for bridging embedded link previews

go.mod

@@ -9,7 +9,8 @@ require (
 	github.com/mattn/go-sqlite3 v1.14.10
 	github.com/prometheus/client_golang v1.11.0
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
-	go.mau.fi/whatsmeow v0.0.0-20220128124639-e64fb976bf15
+	github.com/tidwall/gjson v1.13.0
+	go.mau.fi/whatsmeow v0.0.0-20220204210537-a425ddb0b16c
 	golang.org/x/image v0.0.0-20211028202545-6944b10bf410
 	google.golang.org/protobuf v1.27.1
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
@@ -29,7 +30,6 @@ require (
 	github.com/prometheus/common v0.26.0 // indirect
 	github.com/prometheus/procfs v0.6.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/tidwall/gjson v1.13.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tidwall/sjson v1.2.4 // indirect

go.sum

@@ -140,8 +140,8 @@ github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc=
 github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM=
 go.mau.fi/libsignal v0.0.0-20211109153248-a67163214910 h1:9FFhG0OmkuMau5UEaTgiUQ+7cSbtbOQ7hiWKdN8OI3I=
 go.mau.fi/libsignal v0.0.0-20211109153248-a67163214910/go.mod h1:AufGrvVh+00Nc07Jm4hTquh7yleZyn20tKJI2wCPAKg=
-go.mau.fi/whatsmeow v0.0.0-20220128124639-e64fb976bf15 h1:BmdZu7K6IHsb+sPxvzkEjAINKxTMNeSiJRe1cvfesIY=
+go.mau.fi/whatsmeow v0.0.0-20220204210537-a425ddb0b16c h1:hwuZ1W55J2uSwm029dREAr6crSVa+i5VsF91ltK389k=
-go.mau.fi/whatsmeow v0.0.0-20220128124639-e64fb976bf15/go.mod h1:8jUjOAi3xtGubxcZgG8uSHpAdyQXBRbWAfxkctX/4y4=
+go.mau.fi/whatsmeow v0.0.0-20220204210537-a425ddb0b16c/go.mod h1:8jUjOAi3xtGubxcZgG8uSHpAdyQXBRbWAfxkctX/4y4=
 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

portal.go

@@ -441,7 +441,7 @@ func formatDuration(d time.Duration) string {
 func (portal *Portal) convertMessage(intent *appservice.IntentAPI, source *User, info *types.MessageInfo, waMsg *waProto.Message) *ConvertedMessage {
 	switch {
 	case waMsg.Conversation != nil || waMsg.ExtendedTextMessage != nil:
-		return portal.convertTextMessage(intent, waMsg)
+		return portal.convertTextMessage(intent, source, waMsg)
 	case waMsg.ImageMessage != nil:
 		return portal.convertMediaMessage(intent, source, info, waMsg.GetImageMessage())
 	case waMsg.StickerMessage != nil:
@@ -1491,13 +1491,15 @@ type ConvertedMessage struct {
 	ExpiresIn uint32
 }
 
-func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, msg *waProto.Message) *ConvertedMessage {
+func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, source *User, msg *waProto.Message) *ConvertedMessage {
 	content := &event.MessageEventContent{
 		Body:    msg.GetConversation(),
 		MsgType: event.MsgText,
 	}
 	var replyTo types.MessageID
 	var expiresIn uint32
+	extraAttrs := map[string]interface{}{}
+
 	if msg.GetExtendedTextMessage() != nil {
 		content.Body = msg.GetExtendedTextMessage().GetText()
 
@@ -1507,6 +1509,8 @@ func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, msg *waPr
 			replyTo = contextInfo.GetStanzaId()
 		}
 		expiresIn = contextInfo.GetExpiration()
+
+		extraAttrs["com.beeper.linkpreview"] = portal.convertURLPreviewToBeeper(intent, source, msg.GetExtendedTextMessage())
 	}
 
 	return &ConvertedMessage{
@@ -1515,6 +1519,7 @@ func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, msg *waPr
 		Content:   content,
 		ReplyTo:   replyTo,
 		ExpiresIn: expiresIn,
+		Extra:     extraAttrs,
 	}
 }
 
@@ -1992,10 +1997,10 @@ func (portal *Portal) convertMediaMessage(intent *appservice.IntentAPI, source *
 const thumbnailMaxSize = 72
 const thumbnailMinSize = 24
 
-func createJPEGThumbnail(source []byte) ([]byte, error) {
+func createJPEGThumbnailAndGetSize(source []byte) ([]byte, int, int, error) {
 	src, _, err := image.Decode(bytes.NewReader(source))
 	if err != nil {
-		return nil, fmt.Errorf("failed to decode thumbnail: %w", err)
+		return nil, 0, 0, fmt.Errorf("failed to decode thumbnail: %w", err)
 	}
 	imageBounds := src.Bounds()
 	width, height := imageBounds.Max.X, imageBounds.Max.Y
@@ -2028,9 +2033,14 @@ func createJPEGThumbnail(source []byte) ([]byte, error) {
 	var buf bytes.Buffer
 	err = jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpeg.DefaultQuality})
 	if err != nil {
-		return nil, fmt.Errorf("failed to re-encode thumbnail: %w", err)
+		return nil, width, height, fmt.Errorf("failed to re-encode thumbnail: %w", err)
 	}
-	return buf.Bytes(), nil
+	return buf.Bytes(), width, height, nil
+}
+
+func createJPEGThumbnail(source []byte) ([]byte, error) {
+	data, _, _, err := createJPEGThumbnailAndGetSize(source)
+	return data, err
 }
 
 func (portal *Portal) downloadThumbnail(original []byte, thumbnailURL id.ContentURIString, eventID id.EventID) ([]byte, error) {
@@ -2243,11 +2253,13 @@ func (portal *Portal) convertMatrixMessage(sender *User, evt *event.Event) (*waP
 		if content.MsgType == event.MsgEmote && !relaybotFormatted {
 			text = "/me " + text
 		}
-		if ctxInfo.StanzaId != nil || ctxInfo.MentionedJid != nil || ctxInfo.Expiration != nil {
+		if ctxInfo.StanzaId != nil || ctxInfo.MentionedJid != nil || ctxInfo.Expiration != nil || evt.Content.Raw["com.beeper.linkpreview"] != nil {
 			msg.ExtendedTextMessage = &waProto.ExtendedTextMessage{
 				Text:        &text,
 				ContextInfo: &ctxInfo,
 			}
+
+			portal.convertURLPreviewToWhatsApp(sender, evt, msg.ExtendedTextMessage)
 		} else {
 			msg.Conversation = &text
 		}

urlpreview.go

@@ -0,0 +1,180 @@
+// mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge.
+// Copyright (C) 2022 Tulir Asokan
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"image"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/tidwall/gjson"
+	"google.golang.org/protobuf/proto"
+	"maunium.net/go/mautrix/appservice"
+	"maunium.net/go/mautrix/crypto/attachment"
+	"maunium.net/go/mautrix/event"
+	"maunium.net/go/mautrix/id"
+
+	"go.mau.fi/whatsmeow"
+	waProto "go.mau.fi/whatsmeow/binary/proto"
+)
+
+type BeeperLinkPreview struct {
+	MatchedURL   string `json:"matched_url"`
+	CanonicalURL string `json:"og:url,omitempty"`
+	Title        string `json:"og:title,omitempty"`
+	Type         string `json:"og:type,omitempty"`
+	Description  string `json:"og:description,omitempty"`
+
+	ImageURL        id.ContentURIString      `json:"og:image,omitempty"`
+	ImageEncryption *event.EncryptedFileInfo `json:"beeper:image:encryption,omitempty"`
+
+	ImageSize   int    `json:"matrix:image:size,omitempty"`
+	ImageWidth  int    `json:"og:image:width,omitempty"`
+	ImageHeight int    `json:"og:image:height,omitempty"`
+	ImageType   string `json:"og:image:type,omitempty"`
+}
+
+func (portal *Portal) convertURLPreviewToBeeper(intent *appservice.IntentAPI, source *User, msg *waProto.ExtendedTextMessage) (output *BeeperLinkPreview) {
+	if msg.GetMatchedText() == "" {
+		return
+	}
+
+	output = &BeeperLinkPreview{
+		MatchedURL:   msg.GetMatchedText(),
+		CanonicalURL: msg.GetCanonicalUrl(),
+		Title:        msg.GetTitle(),
+		Description:  msg.GetDescription(),
+	}
+	if len(output.CanonicalURL) == 0 {
+		output.CanonicalURL = output.MatchedURL
+	}
+
+	var thumbnailData []byte
+	if msg.ThumbnailDirectPath != nil {
+		var err error
+		thumbnailData, err = source.Client.DownloadThumbnail(msg)
+		if err != nil {
+			portal.log.Warnfln("Failed to download thumbnail for link preview: %v", err)
+		}
+	} else if msg.JpegThumbnail != nil {
+		thumbnailData = msg.JpegThumbnail
+	}
+	if thumbnailData != nil {
+		output.ImageHeight = int(msg.GetThumbnailHeight())
+		output.ImageWidth = int(msg.GetThumbnailWidth())
+		if output.ImageHeight == 0 || output.ImageWidth == 0 {
+			src, _, err := image.Decode(bytes.NewReader(thumbnailData))
+			if err == nil {
+				imageBounds := src.Bounds()
+				output.ImageWidth, output.ImageHeight = imageBounds.Max.X, imageBounds.Max.Y
+			}
+		}
+		output.ImageSize = len(thumbnailData)
+		output.ImageType = http.DetectContentType(thumbnailData)
+		uploadData, uploadMime := thumbnailData, output.ImageType
+		if portal.Encrypted {
+			crypto := attachment.NewEncryptedFile()
+			uploadData = crypto.Encrypt(uploadData)
+			uploadMime = "application/octet-stream"
+			output.ImageEncryption = &event.EncryptedFileInfo{EncryptedFile: *crypto}
+		}
+		resp, err := intent.UploadBytes(uploadData, uploadMime)
+		if err != nil {
+			portal.log.Warnfln("Failed to reupload thumbnail for link preview: %v", err)
+		} else {
+			if output.ImageEncryption != nil {
+				output.ImageEncryption.URL = resp.ContentURI.CUString()
+			} else {
+				output.ImageURL = resp.ContentURI.CUString()
+			}
+		}
+	}
+	if msg.GetPreviewType() == waProto.ExtendedTextMessage_VIDEO {
+		output.Type = "video.other"
+	}
+
+	return
+}
+
+func (portal *Portal) convertURLPreviewToWhatsApp(sender *User, evt *event.Event, dest *waProto.ExtendedTextMessage) {
+	rawPreview := gjson.GetBytes(evt.Content.VeryRaw, `com\.beeper\.linkpreview`)
+	if !rawPreview.Exists() || !rawPreview.IsObject() {
+		return
+	}
+	var preview BeeperLinkPreview
+	if err := json.Unmarshal([]byte(rawPreview.Raw), &preview); err != nil || len(preview.MatchedURL) == 0 {
+		return
+	}
+
+	dest.MatchedText = &preview.MatchedURL
+	if len(preview.CanonicalURL) > 0 {
+		dest.CanonicalUrl = &preview.CanonicalURL
+	}
+	if len(preview.Description) > 0 {
+		dest.Description = &preview.Description
+	}
+	if len(preview.Title) > 0 {
+		dest.Title = &preview.Title
+	}
+	if strings.HasPrefix(preview.Type, "video.") {
+		dest.PreviewType = waProto.ExtendedTextMessage_VIDEO.Enum()
+	}
+	imageMXC := preview.ImageURL.ParseOrIgnore()
+	if preview.ImageEncryption != nil {
+		imageMXC = preview.ImageEncryption.URL.ParseOrIgnore()
+	}
+	if !imageMXC.IsEmpty() {
+		data, err := portal.MainIntent().DownloadBytes(imageMXC)
+		if err != nil {
+			portal.log.Errorfln("Failed to download URL preview image %s in %s: %v", preview.ImageURL, evt.ID, err)
+			return
+		}
+		if preview.ImageEncryption != nil {
+			data, err = preview.ImageEncryption.Decrypt(data)
+			if err != nil {
+				portal.log.Errorfln("Failed to decrypt URL preview image in %s: %v", evt.ID, err)
+				return
+			}
+		}
+		dest.MediaKeyTimestamp = proto.Int64(time.Now().Unix())
+		uploadResp, err := sender.Client.Upload(context.Background(), data, whatsmeow.MediaLinkThumbnail)
+		if err != nil {
+			portal.log.Errorfln("Failed to upload URL preview thumbnail in %s: %v", evt.ID, err)
+			return
+		}
+		dest.ThumbnailSha256 = uploadResp.FileSHA256
+		dest.ThumbnailEncSha256 = uploadResp.FileEncSHA256
+		dest.ThumbnailDirectPath = &uploadResp.DirectPath
+		dest.MediaKey = uploadResp.MediaKey
+		var width, height int
+		dest.JpegThumbnail, width, height, err = createJPEGThumbnailAndGetSize(data)
+		if err != nil {
+			portal.log.Warnfln("Failed to create JPEG thumbnail for URL preview in %s: %v", evt.ID, err)
+		}
+		if preview.ImageHeight > 0 && preview.ImageWidth > 0 {
+			dest.ThumbnailWidth = proto.Uint32(uint32(preview.ImageWidth))
+			dest.ThumbnailHeight = proto.Uint32(uint32(preview.ImageHeight))
+		} else if width > 0 && height > 0 {
+			dest.ThumbnailWidth = proto.Uint32(uint32(width))
+			dest.ThumbnailHeight = proto.Uint32(uint32(height))
+		}
+	}
+}