From d4334f5df8d29d2a15c28b6f589c273e27bbaf08 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 4 Feb 2022 22:19:55 +0200 Subject: [PATCH] Clean up embedded link preview code --- go.mod | 4 +- go.sum | 4 +- portal.go | 135 +++++-------------------------------------- urlpreview.go | 155 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 172 insertions(+), 126 deletions(-) create mode 100644 urlpreview.go diff --git a/go.mod b/go.mod index 0f663ad..1fa3bd7 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,8 @@ require ( github.com/mattn/go-sqlite3 v1.14.10 github.com/prometheus/client_golang v1.11.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e - go.mau.fi/whatsmeow v0.0.0-20220128124639-e64fb976bf15 + github.com/tidwall/gjson v1.13.0 + go.mau.fi/whatsmeow v0.0.0-20220204175019-e490de34933c golang.org/x/image v0.0.0-20211028202545-6944b10bf410 google.golang.org/protobuf v1.27.1 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b @@ -29,7 +30,6 @@ require ( github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/tidwall/gjson v1.13.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/sjson v1.2.4 // indirect diff --git a/go.sum b/go.sum index 0bc53ee..044bd50 100644 --- a/go.sum +++ b/go.sum @@ -140,8 +140,8 @@ github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc= github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= go.mau.fi/libsignal v0.0.0-20211109153248-a67163214910 h1:9FFhG0OmkuMau5UEaTgiUQ+7cSbtbOQ7hiWKdN8OI3I= go.mau.fi/libsignal v0.0.0-20211109153248-a67163214910/go.mod h1:AufGrvVh+00Nc07Jm4hTquh7yleZyn20tKJI2wCPAKg= -go.mau.fi/whatsmeow v0.0.0-20220128124639-e64fb976bf15 h1:BmdZu7K6IHsb+sPxvzkEjAINKxTMNeSiJRe1cvfesIY= -go.mau.fi/whatsmeow v0.0.0-20220128124639-e64fb976bf15/go.mod h1:8jUjOAi3xtGubxcZgG8uSHpAdyQXBRbWAfxkctX/4y4= +go.mau.fi/whatsmeow v0.0.0-20220204175019-e490de34933c h1:AVSYHQ0N5n3buL+thypCk2jiltD+3+pUQ7oPVhC7I3w= +go.mau.fi/whatsmeow v0.0.0-20220204175019-e490de34933c/go.mod h1:8jUjOAi3xtGubxcZgG8uSHpAdyQXBRbWAfxkctX/4y4= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/portal.go b/portal.go index 77f9a8e..72958d0 100644 --- a/portal.go +++ b/portal.go @@ -441,7 +441,7 @@ func formatDuration(d time.Duration) string { func (portal *Portal) convertMessage(intent *appservice.IntentAPI, source *User, info *types.MessageInfo, waMsg *waProto.Message) *ConvertedMessage { switch { case waMsg.Conversation != nil || waMsg.ExtendedTextMessage != nil: - return portal.convertTextMessage(intent, waMsg) + return portal.convertTextMessage(intent, source, waMsg) case waMsg.ImageMessage != nil: return portal.convertMediaMessage(intent, source, info, waMsg.GetImageMessage()) case waMsg.StickerMessage != nil: @@ -1491,7 +1491,7 @@ type ConvertedMessage struct { ExpiresIn uint32 } -func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, msg *waProto.Message) *ConvertedMessage { +func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, source *User, msg *waProto.Message) *ConvertedMessage { content := &event.MessageEventContent{ Body: msg.GetConversation(), MsgType: event.MsgText, @@ -1510,11 +1510,7 @@ func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, msg *waPr } expiresIn = contextInfo.GetExpiration() - preview := portal.convertUrlPreview(msg.GetExtendedTextMessage()) - - if preview != nil { - extraAttrs["com.beeper.linkpreview"] = preview - } + extraAttrs["com.beeper.linkpreview"] = portal.convertURLPreviewToBeeper(intent, source, msg.GetExtendedTextMessage()) } return &ConvertedMessage{ @@ -2001,10 +1997,10 @@ func (portal *Portal) convertMediaMessage(intent *appservice.IntentAPI, source * const thumbnailMaxSize = 72 const thumbnailMinSize = 24 -func createJPEGThumbnail(source []byte) ([]byte, error) { +func createJPEGThumbnailAndGetSize(source []byte) ([]byte, int, int, error) { src, _, err := image.Decode(bytes.NewReader(source)) if err != nil { - return nil, fmt.Errorf("failed to decode thumbnail: %w", err) + return nil, 0, 0, fmt.Errorf("failed to decode thumbnail: %w", err) } imageBounds := src.Bounds() width, height := imageBounds.Max.X, imageBounds.Max.Y @@ -2037,9 +2033,14 @@ func createJPEGThumbnail(source []byte) ([]byte, error) { var buf bytes.Buffer err = jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpeg.DefaultQuality}) if err != nil { - return nil, fmt.Errorf("failed to re-encode thumbnail: %w", err) + return nil, width, height, fmt.Errorf("failed to re-encode thumbnail: %w", err) } - return buf.Bytes(), nil + return buf.Bytes(), width, height, nil +} + +func createJPEGThumbnail(source []byte) ([]byte, error) { + data, _, _, err := createJPEGThumbnailAndGetSize(source) + return data, err } func (portal *Portal) downloadThumbnail(original []byte, thumbnailURL id.ContentURIString, eventID id.EventID) ([]byte, error) { @@ -2069,116 +2070,6 @@ func (portal *Portal) convertWebPtoPNG(webpImage []byte) ([]byte, error) { return pngBuffer.Bytes(), nil } -func (portal *Portal) convertUrlPreview(source *waProto.ExtendedTextMessage) map[string]interface{} { - if source == nil { - return nil - } - - matchedText := source.GetMatchedText() - - if matchedText == "" { - return nil - } - - canonicalUrl := source.GetCanonicalUrl() - - url := matchedText - if canonicalUrl != "" { - url = canonicalUrl - } - - result := map[string]interface{}{ - "og:title": source.GetTitle(), - "og:url": url, - "og:description": source.GetDescription(), - } - - if len(source.GetJpegThumbnail()) > 0 { - thumbnailMime := http.DetectContentType(source.GetJpegThumbnail()) - uploadedThumbnail, _ := portal.MainIntent().UploadBytes(source.GetJpegThumbnail(), thumbnailMime) - if uploadedThumbnail != nil { - cfg, _, _ := image.DecodeConfig(bytes.NewReader(source.GetJpegThumbnail())) - result["og:image"] = uploadedThumbnail.ContentURI.CUString() - result["og:image:width"] = cfg.Width - result["og:image:height"] = cfg.Height - result["og:image:type"] = thumbnailMime - } - } - - return result -} - -func (portal *Portal) updateExtendedMessageForUrlPreview(source *event.Content, dest *waProto.ExtendedTextMessage) { - if source == nil { - return - } - - embeddedLink, ok := source.Raw["com.beeper.linkpreview"].(map[string]interface{}) - - if !ok || embeddedLink == nil { - return - } - - matchedUrl, ok := embeddedLink["matchedUrl"].(string) - - if !ok || matchedUrl == "" { - return - } - - dest.MatchedText = &matchedUrl - - canonical, ok := embeddedLink["og:url"].(string) - - if ok { - dest.CanonicalUrl = &canonical - } - - description, ok := embeddedLink["og:description"].(string) - - if ok { - dest.Description = &description - } - - rawMXC, ok := embeddedLink["og:image"].(string) - - if !ok || rawMXC == "" { - return - } - - mxc, err := id.ParseContentURI(rawMXC) - if err != nil { - portal.log.Errorln("Malformed content URL %v: %v", rawMXC, err) - return - } - - data, err := portal.MainIntent().DownloadBytes(mxc) - if err != nil { - portal.log.Errorfln("Failed to download media from %s: %v", rawMXC, err) - return - } - - height, ok := embeddedLink["og:image:height"].(float64) - - if !ok { - portal.log.Errorfln("Height missing or invalid %v", embeddedLink["og:image:height"]) - return - } - - width, ok := embeddedLink["og:image:width"].(float64) - - if !ok { - portal.log.Errorfln("Width missing or invalid %v", embeddedLink["og:image:width"]) - return - } - - height32 := uint32(height) - width32 := uint32(width) - - dest.JpegThumbnail = data - dest.ThumbnailHeight = &height32 - dest.ThumbnailWidth = &width32 -} - func (portal *Portal) preprocessMatrixMedia(sender *User, relaybotFormatted bool, content *event.MessageEventContent, eventID id.EventID, mediaType whatsmeow.MediaType) *MediaUpload { var caption string var mentionedJIDs []string @@ -2368,7 +2259,7 @@ func (portal *Portal) convertMatrixMessage(sender *User, evt *event.Event) (*waP ContextInfo: &ctxInfo, } - portal.updateExtendedMessageForUrlPreview(&evt.Content, msg.ExtendedTextMessage) + portal.convertURLPreviewToWhatsApp(sender, evt, msg.ExtendedTextMessage) } else { msg.Conversation = &text } diff --git a/urlpreview.go b/urlpreview.go new file mode 100644 index 0000000..d3c8d5c --- /dev/null +++ b/urlpreview.go @@ -0,0 +1,155 @@ +// mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. +// Copyright (C) 2022 Tulir Asokan +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +import ( + "bytes" + "context" + "encoding/json" + "image" + "net/http" + + "github.com/tidwall/gjson" + "google.golang.org/protobuf/proto" + "maunium.net/go/mautrix/appservice" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" + + "go.mau.fi/whatsmeow" + waProto "go.mau.fi/whatsmeow/binary/proto" +) + +type BeeperLinkPreview struct { + MatchedURL string `json:"matched_url"` + CanonicalURL string `json:"og:url,omitempty"` + Title string `json:"og:title,omitempty"` + Type string `json:"og:type,omitempty"` + Description string `json:"og:description,omitempty"` + Image id.ContentURIString `json:"og:image,omitempty"` + ImageWidth int `json:"og:image:width,omitempty"` + ImageHeight int `json:"og:image:height,omitempty"` + + MatchedURLFallback string `json:"matchedUrl"` +} + +func (portal *Portal) convertURLPreviewToBeeper(intent *appservice.IntentAPI, source *User, msg *waProto.ExtendedTextMessage) (output *BeeperLinkPreview) { + if msg.GetMatchedText() == "" { + return + } + + output = &BeeperLinkPreview{ + MatchedURL: msg.GetMatchedText(), + CanonicalURL: msg.GetCanonicalUrl(), + Title: msg.GetTitle(), + Description: msg.GetDescription(), + } + output.MatchedURLFallback = output.MatchedURL + if len(output.CanonicalURL) == 0 { + output.CanonicalURL = output.MatchedURL + } + + var thumbnailData []byte + if msg.ThumbnailDirectPath != nil { + var err error + thumbnailData, err = source.Client.DownloadThumbnail(msg) + if err != nil { + portal.log.Warnfln("Failed to download thumbnail for link preview: %v", err) + } + } else if msg.JpegThumbnail != nil { + thumbnailData = msg.JpegThumbnail + } + if thumbnailData != nil { + mxc, err := intent.UploadBytes(thumbnailData, http.DetectContentType(thumbnailData)) + if err != nil { + portal.log.Warnfln("Failed to reupload thumbnail for link preview: %v", err) + } else { + output.Image = mxc.ContentURI.CUString() + output.ImageHeight = int(msg.GetThumbnailHeight()) + output.ImageWidth = int(msg.GetThumbnailWidth()) + if output.ImageHeight == 0 || output.ImageWidth == 0 { + src, _, err := image.Decode(bytes.NewReader(thumbnailData)) + if err == nil { + imageBounds := src.Bounds() + output.ImageWidth, output.ImageHeight = imageBounds.Max.X, imageBounds.Max.Y + } + } + } + } + if msg.GetPreviewType() == waProto.ExtendedTextMessage_VIDEO { + output.Type = "video.other" + } + + return +} + +func (portal *Portal) convertURLPreviewToWhatsApp(sender *User, evt *event.Event, dest *waProto.ExtendedTextMessage) { + rawPreview := gjson.GetBytes(evt.Content.VeryRaw, `com\.beeper\.linkpreview`) + if !rawPreview.Exists() || !rawPreview.IsObject() { + return + } + var preview BeeperLinkPreview + if err := json.Unmarshal([]byte(rawPreview.Raw), &preview); err != nil { + return + } + if len(preview.MatchedURL) == 0 { + if len(preview.MatchedURLFallback) == 0 { + return + } else { + preview.MatchedURL = preview.MatchedURLFallback + } + } + + dest.MatchedText = &preview.MatchedURL + if len(preview.CanonicalURL) > 0 { + dest.CanonicalUrl = &preview.CanonicalURL + } + if len(preview.Description) > 0 { + dest.Description = &preview.Description + } + if len(preview.Title) > 0 { + dest.Title = &preview.Title + } + imageMXC := preview.Image.ParseOrIgnore() + if !imageMXC.IsEmpty() { + data, err := portal.MainIntent().DownloadBytes(imageMXC) + if err != nil { + portal.log.Errorfln("Failed to download URL preview image %s: %v", preview.Image, err) + return + } + uploadResp, err := sender.Client.Upload(context.Background(), data, whatsmeow.MediaLinkThumbnail) + if err != nil { + portal.log.Errorfln("Failed to upload URL preview thumbnail in %s: %v", evt.ID, err) + return + } + dest.ThumbnailSha256 = uploadResp.FileSHA256 + dest.ThumbnailEncSha256 = uploadResp.FileEncSHA256 + dest.ThumbnailDirectPath = &uploadResp.DirectPath + dest.MediaKey = uploadResp.MediaKey + var width, height int + dest.JpegThumbnail, width, height, err = createJPEGThumbnailAndGetSize(data) + if err != nil { + portal.log.Warnfln("Failed to create JPEG thumbnail for URL preview in %s: %v", evt.ID, err) + } + if preview.ImageHeight > 0 && preview.ImageWidth > 0 { + dest.ThumbnailWidth = proto.Uint32(uint32(preview.ImageWidth)) + dest.ThumbnailHeight = proto.Uint32(uint32(preview.ImageHeight)) + } else if width > 0 && height > 0 { + dest.ThumbnailWidth = proto.Uint32(uint32(width)) + dest.ThumbnailHeight = proto.Uint32(uint32(height)) + } + } +}