From 3ab04e65c8710f95e38f33713c81ceec1f251a4f Mon Sep 17 00:00:00 2001 From: Adam Van Ymeren Date: Fri, 4 Feb 2022 00:33:21 +0000 Subject: [PATCH 1/4] Add support for bridging embedded link previews Uses experimental com.beeper.linkpreview content extension --- portal.go | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 122 insertions(+), 1 deletion(-) diff --git a/portal.go b/portal.go index cb0bbca..117a1e7 100644 --- a/portal.go +++ b/portal.go @@ -1498,6 +1498,8 @@ func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, msg *waPr } var replyTo types.MessageID var expiresIn uint32 + extraAttrs := map[string]interface{}{} + if msg.GetExtendedTextMessage() != nil { content.Body = msg.GetExtendedTextMessage().GetText() @@ -1507,6 +1509,12 @@ func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, msg *waPr replyTo = contextInfo.GetStanzaId() } expiresIn = contextInfo.GetExpiration() + + preview := portal.convertUrlPreview(msg.GetExtendedTextMessage()); + + if ( preview != nil ) { + extraAttrs["com.beeper.linkpreview"] = preview + } } return &ConvertedMessage{ @@ -1515,6 +1523,7 @@ func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, msg *waPr Content: content, ReplyTo: replyTo, ExpiresIn: expiresIn, + Extra: extraAttrs, } } @@ -2060,6 +2069,116 @@ func (portal *Portal) convertWebPtoPNG(webpImage []byte) ([]byte, error) { return pngBuffer.Bytes(), nil } +func (portal *Portal) convertUrlPreview(source *waProto.ExtendedTextMessage) map[string]interface{} { + if ( source == nil ) { + return nil + } + + matchedText := source.GetMatchedText() + + if ( matchedText == "" ) { + return nil + } + + canonicalUrl := source.GetCanonicalUrl() + + url := matchedText + if ( canonicalUrl != "" ) { + url = canonicalUrl + } + + result := map[string]interface{}{ + "og:title": source.GetTitle(), + "og:url": url, + "og:description": source.GetDescription(), + } + + if len(source.GetJpegThumbnail()) > 0 { + thumbnailMime := http.DetectContentType(source.GetJpegThumbnail()) + uploadedThumbnail, _ := portal.MainIntent().UploadBytes(source.GetJpegThumbnail(), thumbnailMime) + if uploadedThumbnail != nil { + cfg, _, _ := image.DecodeConfig(bytes.NewReader(source.GetJpegThumbnail())) + result["og:image"] = uploadedThumbnail.ContentURI.CUString() + result["og:image:width"] = cfg.Width; + result["og:image:height"] = cfg.Height; + result["og:image:type"] = thumbnailMime; + } + } + + return result +} + +func (portal *Portal) updateExtendedMessageForUrlPreview(source *event.Content, dest *waProto.ExtendedTextMessage) { + if ( source == nil ) { + return + } + + embeddedLink, ok := source.Raw["com.beeper.linkpreview"].(map[string]interface{}); + + if ( !ok || embeddedLink == nil ) { + return + } + + matchedUrl, ok := embeddedLink["matchedUrl"].(string) + + if !ok || matchedUrl == "" { + return + } + + dest.MatchedText = &matchedUrl + + canonical, ok := embeddedLink["og:url"].(string) + + if ok { + dest.CanonicalUrl = &canonical; + } + + description, ok := embeddedLink["og:description"].(string) + + if ok { + dest.Description = &description + } + + rawMXC, ok := embeddedLink["og:image"].(string) + + if !ok || rawMXC == "" { + return + } + + mxc, err := id.ParseContentURI(rawMXC) + if err != nil { + portal.log.Errorln("Malformed content URL %v: %v", rawMXC, err) + return + } + + data, err := portal.MainIntent().DownloadBytes(mxc) + if err != nil { + portal.log.Errorfln("Failed to download media from %s: %v", rawMXC, err) + return + } + + height, ok := embeddedLink["og:image:height"].(float64) + + if !ok { + portal.log.Errorfln("Height missing or invalid %v", embeddedLink["og:image:height"]) + return + } + + width, ok := embeddedLink["og:image:width"].(float64) + + if !ok { + portal.log.Errorfln("Width missing or invalid %v", embeddedLink["og:image:width"]) + return + } + + height32 := uint32(height) + width32 := uint32(width) + + dest.JpegThumbnail = data + dest.ThumbnailHeight = &height32 + dest.ThumbnailWidth = &width32 +} + func (portal *Portal) preprocessMatrixMedia(sender *User, relaybotFormatted bool, content *event.MessageEventContent, eventID id.EventID, mediaType whatsmeow.MediaType) *MediaUpload { var caption string var mentionedJIDs []string @@ -2243,11 +2362,13 @@ func (portal *Portal) convertMatrixMessage(sender *User, evt *event.Event) (*waP if content.MsgType == event.MsgEmote && !relaybotFormatted { text = "/me " + text } - if ctxInfo.StanzaId != nil || ctxInfo.MentionedJid != nil || ctxInfo.Expiration != nil { + if ctxInfo.StanzaId != nil || ctxInfo.MentionedJid != nil || ctxInfo.Expiration != nil || evt.Content.Raw["com.beeper.linkpreview"] != nil { msg.ExtendedTextMessage = &waProto.ExtendedTextMessage{ Text: &text, ContextInfo: &ctxInfo, } + + portal.updateExtendedMessageForUrlPreview(&evt.Content, msg.ExtendedTextMessage) } else { msg.Conversation = &text } From 779e591e6076e101d9d9f980b28211ed300d65c6 Mon Sep 17 00:00:00 2001 From: Adam Van Ymeren Date: Fri, 4 Feb 2022 06:56:32 +0000 Subject: [PATCH 2/4] fix formatting --- portal.go | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/portal.go b/portal.go index 117a1e7..77f9a8e 100644 --- a/portal.go +++ b/portal.go @@ -1510,9 +1510,9 @@ func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, msg *waPr } expiresIn = contextInfo.GetExpiration() - preview := portal.convertUrlPreview(msg.GetExtendedTextMessage()); + preview := portal.convertUrlPreview(msg.GetExtendedTextMessage()) - if ( preview != nil ) { + if preview != nil { extraAttrs["com.beeper.linkpreview"] = preview } } @@ -2070,26 +2070,26 @@ func (portal *Portal) convertWebPtoPNG(webpImage []byte) ([]byte, error) { } func (portal *Portal) convertUrlPreview(source *waProto.ExtendedTextMessage) map[string]interface{} { - if ( source == nil ) { + if source == nil { return nil } matchedText := source.GetMatchedText() - if ( matchedText == "" ) { + if matchedText == "" { return nil } canonicalUrl := source.GetCanonicalUrl() url := matchedText - if ( canonicalUrl != "" ) { - url = canonicalUrl + if canonicalUrl != "" { + url = canonicalUrl } result := map[string]interface{}{ - "og:title": source.GetTitle(), - "og:url": url, + "og:title": source.GetTitle(), + "og:url": url, "og:description": source.GetDescription(), } @@ -2099,9 +2099,9 @@ func (portal *Portal) convertUrlPreview(source *waProto.ExtendedTextMessage) map if uploadedThumbnail != nil { cfg, _, _ := image.DecodeConfig(bytes.NewReader(source.GetJpegThumbnail())) result["og:image"] = uploadedThumbnail.ContentURI.CUString() - result["og:image:width"] = cfg.Width; - result["og:image:height"] = cfg.Height; - result["og:image:type"] = thumbnailMime; + result["og:image:width"] = cfg.Width + result["og:image:height"] = cfg.Height + result["og:image:type"] = thumbnailMime } } @@ -2109,13 +2109,13 @@ func (portal *Portal) convertUrlPreview(source *waProto.ExtendedTextMessage) map } func (portal *Portal) updateExtendedMessageForUrlPreview(source *event.Content, dest *waProto.ExtendedTextMessage) { - if ( source == nil ) { + if source == nil { return } - embeddedLink, ok := source.Raw["com.beeper.linkpreview"].(map[string]interface{}); + embeddedLink, ok := source.Raw["com.beeper.linkpreview"].(map[string]interface{}) - if ( !ok || embeddedLink == nil ) { + if !ok || embeddedLink == nil { return } @@ -2128,9 +2128,9 @@ func (portal *Portal) updateExtendedMessageForUrlPreview(source *event.Content, dest.MatchedText = &matchedUrl canonical, ok := embeddedLink["og:url"].(string) - + if ok { - dest.CanonicalUrl = &canonical; + dest.CanonicalUrl = &canonical } description, ok := embeddedLink["og:description"].(string) @@ -2158,7 +2158,7 @@ func (portal *Portal) updateExtendedMessageForUrlPreview(source *event.Content, } height, ok := embeddedLink["og:image:height"].(float64) - + if !ok { portal.log.Errorfln("Height missing or invalid %v", embeddedLink["og:image:height"]) return @@ -2367,7 +2367,7 @@ func (portal *Portal) convertMatrixMessage(sender *User, evt *event.Event) (*waP Text: &text, ContextInfo: &ctxInfo, } - + portal.updateExtendedMessageForUrlPreview(&evt.Content, msg.ExtendedTextMessage) } else { msg.Conversation = &text From d4334f5df8d29d2a15c28b6f589c273e27bbaf08 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 4 Feb 2022 22:19:55 +0200 Subject: [PATCH 3/4] Clean up embedded link preview code --- go.mod | 4 +- go.sum | 4 +- portal.go | 135 +++++-------------------------------------- urlpreview.go | 155 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 172 insertions(+), 126 deletions(-) create mode 100644 urlpreview.go diff --git a/go.mod b/go.mod index 0f663ad..1fa3bd7 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,8 @@ require ( github.com/mattn/go-sqlite3 v1.14.10 github.com/prometheus/client_golang v1.11.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e - go.mau.fi/whatsmeow v0.0.0-20220128124639-e64fb976bf15 + github.com/tidwall/gjson v1.13.0 + go.mau.fi/whatsmeow v0.0.0-20220204175019-e490de34933c golang.org/x/image v0.0.0-20211028202545-6944b10bf410 google.golang.org/protobuf v1.27.1 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b @@ -29,7 +30,6 @@ require ( github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/tidwall/gjson v1.13.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/sjson v1.2.4 // indirect diff --git a/go.sum b/go.sum index 0bc53ee..044bd50 100644 --- a/go.sum +++ b/go.sum @@ -140,8 +140,8 @@ github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc= github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= go.mau.fi/libsignal v0.0.0-20211109153248-a67163214910 h1:9FFhG0OmkuMau5UEaTgiUQ+7cSbtbOQ7hiWKdN8OI3I= go.mau.fi/libsignal v0.0.0-20211109153248-a67163214910/go.mod h1:AufGrvVh+00Nc07Jm4hTquh7yleZyn20tKJI2wCPAKg= -go.mau.fi/whatsmeow v0.0.0-20220128124639-e64fb976bf15 h1:BmdZu7K6IHsb+sPxvzkEjAINKxTMNeSiJRe1cvfesIY= -go.mau.fi/whatsmeow v0.0.0-20220128124639-e64fb976bf15/go.mod h1:8jUjOAi3xtGubxcZgG8uSHpAdyQXBRbWAfxkctX/4y4= +go.mau.fi/whatsmeow v0.0.0-20220204175019-e490de34933c h1:AVSYHQ0N5n3buL+thypCk2jiltD+3+pUQ7oPVhC7I3w= +go.mau.fi/whatsmeow v0.0.0-20220204175019-e490de34933c/go.mod h1:8jUjOAi3xtGubxcZgG8uSHpAdyQXBRbWAfxkctX/4y4= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/portal.go b/portal.go index 77f9a8e..72958d0 100644 --- a/portal.go +++ b/portal.go @@ -441,7 +441,7 @@ func formatDuration(d time.Duration) string { func (portal *Portal) convertMessage(intent *appservice.IntentAPI, source *User, info *types.MessageInfo, waMsg *waProto.Message) *ConvertedMessage { switch { case waMsg.Conversation != nil || waMsg.ExtendedTextMessage != nil: - return portal.convertTextMessage(intent, waMsg) + return portal.convertTextMessage(intent, source, waMsg) case waMsg.ImageMessage != nil: return portal.convertMediaMessage(intent, source, info, waMsg.GetImageMessage()) case waMsg.StickerMessage != nil: @@ -1491,7 +1491,7 @@ type ConvertedMessage struct { ExpiresIn uint32 } -func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, msg *waProto.Message) *ConvertedMessage { +func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, source *User, msg *waProto.Message) *ConvertedMessage { content := &event.MessageEventContent{ Body: msg.GetConversation(), MsgType: event.MsgText, @@ -1510,11 +1510,7 @@ func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, msg *waPr } expiresIn = contextInfo.GetExpiration() - preview := portal.convertUrlPreview(msg.GetExtendedTextMessage()) - - if preview != nil { - extraAttrs["com.beeper.linkpreview"] = preview - } + extraAttrs["com.beeper.linkpreview"] = portal.convertURLPreviewToBeeper(intent, source, msg.GetExtendedTextMessage()) } return &ConvertedMessage{ @@ -2001,10 +1997,10 @@ func (portal *Portal) convertMediaMessage(intent *appservice.IntentAPI, source * const thumbnailMaxSize = 72 const thumbnailMinSize = 24 -func createJPEGThumbnail(source []byte) ([]byte, error) { +func createJPEGThumbnailAndGetSize(source []byte) ([]byte, int, int, error) { src, _, err := image.Decode(bytes.NewReader(source)) if err != nil { - return nil, fmt.Errorf("failed to decode thumbnail: %w", err) + return nil, 0, 0, fmt.Errorf("failed to decode thumbnail: %w", err) } imageBounds := src.Bounds() width, height := imageBounds.Max.X, imageBounds.Max.Y @@ -2037,9 +2033,14 @@ func createJPEGThumbnail(source []byte) ([]byte, error) { var buf bytes.Buffer err = jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpeg.DefaultQuality}) if err != nil { - return nil, fmt.Errorf("failed to re-encode thumbnail: %w", err) + return nil, width, height, fmt.Errorf("failed to re-encode thumbnail: %w", err) } - return buf.Bytes(), nil + return buf.Bytes(), width, height, nil +} + +func createJPEGThumbnail(source []byte) ([]byte, error) { + data, _, _, err := createJPEGThumbnailAndGetSize(source) + return data, err } func (portal *Portal) downloadThumbnail(original []byte, thumbnailURL id.ContentURIString, eventID id.EventID) ([]byte, error) { @@ -2069,116 +2070,6 @@ func (portal *Portal) convertWebPtoPNG(webpImage []byte) ([]byte, error) { return pngBuffer.Bytes(), nil } -func (portal *Portal) convertUrlPreview(source *waProto.ExtendedTextMessage) map[string]interface{} { - if source == nil { - return nil - } - - matchedText := source.GetMatchedText() - - if matchedText == "" { - return nil - } - - canonicalUrl := source.GetCanonicalUrl() - - url := matchedText - if canonicalUrl != "" { - url = canonicalUrl - } - - result := map[string]interface{}{ - "og:title": source.GetTitle(), - "og:url": url, - "og:description": source.GetDescription(), - } - - if len(source.GetJpegThumbnail()) > 0 { - thumbnailMime := http.DetectContentType(source.GetJpegThumbnail()) - uploadedThumbnail, _ := portal.MainIntent().UploadBytes(source.GetJpegThumbnail(), thumbnailMime) - if uploadedThumbnail != nil { - cfg, _, _ := image.DecodeConfig(bytes.NewReader(source.GetJpegThumbnail())) - result["og:image"] = uploadedThumbnail.ContentURI.CUString() - result["og:image:width"] = cfg.Width - result["og:image:height"] = cfg.Height - result["og:image:type"] = thumbnailMime - } - } - - return result -} - -func (portal *Portal) updateExtendedMessageForUrlPreview(source *event.Content, dest *waProto.ExtendedTextMessage) { - if source == nil { - return - } - - embeddedLink, ok := source.Raw["com.beeper.linkpreview"].(map[string]interface{}) - - if !ok || embeddedLink == nil { - return - } - - matchedUrl, ok := embeddedLink["matchedUrl"].(string) - - if !ok || matchedUrl == "" { - return - } - - dest.MatchedText = &matchedUrl - - canonical, ok := embeddedLink["og:url"].(string) - - if ok { - dest.CanonicalUrl = &canonical - } - - description, ok := embeddedLink["og:description"].(string) - - if ok { - dest.Description = &description - } - - rawMXC, ok := embeddedLink["og:image"].(string) - - if !ok || rawMXC == "" { - return - } - - mxc, err := id.ParseContentURI(rawMXC) - if err != nil { - portal.log.Errorln("Malformed content URL %v: %v", rawMXC, err) - return - } - - data, err := portal.MainIntent().DownloadBytes(mxc) - if err != nil { - portal.log.Errorfln("Failed to download media from %s: %v", rawMXC, err) - return - } - - height, ok := embeddedLink["og:image:height"].(float64) - - if !ok { - portal.log.Errorfln("Height missing or invalid %v", embeddedLink["og:image:height"]) - return - } - - width, ok := embeddedLink["og:image:width"].(float64) - - if !ok { - portal.log.Errorfln("Width missing or invalid %v", embeddedLink["og:image:width"]) - return - } - - height32 := uint32(height) - width32 := uint32(width) - - dest.JpegThumbnail = data - dest.ThumbnailHeight = &height32 - dest.ThumbnailWidth = &width32 -} - func (portal *Portal) preprocessMatrixMedia(sender *User, relaybotFormatted bool, content *event.MessageEventContent, eventID id.EventID, mediaType whatsmeow.MediaType) *MediaUpload { var caption string var mentionedJIDs []string @@ -2368,7 +2259,7 @@ func (portal *Portal) convertMatrixMessage(sender *User, evt *event.Event) (*waP ContextInfo: &ctxInfo, } - portal.updateExtendedMessageForUrlPreview(&evt.Content, msg.ExtendedTextMessage) + portal.convertURLPreviewToWhatsApp(sender, evt, msg.ExtendedTextMessage) } else { msg.Conversation = &text } diff --git a/urlpreview.go b/urlpreview.go new file mode 100644 index 0000000..d3c8d5c --- /dev/null +++ b/urlpreview.go @@ -0,0 +1,155 @@ +// mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. +// Copyright (C) 2022 Tulir Asokan +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +import ( + "bytes" + "context" + "encoding/json" + "image" + "net/http" + + "github.com/tidwall/gjson" + "google.golang.org/protobuf/proto" + "maunium.net/go/mautrix/appservice" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" + + "go.mau.fi/whatsmeow" + waProto "go.mau.fi/whatsmeow/binary/proto" +) + +type BeeperLinkPreview struct { + MatchedURL string `json:"matched_url"` + CanonicalURL string `json:"og:url,omitempty"` + Title string `json:"og:title,omitempty"` + Type string `json:"og:type,omitempty"` + Description string `json:"og:description,omitempty"` + Image id.ContentURIString `json:"og:image,omitempty"` + ImageWidth int `json:"og:image:width,omitempty"` + ImageHeight int `json:"og:image:height,omitempty"` + + MatchedURLFallback string `json:"matchedUrl"` +} + +func (portal *Portal) convertURLPreviewToBeeper(intent *appservice.IntentAPI, source *User, msg *waProto.ExtendedTextMessage) (output *BeeperLinkPreview) { + if msg.GetMatchedText() == "" { + return + } + + output = &BeeperLinkPreview{ + MatchedURL: msg.GetMatchedText(), + CanonicalURL: msg.GetCanonicalUrl(), + Title: msg.GetTitle(), + Description: msg.GetDescription(), + } + output.MatchedURLFallback = output.MatchedURL + if len(output.CanonicalURL) == 0 { + output.CanonicalURL = output.MatchedURL + } + + var thumbnailData []byte + if msg.ThumbnailDirectPath != nil { + var err error + thumbnailData, err = source.Client.DownloadThumbnail(msg) + if err != nil { + portal.log.Warnfln("Failed to download thumbnail for link preview: %v", err) + } + } else if msg.JpegThumbnail != nil { + thumbnailData = msg.JpegThumbnail + } + if thumbnailData != nil { + mxc, err := intent.UploadBytes(thumbnailData, http.DetectContentType(thumbnailData)) + if err != nil { + portal.log.Warnfln("Failed to reupload thumbnail for link preview: %v", err) + } else { + output.Image = mxc.ContentURI.CUString() + output.ImageHeight = int(msg.GetThumbnailHeight()) + output.ImageWidth = int(msg.GetThumbnailWidth()) + if output.ImageHeight == 0 || output.ImageWidth == 0 { + src, _, err := image.Decode(bytes.NewReader(thumbnailData)) + if err == nil { + imageBounds := src.Bounds() + output.ImageWidth, output.ImageHeight = imageBounds.Max.X, imageBounds.Max.Y + } + } + } + } + if msg.GetPreviewType() == waProto.ExtendedTextMessage_VIDEO { + output.Type = "video.other" + } + + return +} + +func (portal *Portal) convertURLPreviewToWhatsApp(sender *User, evt *event.Event, dest *waProto.ExtendedTextMessage) { + rawPreview := gjson.GetBytes(evt.Content.VeryRaw, `com\.beeper\.linkpreview`) + if !rawPreview.Exists() || !rawPreview.IsObject() { + return + } + var preview BeeperLinkPreview + if err := json.Unmarshal([]byte(rawPreview.Raw), &preview); err != nil { + return + } + if len(preview.MatchedURL) == 0 { + if len(preview.MatchedURLFallback) == 0 { + return + } else { + preview.MatchedURL = preview.MatchedURLFallback + } + } + + dest.MatchedText = &preview.MatchedURL + if len(preview.CanonicalURL) > 0 { + dest.CanonicalUrl = &preview.CanonicalURL + } + if len(preview.Description) > 0 { + dest.Description = &preview.Description + } + if len(preview.Title) > 0 { + dest.Title = &preview.Title + } + imageMXC := preview.Image.ParseOrIgnore() + if !imageMXC.IsEmpty() { + data, err := portal.MainIntent().DownloadBytes(imageMXC) + if err != nil { + portal.log.Errorfln("Failed to download URL preview image %s: %v", preview.Image, err) + return + } + uploadResp, err := sender.Client.Upload(context.Background(), data, whatsmeow.MediaLinkThumbnail) + if err != nil { + portal.log.Errorfln("Failed to upload URL preview thumbnail in %s: %v", evt.ID, err) + return + } + dest.ThumbnailSha256 = uploadResp.FileSHA256 + dest.ThumbnailEncSha256 = uploadResp.FileEncSHA256 + dest.ThumbnailDirectPath = &uploadResp.DirectPath + dest.MediaKey = uploadResp.MediaKey + var width, height int + dest.JpegThumbnail, width, height, err = createJPEGThumbnailAndGetSize(data) + if err != nil { + portal.log.Warnfln("Failed to create JPEG thumbnail for URL preview in %s: %v", evt.ID, err) + } + if preview.ImageHeight > 0 && preview.ImageWidth > 0 { + dest.ThumbnailWidth = proto.Uint32(uint32(preview.ImageWidth)) + dest.ThumbnailHeight = proto.Uint32(uint32(preview.ImageHeight)) + } else if width > 0 && height > 0 { + dest.ThumbnailWidth = proto.Uint32(uint32(width)) + dest.ThumbnailHeight = proto.Uint32(uint32(height)) + } + } +} From 9fee8a50a44f1cd78da5206f94c4892d6f987e19 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 4 Feb 2022 23:06:35 +0200 Subject: [PATCH 4/4] Add support for encrypting preview image --- go.mod | 2 +- go.sum | 4 +-- urlpreview.go | 85 +++++++++++++++++++++++++++++++++------------------ 3 files changed, 58 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index 1fa3bd7..ff177ad 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/prometheus/client_golang v1.11.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/tidwall/gjson v1.13.0 - go.mau.fi/whatsmeow v0.0.0-20220204175019-e490de34933c + go.mau.fi/whatsmeow v0.0.0-20220204210537-a425ddb0b16c golang.org/x/image v0.0.0-20211028202545-6944b10bf410 google.golang.org/protobuf v1.27.1 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b diff --git a/go.sum b/go.sum index 044bd50..849f970 100644 --- a/go.sum +++ b/go.sum @@ -140,8 +140,8 @@ github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc= github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= go.mau.fi/libsignal v0.0.0-20211109153248-a67163214910 h1:9FFhG0OmkuMau5UEaTgiUQ+7cSbtbOQ7hiWKdN8OI3I= go.mau.fi/libsignal v0.0.0-20211109153248-a67163214910/go.mod h1:AufGrvVh+00Nc07Jm4hTquh7yleZyn20tKJI2wCPAKg= -go.mau.fi/whatsmeow v0.0.0-20220204175019-e490de34933c h1:AVSYHQ0N5n3buL+thypCk2jiltD+3+pUQ7oPVhC7I3w= -go.mau.fi/whatsmeow v0.0.0-20220204175019-e490de34933c/go.mod h1:8jUjOAi3xtGubxcZgG8uSHpAdyQXBRbWAfxkctX/4y4= +go.mau.fi/whatsmeow v0.0.0-20220204210537-a425ddb0b16c h1:hwuZ1W55J2uSwm029dREAr6crSVa+i5VsF91ltK389k= +go.mau.fi/whatsmeow v0.0.0-20220204210537-a425ddb0b16c/go.mod h1:8jUjOAi3xtGubxcZgG8uSHpAdyQXBRbWAfxkctX/4y4= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/urlpreview.go b/urlpreview.go index d3c8d5c..ff5b8d9 100644 --- a/urlpreview.go +++ b/urlpreview.go @@ -22,10 +22,13 @@ import ( "encoding/json" "image" "net/http" + "strings" + "time" "github.com/tidwall/gjson" "google.golang.org/protobuf/proto" "maunium.net/go/mautrix/appservice" + "maunium.net/go/mautrix/crypto/attachment" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" @@ -34,16 +37,19 @@ import ( ) type BeeperLinkPreview struct { - MatchedURL string `json:"matched_url"` - CanonicalURL string `json:"og:url,omitempty"` - Title string `json:"og:title,omitempty"` - Type string `json:"og:type,omitempty"` - Description string `json:"og:description,omitempty"` - Image id.ContentURIString `json:"og:image,omitempty"` - ImageWidth int `json:"og:image:width,omitempty"` - ImageHeight int `json:"og:image:height,omitempty"` + MatchedURL string `json:"matched_url"` + CanonicalURL string `json:"og:url,omitempty"` + Title string `json:"og:title,omitempty"` + Type string `json:"og:type,omitempty"` + Description string `json:"og:description,omitempty"` - MatchedURLFallback string `json:"matchedUrl"` + ImageURL id.ContentURIString `json:"og:image,omitempty"` + ImageEncryption *event.EncryptedFileInfo `json:"beeper:image:encryption,omitempty"` + + ImageSize int `json:"matrix:image:size,omitempty"` + ImageWidth int `json:"og:image:width,omitempty"` + ImageHeight int `json:"og:image:height,omitempty"` + ImageType string `json:"og:image:type,omitempty"` } func (portal *Portal) convertURLPreviewToBeeper(intent *appservice.IntentAPI, source *User, msg *waProto.ExtendedTextMessage) (output *BeeperLinkPreview) { @@ -57,7 +63,6 @@ func (portal *Portal) convertURLPreviewToBeeper(intent *appservice.IntentAPI, so Title: msg.GetTitle(), Description: msg.GetDescription(), } - output.MatchedURLFallback = output.MatchedURL if len(output.CanonicalURL) == 0 { output.CanonicalURL = output.MatchedURL } @@ -73,19 +78,32 @@ func (portal *Portal) convertURLPreviewToBeeper(intent *appservice.IntentAPI, so thumbnailData = msg.JpegThumbnail } if thumbnailData != nil { - mxc, err := intent.UploadBytes(thumbnailData, http.DetectContentType(thumbnailData)) + output.ImageHeight = int(msg.GetThumbnailHeight()) + output.ImageWidth = int(msg.GetThumbnailWidth()) + if output.ImageHeight == 0 || output.ImageWidth == 0 { + src, _, err := image.Decode(bytes.NewReader(thumbnailData)) + if err == nil { + imageBounds := src.Bounds() + output.ImageWidth, output.ImageHeight = imageBounds.Max.X, imageBounds.Max.Y + } + } + output.ImageSize = len(thumbnailData) + output.ImageType = http.DetectContentType(thumbnailData) + uploadData, uploadMime := thumbnailData, output.ImageType + if portal.Encrypted { + crypto := attachment.NewEncryptedFile() + uploadData = crypto.Encrypt(uploadData) + uploadMime = "application/octet-stream" + output.ImageEncryption = &event.EncryptedFileInfo{EncryptedFile: *crypto} + } + resp, err := intent.UploadBytes(uploadData, uploadMime) if err != nil { portal.log.Warnfln("Failed to reupload thumbnail for link preview: %v", err) } else { - output.Image = mxc.ContentURI.CUString() - output.ImageHeight = int(msg.GetThumbnailHeight()) - output.ImageWidth = int(msg.GetThumbnailWidth()) - if output.ImageHeight == 0 || output.ImageWidth == 0 { - src, _, err := image.Decode(bytes.NewReader(thumbnailData)) - if err == nil { - imageBounds := src.Bounds() - output.ImageWidth, output.ImageHeight = imageBounds.Max.X, imageBounds.Max.Y - } + if output.ImageEncryption != nil { + output.ImageEncryption.URL = resp.ContentURI.CUString() + } else { + output.ImageURL = resp.ContentURI.CUString() } } } @@ -102,16 +120,9 @@ func (portal *Portal) convertURLPreviewToWhatsApp(sender *User, evt *event.Event return } var preview BeeperLinkPreview - if err := json.Unmarshal([]byte(rawPreview.Raw), &preview); err != nil { + if err := json.Unmarshal([]byte(rawPreview.Raw), &preview); err != nil || len(preview.MatchedURL) == 0 { return } - if len(preview.MatchedURL) == 0 { - if len(preview.MatchedURLFallback) == 0 { - return - } else { - preview.MatchedURL = preview.MatchedURLFallback - } - } dest.MatchedText = &preview.MatchedURL if len(preview.CanonicalURL) > 0 { @@ -123,13 +134,27 @@ func (portal *Portal) convertURLPreviewToWhatsApp(sender *User, evt *event.Event if len(preview.Title) > 0 { dest.Title = &preview.Title } - imageMXC := preview.Image.ParseOrIgnore() + if strings.HasPrefix(preview.Type, "video.") { + dest.PreviewType = waProto.ExtendedTextMessage_VIDEO.Enum() + } + imageMXC := preview.ImageURL.ParseOrIgnore() + if preview.ImageEncryption != nil { + imageMXC = preview.ImageEncryption.URL.ParseOrIgnore() + } if !imageMXC.IsEmpty() { data, err := portal.MainIntent().DownloadBytes(imageMXC) if err != nil { - portal.log.Errorfln("Failed to download URL preview image %s: %v", preview.Image, err) + portal.log.Errorfln("Failed to download URL preview image %s in %s: %v", preview.ImageURL, evt.ID, err) return } + if preview.ImageEncryption != nil { + data, err = preview.ImageEncryption.Decrypt(data) + if err != nil { + portal.log.Errorfln("Failed to decrypt URL preview image in %s: %v", evt.ID, err) + return + } + } + dest.MediaKeyTimestamp = proto.Int64(time.Now().Unix()) uploadResp, err := sender.Client.Upload(context.Background(), data, whatsmeow.MediaLinkThumbnail) if err != nil { portal.log.Errorfln("Failed to upload URL preview thumbnail in %s: %v", evt.ID, err)