MirrorHub/mautrix-whatsapp
0
0
Fork 1
mirror of https://github.com/tulir/mautrix-whatsapp synced 2024-12-14 01:14:29 +01:00
Code Issues Projects Releases Wiki Activity

Clean up embedded link preview code

Browse source
This commit is contained in:
Tulir Asokan 2022-02-04 22:19:55 +02:00
parent 779e591e60
commit d4334f5df8
4 changed files with 172 additions and 126 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
go.mod
View file

@ -9,7 +9,8 @@ require (
github.com/mattn/go-sqlite3 v1.14.10
github.com/prometheus/client_golang v1.11.0
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
go.mau.fi/whatsmeow v0.0.0-20220128124639-e64fb976bf15
github.com/tidwall/gjson v1.13.0
go.mau.fi/whatsmeow v0.0.0-20220204175019-e490de34933c
golang.org/x/image v0.0.0-20211028202545-6944b10bf410
google.golang.org/protobuf v1.27.1
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
@ -29,7 +30,6 @@ require (
github.com/prometheus/common v0.26.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/tidwall/gjson v1.13.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tidwall/sjson v1.2.4 // indirect

4
go.sum
View file

@ -140,8 +140,8 @@ github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc=
github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM=
go.mau.fi/libsignal v0.0.0-20211109153248-a67163214910 h1:9FFhG0OmkuMau5UEaTgiUQ+7cSbtbOQ7hiWKdN8OI3I=
go.mau.fi/libsignal v0.0.0-20211109153248-a67163214910/go.mod h1:AufGrvVh+00Nc07Jm4hTquh7yleZyn20tKJI2wCPAKg=
go.mau.fi/whatsmeow v0.0.0-20220128124639-e64fb976bf15 h1:BmdZu7K6IHsb+sPxvzkEjAINKxTMNeSiJRe1cvfesIY=
go.mau.fi/whatsmeow v0.0.0-20220128124639-e64fb976bf15/go.mod h1:8jUjOAi3xtGubxcZgG8uSHpAdyQXBRbWAfxkctX/4y4=
go.mau.fi/whatsmeow v0.0.0-20220204175019-e490de34933c h1:AVSYHQ0N5n3buL+thypCk2jiltD+3+pUQ7oPVhC7I3w=
go.mau.fi/whatsmeow v0.0.0-20220204175019-e490de34933c/go.mod h1:8jUjOAi3xtGubxcZgG8uSHpAdyQXBRbWAfxkctX/4y4=
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

135
portal.go
View file

@ -441,7 +441,7 @@ func formatDuration(d time.Duration) string {
func (portal *Portal) convertMessage(intent *appservice.IntentAPI, source *User, info *types.MessageInfo, waMsg *waProto.Message) *ConvertedMessage {
switch {
case waMsg.Conversation != nil || waMsg.ExtendedTextMessage != nil:
return portal.convertTextMessage(intent, waMsg)
return portal.convertTextMessage(intent, source, waMsg)
case waMsg.ImageMessage != nil:
return portal.convertMediaMessage(intent, source, info, waMsg.GetImageMessage())
case waMsg.StickerMessage != nil:
@ -1491,7 +1491,7 @@ type ConvertedMessage struct {
ExpiresIn uint32
}
func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, msg *waProto.Message) *ConvertedMessage {
func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, source *User, msg *waProto.Message) *ConvertedMessage {
content := &event.MessageEventContent{
Body: msg.GetConversation(),
MsgType: event.MsgText,
@ -1510,11 +1510,7 @@ func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, msg *waPr
}
expiresIn = contextInfo.GetExpiration()
preview := portal.convertUrlPreview(msg.GetExtendedTextMessage())
if preview != nil {
extraAttrs["com.beeper.linkpreview"] = preview
}
extraAttrs["com.beeper.linkpreview"] = portal.convertURLPreviewToBeeper(intent, source, msg.GetExtendedTextMessage())
}
return &ConvertedMessage{
@ -2001,10 +1997,10 @@ func (portal *Portal) convertMediaMessage(intent *appservice.IntentAPI, source *
const thumbnailMaxSize = 72
const thumbnailMinSize = 24
func createJPEGThumbnail(source []byte) ([]byte, error) {
func createJPEGThumbnailAndGetSize(source []byte) ([]byte, int, int, error) {
src, _, err := image.Decode(bytes.NewReader(source))
if err != nil {
return nil, fmt.Errorf("failed to decode thumbnail: %w", err)
return nil, 0, 0, fmt.Errorf("failed to decode thumbnail: %w", err)
}
imageBounds := src.Bounds()
width, height := imageBounds.Max.X, imageBounds.Max.Y
@ -2037,9 +2033,14 @@ func createJPEGThumbnail(source []byte) ([]byte, error) {
var buf bytes.Buffer
err = jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpeg.DefaultQuality})
if err != nil {
return nil, fmt.Errorf("failed to re-encode thumbnail: %w", err)
return nil, width, height, fmt.Errorf("failed to re-encode thumbnail: %w", err)
}
return buf.Bytes(), nil
return buf.Bytes(), width, height, nil
}
func createJPEGThumbnail(source []byte) ([]byte, error) {
data, _, _, err := createJPEGThumbnailAndGetSize(source)
return data, err
}
func (portal *Portal) downloadThumbnail(original []byte, thumbnailURL id.ContentURIString, eventID id.EventID) ([]byte, error) {
@ -2069,116 +2070,6 @@ func (portal *Portal) convertWebPtoPNG(webpImage []byte) ([]byte, error) {
return pngBuffer.Bytes(), nil
}
func (portal *Portal) convertUrlPreview(source *waProto.ExtendedTextMessage) map[string]interface{} {
if source == nil {
return nil
}
matchedText := source.GetMatchedText()
if matchedText == "" {
return nil
}
canonicalUrl := source.GetCanonicalUrl()
url := matchedText
if canonicalUrl != "" {
url = canonicalUrl
}
result := map[string]interface{}{
"og:title": source.GetTitle(),
"og:url": url,
"og:description": source.GetDescription(),
}
if len(source.GetJpegThumbnail()) > 0 {
thumbnailMime := http.DetectContentType(source.GetJpegThumbnail())
uploadedThumbnail, _ := portal.MainIntent().UploadBytes(source.GetJpegThumbnail(), thumbnailMime)
if uploadedThumbnail != nil {
cfg, _, _ := image.DecodeConfig(bytes.NewReader(source.GetJpegThumbnail()))
result["og:image"] = uploadedThumbnail.ContentURI.CUString()
result["og:image:width"] = cfg.Width
result["og:image:height"] = cfg.Height
result["og:image:type"] = thumbnailMime
}
}
return result
}
func (portal *Portal) updateExtendedMessageForUrlPreview(source *event.Content, dest *waProto.ExtendedTextMessage) {
if source == nil {
return
}
embeddedLink, ok := source.Raw["com.beeper.linkpreview"].(map[string]interface{})
if !ok || embeddedLink == nil {
return
}
matchedUrl, ok := embeddedLink["matchedUrl"].(string)
if !ok || matchedUrl == "" {
return
}
dest.MatchedText = &matchedUrl
canonical, ok := embeddedLink["og:url"].(string)
if ok {
dest.CanonicalUrl = &canonical
}
description, ok := embeddedLink["og:description"].(string)
if ok {
dest.Description = &description
}
rawMXC, ok := embeddedLink["og:image"].(string)
if !ok || rawMXC == "" {
return
}
mxc, err := id.ParseContentURI(rawMXC)
if err != nil {
portal.log.Errorln("Malformed content URL %v: %v", rawMXC, err)
return
}
data, err := portal.MainIntent().DownloadBytes(mxc)
if err != nil {
portal.log.Errorfln("Failed to download media from %s: %v", rawMXC, err)
return
}
height, ok := embeddedLink["og:image:height"].(float64)
if !ok {
portal.log.Errorfln("Height missing or invalid %v", embeddedLink["og:image:height"])
return
}
width, ok := embeddedLink["og:image:width"].(float64)
if !ok {
portal.log.Errorfln("Width missing or invalid %v", embeddedLink["og:image:width"])
return
}
height32 := uint32(height)
width32 := uint32(width)
dest.JpegThumbnail = data
dest.ThumbnailHeight = &height32
dest.ThumbnailWidth = &width32
}
func (portal *Portal) preprocessMatrixMedia(sender *User, relaybotFormatted bool, content *event.MessageEventContent, eventID id.EventID, mediaType whatsmeow.MediaType) *MediaUpload {
var caption string
var mentionedJIDs []string
@ -2368,7 +2259,7 @@ func (portal *Portal) convertMatrixMessage(sender *User, evt *event.Event) (*waP
ContextInfo: &ctxInfo,
}
portal.updateExtendedMessageForUrlPreview(&evt.Content, msg.ExtendedTextMessage)
portal.convertURLPreviewToWhatsApp(sender, evt, msg.ExtendedTextMessage)
} else {
msg.Conversation = &text
}

155
urlpreview.go Normal file
View file

@ -0,0 +1,155 @@
// mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge.
// Copyright (C) 2022 Tulir Asokan
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package main
import (
"bytes"
"context"
"encoding/json"
"image"
"net/http"
"github.com/tidwall/gjson"
"google.golang.org/protobuf/proto"
"maunium.net/go/mautrix/appservice"
"maunium.net/go/mautrix/event"
"maunium.net/go/mautrix/id"
"go.mau.fi/whatsmeow"
waProto "go.mau.fi/whatsmeow/binary/proto"
)
type BeeperLinkPreview struct {
MatchedURL string `json:"matched_url"`
CanonicalURL string `json:"og:url,omitempty"`
Title string `json:"og:title,omitempty"`
Type string `json:"og:type,omitempty"`
Description string `json:"og:description,omitempty"`
Image id.ContentURIString `json:"og:image,omitempty"`
ImageWidth int `json:"og:image:width,omitempty"`
ImageHeight int `json:"og:image:height,omitempty"`
MatchedURLFallback string `json:"matchedUrl"`
}
func (portal *Portal) convertURLPreviewToBeeper(intent *appservice.IntentAPI, source *User, msg *waProto.ExtendedTextMessage) (output *BeeperLinkPreview) {
if msg.GetMatchedText() == "" {
return
}
output = &BeeperLinkPreview{
MatchedURL: msg.GetMatchedText(),
CanonicalURL: msg.GetCanonicalUrl(),
Title: msg.GetTitle(),
Description: msg.GetDescription(),
}
output.MatchedURLFallback = output.MatchedURL
if len(output.CanonicalURL) == 0 {
output.CanonicalURL = output.MatchedURL
}
var thumbnailData []byte
if msg.ThumbnailDirectPath != nil {
var err error
thumbnailData, err = source.Client.DownloadThumbnail(msg)
if err != nil {
portal.log.Warnfln("Failed to download thumbnail for link preview: %v", err)
}
} else if msg.JpegThumbnail != nil {
thumbnailData = msg.JpegThumbnail
}
if thumbnailData != nil {
mxc, err := intent.UploadBytes(thumbnailData, http.DetectContentType(thumbnailData))
if err != nil {
portal.log.Warnfln("Failed to reupload thumbnail for link preview: %v", err)
} else {
output.Image = mxc.ContentURI.CUString()
output.ImageHeight = int(msg.GetThumbnailHeight())
output.ImageWidth = int(msg.GetThumbnailWidth())
if output.ImageHeight == 0 || output.ImageWidth == 0 {
src, _, err := image.Decode(bytes.NewReader(thumbnailData))
if err == nil {
imageBounds := src.Bounds()
output.ImageWidth, output.ImageHeight = imageBounds.Max.X, imageBounds.Max.Y
}
}
}
}
if msg.GetPreviewType() == waProto.ExtendedTextMessage_VIDEO {
output.Type = "video.other"
}
return
}
func (portal *Portal) convertURLPreviewToWhatsApp(sender *User, evt *event.Event, dest *waProto.ExtendedTextMessage) {
rawPreview := gjson.GetBytes(evt.Content.VeryRaw, `com\.beeper\.linkpreview`)
if !rawPreview.Exists() || !rawPreview.IsObject() {
return
}
var preview BeeperLinkPreview
if err := json.Unmarshal([]byte(rawPreview.Raw), &preview); err != nil {
return
}
if len(preview.MatchedURL) == 0 {
if len(preview.MatchedURLFallback) == 0 {
return
} else {
preview.MatchedURL = preview.MatchedURLFallback
}
}
dest.MatchedText = &preview.MatchedURL
if len(preview.CanonicalURL) > 0 {
dest.CanonicalUrl = &preview.CanonicalURL
}
if len(preview.Description) > 0 {
dest.Description = &preview.Description
}
if len(preview.Title) > 0 {
dest.Title = &preview.Title
}
imageMXC := preview.Image.ParseOrIgnore()
if !imageMXC.IsEmpty() {
data, err := portal.MainIntent().DownloadBytes(imageMXC)
if err != nil {
portal.log.Errorfln("Failed to download URL preview image %s: %v", preview.Image, err)
return
}
uploadResp, err := sender.Client.Upload(context.Background(), data, whatsmeow.MediaLinkThumbnail)
if err != nil {
portal.log.Errorfln("Failed to upload URL preview thumbnail in %s: %v", evt.ID, err)
return
}
dest.ThumbnailSha256 = uploadResp.FileSHA256
dest.ThumbnailEncSha256 = uploadResp.FileEncSHA256
dest.ThumbnailDirectPath = &uploadResp.DirectPath
dest.MediaKey = uploadResp.MediaKey
var width, height int
dest.JpegThumbnail, width, height, err = createJPEGThumbnailAndGetSize(data)
if err != nil {
portal.log.Warnfln("Failed to create JPEG thumbnail for URL preview in %s: %v", evt.ID, err)
}
if preview.ImageHeight > 0 && preview.ImageWidth > 0 {
dest.ThumbnailWidth = proto.Uint32(uint32(preview.ImageWidth))
dest.ThumbnailHeight = proto.Uint32(uint32(preview.ImageHeight))
} else if width > 0 && height > 0 {
dest.ThumbnailWidth = proto.Uint32(uint32(width))
dest.ThumbnailHeight = proto.Uint32(uint32(height))
}
}
}