forked from MirrorHub/mautrix-whatsapp
Merge pull request #435 from mautrix/adamvy/link-preview
Add support for bridging embedded link previews
This commit is contained in:
Tulir Asokan
GitHub
commit
04f8518ad9
4 changed files with 203 additions and 11 deletions
4
go.mod
4
go.mod
|
|
@ -9,7 +9,8 @@ require (
|
|||
github.com/mattn/go-sqlite3 v1.14.10
|
||||
github.com/prometheus/client_golang v1.11.0
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
|
||||
go.mau.fi/whatsmeow v0.0.0-20220128124639-e64fb976bf15
|
||||
github.com/tidwall/gjson v1.13.0
|
||||
go.mau.fi/whatsmeow v0.0.0-20220204210537-a425ddb0b16c
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410
|
||||
google.golang.org/protobuf v1.27.1
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
|
||||
|
|
@ -29,7 +30,6 @@ require (
|
|||
github.com/prometheus/common v0.26.0 // indirect
|
||||
github.com/prometheus/procfs v0.6.0 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/tidwall/gjson v1.13.0 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
github.com/tidwall/sjson v1.2.4 // indirect
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -140,8 +140,8 @@ github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc=
|
|||
github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM=
|
||||
go.mau.fi/libsignal v0.0.0-20211109153248-a67163214910 h1:9FFhG0OmkuMau5UEaTgiUQ+7cSbtbOQ7hiWKdN8OI3I=
|
||||
go.mau.fi/libsignal v0.0.0-20211109153248-a67163214910/go.mod h1:AufGrvVh+00Nc07Jm4hTquh7yleZyn20tKJI2wCPAKg=
|
||||
go.mau.fi/whatsmeow v0.0.0-20220128124639-e64fb976bf15 h1:BmdZu7K6IHsb+sPxvzkEjAINKxTMNeSiJRe1cvfesIY=
|
||||
go.mau.fi/whatsmeow v0.0.0-20220128124639-e64fb976bf15/go.mod h1:8jUjOAi3xtGubxcZgG8uSHpAdyQXBRbWAfxkctX/4y4=
|
||||
go.mau.fi/whatsmeow v0.0.0-20220204210537-a425ddb0b16c h1:hwuZ1W55J2uSwm029dREAr6crSVa+i5VsF91ltK389k=
|
||||
go.mau.fi/whatsmeow v0.0.0-20220204210537-a425ddb0b16c/go.mod h1:8jUjOAi3xtGubxcZgG8uSHpAdyQXBRbWAfxkctX/4y4=
|
||||
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
|
|
|
|||
26
portal.go
26
portal.go
|
|
@ -441,7 +441,7 @@ func formatDuration(d time.Duration) string {
|
|||
func (portal *Portal) convertMessage(intent *appservice.IntentAPI, source *User, info *types.MessageInfo, waMsg *waProto.Message) *ConvertedMessage {
|
||||
switch {
|
||||
case waMsg.Conversation != nil || waMsg.ExtendedTextMessage != nil:
|
||||
return portal.convertTextMessage(intent, waMsg)
|
||||
return portal.convertTextMessage(intent, source, waMsg)
|
||||
case waMsg.ImageMessage != nil:
|
||||
return portal.convertMediaMessage(intent, source, info, waMsg.GetImageMessage())
|
||||
case waMsg.StickerMessage != nil:
|
||||
|
|
@ -1491,13 +1491,15 @@ type ConvertedMessage struct {
|
|||
ExpiresIn uint32
|
||||
}
|
||||
|
||||
func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, msg *waProto.Message) *ConvertedMessage {
|
||||
func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, source *User, msg *waProto.Message) *ConvertedMessage {
|
||||
content := &event.MessageEventContent{
|
||||
Body: msg.GetConversation(),
|
||||
MsgType: event.MsgText,
|
||||
}
|
||||
var replyTo types.MessageID
|
||||
var expiresIn uint32
|
||||
extraAttrs := map[string]interface{}{}
|
||||
|
||||
if msg.GetExtendedTextMessage() != nil {
|
||||
content.Body = msg.GetExtendedTextMessage().GetText()
|
||||
|
||||
|
|
@ -1507,6 +1509,8 @@ func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, msg *waPr
|
|||
replyTo = contextInfo.GetStanzaId()
|
||||
}
|
||||
expiresIn = contextInfo.GetExpiration()
|
||||
|
||||
extraAttrs["com.beeper.linkpreview"] = portal.convertURLPreviewToBeeper(intent, source, msg.GetExtendedTextMessage())
|
||||
}
|
||||
|
||||
return &ConvertedMessage{
|
||||
|
|
@ -1515,6 +1519,7 @@ func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, msg *waPr
|
|||
Content: content,
|
||||
ReplyTo: replyTo,
|
||||
ExpiresIn: expiresIn,
|
||||
Extra: extraAttrs,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1992,10 +1997,10 @@ func (portal *Portal) convertMediaMessage(intent *appservice.IntentAPI, source *
|
|||
const thumbnailMaxSize = 72
|
||||
const thumbnailMinSize = 24
|
||||
|
||||
func createJPEGThumbnail(source []byte) ([]byte, error) {
|
||||
func createJPEGThumbnailAndGetSize(source []byte) ([]byte, int, int, error) {
|
||||
src, _, err := image.Decode(bytes.NewReader(source))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode thumbnail: %w", err)
|
||||
return nil, 0, 0, fmt.Errorf("failed to decode thumbnail: %w", err)
|
||||
}
|
||||
imageBounds := src.Bounds()
|
||||
width, height := imageBounds.Max.X, imageBounds.Max.Y
|
||||
|
|
@ -2028,9 +2033,14 @@ func createJPEGThumbnail(source []byte) ([]byte, error) {
|
|||
var buf bytes.Buffer
|
||||
err = jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpeg.DefaultQuality})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to re-encode thumbnail: %w", err)
|
||||
return nil, width, height, fmt.Errorf("failed to re-encode thumbnail: %w", err)
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
return buf.Bytes(), width, height, nil
|
||||
}
|
||||
|
||||
func createJPEGThumbnail(source []byte) ([]byte, error) {
|
||||
data, _, _, err := createJPEGThumbnailAndGetSize(source)
|
||||
return data, err
|
||||
}
|
||||
|
||||
func (portal *Portal) downloadThumbnail(original []byte, thumbnailURL id.ContentURIString, eventID id.EventID) ([]byte, error) {
|
||||
|
|
@ -2243,11 +2253,13 @@ func (portal *Portal) convertMatrixMessage(sender *User, evt *event.Event) (*waP
|
|||
if content.MsgType == event.MsgEmote && !relaybotFormatted {
|
||||
text = "/me " + text
|
||||
}
|
||||
if ctxInfo.StanzaId != nil || ctxInfo.MentionedJid != nil || ctxInfo.Expiration != nil {
|
||||
if ctxInfo.StanzaId != nil || ctxInfo.MentionedJid != nil || ctxInfo.Expiration != nil || evt.Content.Raw["com.beeper.linkpreview"] != nil {
|
||||
msg.ExtendedTextMessage = &waProto.ExtendedTextMessage{
|
||||
Text: &text,
|
||||
ContextInfo: &ctxInfo,
|
||||
}
|
||||
|
||||
portal.convertURLPreviewToWhatsApp(sender, evt, msg.ExtendedTextMessage)
|
||||
} else {
|
||||
msg.Conversation = &text
|
||||
}
|
||||
|
|
|
|||
180
urlpreview.go
Normal file
180
urlpreview.go
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
// mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge.
|
||||
// Copyright (C) 2022 Tulir Asokan
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"image"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/tidwall/gjson"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"maunium.net/go/mautrix/appservice"
|
||||
"maunium.net/go/mautrix/crypto/attachment"
|
||||
"maunium.net/go/mautrix/event"
|
||||
"maunium.net/go/mautrix/id"
|
||||
|
||||
"go.mau.fi/whatsmeow"
|
||||
waProto "go.mau.fi/whatsmeow/binary/proto"
|
||||
)
|
||||
|
||||
type BeeperLinkPreview struct {
|
||||
MatchedURL string `json:"matched_url"`
|
||||
CanonicalURL string `json:"og:url,omitempty"`
|
||||
Title string `json:"og:title,omitempty"`
|
||||
Type string `json:"og:type,omitempty"`
|
||||
Description string `json:"og:description,omitempty"`
|
||||
|
||||
ImageURL id.ContentURIString `json:"og:image,omitempty"`
|
||||
ImageEncryption *event.EncryptedFileInfo `json:"beeper:image:encryption,omitempty"`
|
||||
|
||||
ImageSize int `json:"matrix:image:size,omitempty"`
|
||||
ImageWidth int `json:"og:image:width,omitempty"`
|
||||
ImageHeight int `json:"og:image:height,omitempty"`
|
||||
ImageType string `json:"og:image:type,omitempty"`
|
||||
}
|
||||
|
||||
func (portal *Portal) convertURLPreviewToBeeper(intent *appservice.IntentAPI, source *User, msg *waProto.ExtendedTextMessage) (output *BeeperLinkPreview) {
|
||||
if msg.GetMatchedText() == "" {
|
||||
return
|
||||
}
|
||||
|
||||
output = &BeeperLinkPreview{
|
||||
MatchedURL: msg.GetMatchedText(),
|
||||
CanonicalURL: msg.GetCanonicalUrl(),
|
||||
Title: msg.GetTitle(),
|
||||
Description: msg.GetDescription(),
|
||||
}
|
||||
if len(output.CanonicalURL) == 0 {
|
||||
output.CanonicalURL = output.MatchedURL
|
||||
}
|
||||
|
||||
var thumbnailData []byte
|
||||
if msg.ThumbnailDirectPath != nil {
|
||||
var err error
|
||||
thumbnailData, err = source.Client.DownloadThumbnail(msg)
|
||||
if err != nil {
|
||||
portal.log.Warnfln("Failed to download thumbnail for link preview: %v", err)
|
||||
}
|
||||
} else if msg.JpegThumbnail != nil {
|
||||
thumbnailData = msg.JpegThumbnail
|
||||
}
|
||||
if thumbnailData != nil {
|
||||
output.ImageHeight = int(msg.GetThumbnailHeight())
|
||||
output.ImageWidth = int(msg.GetThumbnailWidth())
|
||||
if output.ImageHeight == 0 || output.ImageWidth == 0 {
|
||||
src, _, err := image.Decode(bytes.NewReader(thumbnailData))
|
||||
if err == nil {
|
||||
imageBounds := src.Bounds()
|
||||
output.ImageWidth, output.ImageHeight = imageBounds.Max.X, imageBounds.Max.Y
|
||||
}
|
||||
}
|
||||
output.ImageSize = len(thumbnailData)
|
||||
output.ImageType = http.DetectContentType(thumbnailData)
|
||||
uploadData, uploadMime := thumbnailData, output.ImageType
|
||||
if portal.Encrypted {
|
||||
crypto := attachment.NewEncryptedFile()
|
||||
uploadData = crypto.Encrypt(uploadData)
|
||||
uploadMime = "application/octet-stream"
|
||||
output.ImageEncryption = &event.EncryptedFileInfo{EncryptedFile: *crypto}
|
||||
}
|
||||
resp, err := intent.UploadBytes(uploadData, uploadMime)
|
||||
if err != nil {
|
||||
portal.log.Warnfln("Failed to reupload thumbnail for link preview: %v", err)
|
||||
} else {
|
||||
if output.ImageEncryption != nil {
|
||||
output.ImageEncryption.URL = resp.ContentURI.CUString()
|
||||
} else {
|
||||
output.ImageURL = resp.ContentURI.CUString()
|
||||
}
|
||||
}
|
||||
}
|
||||
if msg.GetPreviewType() == waProto.ExtendedTextMessage_VIDEO {
|
||||
output.Type = "video.other"
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (portal *Portal) convertURLPreviewToWhatsApp(sender *User, evt *event.Event, dest *waProto.ExtendedTextMessage) {
|
||||
rawPreview := gjson.GetBytes(evt.Content.VeryRaw, `com\.beeper\.linkpreview`)
|
||||
if !rawPreview.Exists() || !rawPreview.IsObject() {
|
||||
return
|
||||
}
|
||||
var preview BeeperLinkPreview
|
||||
if err := json.Unmarshal([]byte(rawPreview.Raw), &preview); err != nil || len(preview.MatchedURL) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
dest.MatchedText = &preview.MatchedURL
|
||||
if len(preview.CanonicalURL) > 0 {
|
||||
dest.CanonicalUrl = &preview.CanonicalURL
|
||||
}
|
||||
if len(preview.Description) > 0 {
|
||||
dest.Description = &preview.Description
|
||||
}
|
||||
if len(preview.Title) > 0 {
|
||||
dest.Title = &preview.Title
|
||||
}
|
||||
if strings.HasPrefix(preview.Type, "video.") {
|
||||
dest.PreviewType = waProto.ExtendedTextMessage_VIDEO.Enum()
|
||||
}
|
||||
imageMXC := preview.ImageURL.ParseOrIgnore()
|
||||
if preview.ImageEncryption != nil {
|
||||
imageMXC = preview.ImageEncryption.URL.ParseOrIgnore()
|
||||
}
|
||||
if !imageMXC.IsEmpty() {
|
||||
data, err := portal.MainIntent().DownloadBytes(imageMXC)
|
||||
if err != nil {
|
||||
portal.log.Errorfln("Failed to download URL preview image %s in %s: %v", preview.ImageURL, evt.ID, err)
|
||||
return
|
||||
}
|
||||
if preview.ImageEncryption != nil {
|
||||
data, err = preview.ImageEncryption.Decrypt(data)
|
||||
if err != nil {
|
||||
portal.log.Errorfln("Failed to decrypt URL preview image in %s: %v", evt.ID, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
dest.MediaKeyTimestamp = proto.Int64(time.Now().Unix())
|
||||
uploadResp, err := sender.Client.Upload(context.Background(), data, whatsmeow.MediaLinkThumbnail)
|
||||
if err != nil {
|
||||
portal.log.Errorfln("Failed to upload URL preview thumbnail in %s: %v", evt.ID, err)
|
||||
return
|
||||
}
|
||||
dest.ThumbnailSha256 = uploadResp.FileSHA256
|
||||
dest.ThumbnailEncSha256 = uploadResp.FileEncSHA256
|
||||
dest.ThumbnailDirectPath = &uploadResp.DirectPath
|
||||
dest.MediaKey = uploadResp.MediaKey
|
||||
var width, height int
|
||||
dest.JpegThumbnail, width, height, err = createJPEGThumbnailAndGetSize(data)
|
||||
if err != nil {
|
||||
portal.log.Warnfln("Failed to create JPEG thumbnail for URL preview in %s: %v", evt.ID, err)
|
||||
}
|
||||
if preview.ImageHeight > 0 && preview.ImageWidth > 0 {
|
||||
dest.ThumbnailWidth = proto.Uint32(uint32(preview.ImageWidth))
|
||||
dest.ThumbnailHeight = proto.Uint32(uint32(preview.ImageHeight))
|
||||
} else if width > 0 && height > 0 {
|
||||
dest.ThumbnailWidth = proto.Uint32(uint32(width))
|
||||
dest.ThumbnailHeight = proto.Uint32(uint32(height))
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue