diff --git a/go.mod b/go.mod index 0f663ad..ff177ad 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,8 @@ require ( github.com/mattn/go-sqlite3 v1.14.10 github.com/prometheus/client_golang v1.11.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e - go.mau.fi/whatsmeow v0.0.0-20220128124639-e64fb976bf15 + github.com/tidwall/gjson v1.13.0 + go.mau.fi/whatsmeow v0.0.0-20220204210537-a425ddb0b16c golang.org/x/image v0.0.0-20211028202545-6944b10bf410 google.golang.org/protobuf v1.27.1 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b @@ -29,7 +30,6 @@ require ( github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/tidwall/gjson v1.13.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/sjson v1.2.4 // indirect diff --git a/go.sum b/go.sum index 0bc53ee..849f970 100644 --- a/go.sum +++ b/go.sum @@ -140,8 +140,8 @@ github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc= github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= go.mau.fi/libsignal v0.0.0-20211109153248-a67163214910 h1:9FFhG0OmkuMau5UEaTgiUQ+7cSbtbOQ7hiWKdN8OI3I= go.mau.fi/libsignal v0.0.0-20211109153248-a67163214910/go.mod h1:AufGrvVh+00Nc07Jm4hTquh7yleZyn20tKJI2wCPAKg= -go.mau.fi/whatsmeow v0.0.0-20220128124639-e64fb976bf15 h1:BmdZu7K6IHsb+sPxvzkEjAINKxTMNeSiJRe1cvfesIY= -go.mau.fi/whatsmeow v0.0.0-20220128124639-e64fb976bf15/go.mod h1:8jUjOAi3xtGubxcZgG8uSHpAdyQXBRbWAfxkctX/4y4= +go.mau.fi/whatsmeow v0.0.0-20220204210537-a425ddb0b16c h1:hwuZ1W55J2uSwm029dREAr6crSVa+i5VsF91ltK389k= +go.mau.fi/whatsmeow v0.0.0-20220204210537-a425ddb0b16c/go.mod h1:8jUjOAi3xtGubxcZgG8uSHpAdyQXBRbWAfxkctX/4y4= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/portal.go b/portal.go index cb0bbca..72958d0 100644 --- a/portal.go +++ b/portal.go @@ -441,7 +441,7 @@ func formatDuration(d time.Duration) string { func (portal *Portal) convertMessage(intent *appservice.IntentAPI, source *User, info *types.MessageInfo, waMsg *waProto.Message) *ConvertedMessage { switch { case waMsg.Conversation != nil || waMsg.ExtendedTextMessage != nil: - return portal.convertTextMessage(intent, waMsg) + return portal.convertTextMessage(intent, source, waMsg) case waMsg.ImageMessage != nil: return portal.convertMediaMessage(intent, source, info, waMsg.GetImageMessage()) case waMsg.StickerMessage != nil: @@ -1491,13 +1491,15 @@ type ConvertedMessage struct { ExpiresIn uint32 } -func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, msg *waProto.Message) *ConvertedMessage { +func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, source *User, msg *waProto.Message) *ConvertedMessage { content := &event.MessageEventContent{ Body: msg.GetConversation(), MsgType: event.MsgText, } var replyTo types.MessageID var expiresIn uint32 + extraAttrs := map[string]interface{}{} + if msg.GetExtendedTextMessage() != nil { content.Body = msg.GetExtendedTextMessage().GetText() @@ -1507,6 +1509,8 @@ func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, msg *waPr replyTo = contextInfo.GetStanzaId() } expiresIn = contextInfo.GetExpiration() + + extraAttrs["com.beeper.linkpreview"] = portal.convertURLPreviewToBeeper(intent, source, msg.GetExtendedTextMessage()) } return &ConvertedMessage{ @@ -1515,6 +1519,7 @@ func (portal *Portal) convertTextMessage(intent *appservice.IntentAPI, msg *waPr Content: content, ReplyTo: replyTo, ExpiresIn: expiresIn, + Extra: extraAttrs, } } @@ -1992,10 +1997,10 @@ func (portal *Portal) convertMediaMessage(intent *appservice.IntentAPI, source * const thumbnailMaxSize = 72 const thumbnailMinSize = 24 -func createJPEGThumbnail(source []byte) ([]byte, error) { +func createJPEGThumbnailAndGetSize(source []byte) ([]byte, int, int, error) { src, _, err := image.Decode(bytes.NewReader(source)) if err != nil { - return nil, fmt.Errorf("failed to decode thumbnail: %w", err) + return nil, 0, 0, fmt.Errorf("failed to decode thumbnail: %w", err) } imageBounds := src.Bounds() width, height := imageBounds.Max.X, imageBounds.Max.Y @@ -2028,9 +2033,14 @@ func createJPEGThumbnail(source []byte) ([]byte, error) { var buf bytes.Buffer err = jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpeg.DefaultQuality}) if err != nil { - return nil, fmt.Errorf("failed to re-encode thumbnail: %w", err) + return nil, width, height, fmt.Errorf("failed to re-encode thumbnail: %w", err) } - return buf.Bytes(), nil + return buf.Bytes(), width, height, nil +} + +func createJPEGThumbnail(source []byte) ([]byte, error) { + data, _, _, err := createJPEGThumbnailAndGetSize(source) + return data, err } func (portal *Portal) downloadThumbnail(original []byte, thumbnailURL id.ContentURIString, eventID id.EventID) ([]byte, error) { @@ -2243,11 +2253,13 @@ func (portal *Portal) convertMatrixMessage(sender *User, evt *event.Event) (*waP if content.MsgType == event.MsgEmote && !relaybotFormatted { text = "/me " + text } - if ctxInfo.StanzaId != nil || ctxInfo.MentionedJid != nil || ctxInfo.Expiration != nil { + if ctxInfo.StanzaId != nil || ctxInfo.MentionedJid != nil || ctxInfo.Expiration != nil || evt.Content.Raw["com.beeper.linkpreview"] != nil { msg.ExtendedTextMessage = &waProto.ExtendedTextMessage{ Text: &text, ContextInfo: &ctxInfo, } + + portal.convertURLPreviewToWhatsApp(sender, evt, msg.ExtendedTextMessage) } else { msg.Conversation = &text } diff --git a/urlpreview.go b/urlpreview.go new file mode 100644 index 0000000..ff5b8d9 --- /dev/null +++ b/urlpreview.go @@ -0,0 +1,180 @@ +// mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge. +// Copyright (C) 2022 Tulir Asokan +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +import ( + "bytes" + "context" + "encoding/json" + "image" + "net/http" + "strings" + "time" + + "github.com/tidwall/gjson" + "google.golang.org/protobuf/proto" + "maunium.net/go/mautrix/appservice" + "maunium.net/go/mautrix/crypto/attachment" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" + + "go.mau.fi/whatsmeow" + waProto "go.mau.fi/whatsmeow/binary/proto" +) + +type BeeperLinkPreview struct { + MatchedURL string `json:"matched_url"` + CanonicalURL string `json:"og:url,omitempty"` + Title string `json:"og:title,omitempty"` + Type string `json:"og:type,omitempty"` + Description string `json:"og:description,omitempty"` + + ImageURL id.ContentURIString `json:"og:image,omitempty"` + ImageEncryption *event.EncryptedFileInfo `json:"beeper:image:encryption,omitempty"` + + ImageSize int `json:"matrix:image:size,omitempty"` + ImageWidth int `json:"og:image:width,omitempty"` + ImageHeight int `json:"og:image:height,omitempty"` + ImageType string `json:"og:image:type,omitempty"` +} + +func (portal *Portal) convertURLPreviewToBeeper(intent *appservice.IntentAPI, source *User, msg *waProto.ExtendedTextMessage) (output *BeeperLinkPreview) { + if msg.GetMatchedText() == "" { + return + } + + output = &BeeperLinkPreview{ + MatchedURL: msg.GetMatchedText(), + CanonicalURL: msg.GetCanonicalUrl(), + Title: msg.GetTitle(), + Description: msg.GetDescription(), + } + if len(output.CanonicalURL) == 0 { + output.CanonicalURL = output.MatchedURL + } + + var thumbnailData []byte + if msg.ThumbnailDirectPath != nil { + var err error + thumbnailData, err = source.Client.DownloadThumbnail(msg) + if err != nil { + portal.log.Warnfln("Failed to download thumbnail for link preview: %v", err) + } + } else if msg.JpegThumbnail != nil { + thumbnailData = msg.JpegThumbnail + } + if thumbnailData != nil { + output.ImageHeight = int(msg.GetThumbnailHeight()) + output.ImageWidth = int(msg.GetThumbnailWidth()) + if output.ImageHeight == 0 || output.ImageWidth == 0 { + src, _, err := image.Decode(bytes.NewReader(thumbnailData)) + if err == nil { + imageBounds := src.Bounds() + output.ImageWidth, output.ImageHeight = imageBounds.Max.X, imageBounds.Max.Y + } + } + output.ImageSize = len(thumbnailData) + output.ImageType = http.DetectContentType(thumbnailData) + uploadData, uploadMime := thumbnailData, output.ImageType + if portal.Encrypted { + crypto := attachment.NewEncryptedFile() + uploadData = crypto.Encrypt(uploadData) + uploadMime = "application/octet-stream" + output.ImageEncryption = &event.EncryptedFileInfo{EncryptedFile: *crypto} + } + resp, err := intent.UploadBytes(uploadData, uploadMime) + if err != nil { + portal.log.Warnfln("Failed to reupload thumbnail for link preview: %v", err) + } else { + if output.ImageEncryption != nil { + output.ImageEncryption.URL = resp.ContentURI.CUString() + } else { + output.ImageURL = resp.ContentURI.CUString() + } + } + } + if msg.GetPreviewType() == waProto.ExtendedTextMessage_VIDEO { + output.Type = "video.other" + } + + return +} + +func (portal *Portal) convertURLPreviewToWhatsApp(sender *User, evt *event.Event, dest *waProto.ExtendedTextMessage) { + rawPreview := gjson.GetBytes(evt.Content.VeryRaw, `com\.beeper\.linkpreview`) + if !rawPreview.Exists() || !rawPreview.IsObject() { + return + } + var preview BeeperLinkPreview + if err := json.Unmarshal([]byte(rawPreview.Raw), &preview); err != nil || len(preview.MatchedURL) == 0 { + return + } + + dest.MatchedText = &preview.MatchedURL + if len(preview.CanonicalURL) > 0 { + dest.CanonicalUrl = &preview.CanonicalURL + } + if len(preview.Description) > 0 { + dest.Description = &preview.Description + } + if len(preview.Title) > 0 { + dest.Title = &preview.Title + } + if strings.HasPrefix(preview.Type, "video.") { + dest.PreviewType = waProto.ExtendedTextMessage_VIDEO.Enum() + } + imageMXC := preview.ImageURL.ParseOrIgnore() + if preview.ImageEncryption != nil { + imageMXC = preview.ImageEncryption.URL.ParseOrIgnore() + } + if !imageMXC.IsEmpty() { + data, err := portal.MainIntent().DownloadBytes(imageMXC) + if err != nil { + portal.log.Errorfln("Failed to download URL preview image %s in %s: %v", preview.ImageURL, evt.ID, err) + return + } + if preview.ImageEncryption != nil { + data, err = preview.ImageEncryption.Decrypt(data) + if err != nil { + portal.log.Errorfln("Failed to decrypt URL preview image in %s: %v", evt.ID, err) + return + } + } + dest.MediaKeyTimestamp = proto.Int64(time.Now().Unix()) + uploadResp, err := sender.Client.Upload(context.Background(), data, whatsmeow.MediaLinkThumbnail) + if err != nil { + portal.log.Errorfln("Failed to upload URL preview thumbnail in %s: %v", evt.ID, err) + return + } + dest.ThumbnailSha256 = uploadResp.FileSHA256 + dest.ThumbnailEncSha256 = uploadResp.FileEncSHA256 + dest.ThumbnailDirectPath = &uploadResp.DirectPath + dest.MediaKey = uploadResp.MediaKey + var width, height int + dest.JpegThumbnail, width, height, err = createJPEGThumbnailAndGetSize(data) + if err != nil { + portal.log.Warnfln("Failed to create JPEG thumbnail for URL preview in %s: %v", evt.ID, err) + } + if preview.ImageHeight > 0 && preview.ImageWidth > 0 { + dest.ThumbnailWidth = proto.Uint32(uint32(preview.ImageWidth)) + dest.ThumbnailHeight = proto.Uint32(uint32(preview.ImageHeight)) + } else if width > 0 && height > 0 { + dest.ThumbnailWidth = proto.Uint32(uint32(width)) + dest.ThumbnailHeight = proto.Uint32(uint32(height)) + } + } +}