voice messages: bridge from WhatsApp to native Matrix voice messages

Co-authored-by: Tulir Asokan <tulir@maunium.net>
This commit is contained in:
Sumner Evans 2022-01-03 17:02:06 -07:00
parent 6607269e46
commit a0a1c0fd45
No known key found for this signature in database
GPG key ID: 8904527AB50022FD
3 changed files with 29 additions and 89 deletions

2
go.mod
View file

@ -14,7 +14,7 @@ require (
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
maunium.net/go/mauflag v1.0.0 maunium.net/go/mauflag v1.0.0
maunium.net/go/maulogger/v2 v2.3.2 maunium.net/go/maulogger/v2 v2.3.2
maunium.net/go/mautrix v0.10.9-0.20220104115646-3b28f2d770f5 maunium.net/go/mautrix v0.10.9-0.20220104174622-d2f80cb1e487
) )
require ( require (

4
go.sum
View file

@ -222,5 +222,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M=
maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA=
maunium.net/go/maulogger/v2 v2.3.2 h1:1XmIYmMd3PoQfp9J+PaHhpt80zpfmMqaShzUTC7FwY0= maunium.net/go/maulogger/v2 v2.3.2 h1:1XmIYmMd3PoQfp9J+PaHhpt80zpfmMqaShzUTC7FwY0=
maunium.net/go/maulogger/v2 v2.3.2/go.mod h1:TYWy7wKwz/tIXTpsx8G3mZseIRiC5DoMxSZazOHy68A= maunium.net/go/maulogger/v2 v2.3.2/go.mod h1:TYWy7wKwz/tIXTpsx8G3mZseIRiC5DoMxSZazOHy68A=
maunium.net/go/mautrix v0.10.9-0.20220104115646-3b28f2d770f5 h1:kHK8/Sc4ol6YfV0PCIqvv0gZ1FQ25fPvPiXZ6V3J6MM= maunium.net/go/mautrix v0.10.9-0.20220104174622-d2f80cb1e487 h1:cDpuyzkGDdNShKhT0xN14Ag0tG2Sp8H6rz1/TGxWEbI=
maunium.net/go/mautrix v0.10.9-0.20220104115646-3b28f2d770f5/go.mod h1:4XljZZGZiIlpfbQ+Tt2ykjapskJ8a7Z2i9y/+YaceF8= maunium.net/go/mautrix v0.10.9-0.20220104174622-d2f80cb1e487/go.mod h1:4XljZZGZiIlpfbQ+Tt2ykjapskJ8a7Z2i9y/+YaceF8=

112
portal.go
View file

@ -27,13 +27,9 @@ import (
_ "image/gif" _ "image/gif"
"image/jpeg" "image/jpeg"
"image/png" "image/png"
"io"
"math" "math"
"mime" "mime"
"net/http" "net/http"
"os"
"os/exec"
"path/filepath"
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
@ -51,6 +47,8 @@ import (
"maunium.net/go/mautrix/event" "maunium.net/go/mautrix/event"
"maunium.net/go/mautrix/format" "maunium.net/go/mautrix/format"
"maunium.net/go/mautrix/id" "maunium.net/go/mautrix/id"
"maunium.net/go/mautrix/util"
"maunium.net/go/mautrix/util/ffmpeg"
"go.mau.fi/whatsmeow" "go.mau.fi/whatsmeow"
waProto "go.mau.fi/whatsmeow/binary/proto" waProto "go.mau.fi/whatsmeow/binary/proto"
@ -1315,6 +1313,13 @@ func (portal *Portal) sendMessage(intent *appservice.IntentAPI, eventType event.
if err != nil { if err != nil {
return nil, err return nil, err
} }
if intent.IsCustomPuppet {
wrappedContent.Raw = map[string]interface{}{doublePuppetKey: doublePuppetValue}
} else {
wrappedContent.Raw = nil
}
_, _ = intent.UserTyping(portal.MXID, false, 0) _, _ = intent.UserTyping(portal.MXID, false, 0)
if timestamp == 0 { if timestamp == 0 {
return intent.SendMessageEvent(portal.MXID, eventType, &wrappedContent) return intent.SendMessageEvent(portal.MXID, eventType, &wrappedContent)
@ -1661,32 +1666,6 @@ type MediaMessageWithDuration interface {
GetSeconds() uint32 GetSeconds() uint32
} }
// MimeExtensionSanityOverrides includes extensions for various common mimetypes.
//
// This is necessary because sometimes the OS mimetype database and Go interact in weird ways,
// which causes very obscure extensions to be first in the array for common mimetypes
// (e.g. image/jpeg -> .jpe, text/plain -> ,v).
var MimeExtensionSanityOverrides = map[string]string{
"image/png": ".png",
"image/webp": ".webp",
"image/jpeg": ".jpg",
"image/tiff": ".tiff",
"image/heif": ".heic",
"image/heic": ".heic",
"audio/mpeg": ".mp3",
"audio/ogg": ".ogg",
"audio/webm": ".webm",
"video/mp4": ".mp4",
"video/mpeg": ".mpeg",
"video/webm": ".webm",
"text/plain": ".txt",
"text/html": ".html",
"application/xml": ".xml",
}
func (portal *Portal) convertMediaMessage(intent *appservice.IntentAPI, source *User, info *types.MessageInfo, msg MediaMessage) *ConvertedMessage { func (portal *Portal) convertMediaMessage(intent *appservice.IntentAPI, source *User, info *types.MessageInfo, msg MediaMessage) *ConvertedMessage {
messageWithCaption, ok := msg.(MediaMessageWithCaption) messageWithCaption, ok := msg.(MediaMessageWithCaption)
var captionContent *event.MessageEventContent var captionContent *event.MessageEventContent
@ -1763,14 +1742,7 @@ func (portal *Portal) convertMediaMessage(intent *appservice.IntentAPI, source *
content.Body = mimeClass content.Body = mimeClass
} }
ext, ok := MimeExtensionSanityOverrides[strings.Split(msg.GetMimetype(), ";")[0]] content.Body += util.ExtensionFromMimetype(msg.GetMimetype())
if !ok {
exts, _ := mime.ExtensionsByType(msg.GetMimetype())
if len(exts) > 0 {
ext = exts[0]
}
}
content.Body += ext
} }
msgWithDuration, ok := msg.(MediaMessageWithDuration) msgWithDuration, ok := msg.(MediaMessageWithDuration)
@ -1829,12 +1801,24 @@ func (portal *Portal) convertMediaMessage(intent *appservice.IntentAPI, source *
eventType = event.EventSticker eventType = event.EventSticker
} }
audioMessage, ok := msg.(*waProto.AudioMessage)
extraContent := map[string]interface{}{}
if ok {
extraContent["org.matrix.msc1767.audio"] = map[string]interface{}{
"duration": int(audioMessage.GetSeconds()) * 1000,
}
if audioMessage.GetPtt() {
extraContent["org.matrix.msc3245.voice"] = map[string]interface{}{}
}
}
return &ConvertedMessage{ return &ConvertedMessage{
Intent: intent, Intent: intent,
Type: eventType, Type: eventType,
Content: content, Content: content,
Caption: captionContent, Caption: captionContent,
ReplyTo: msg.GetContextInfo().GetStanzaId(), ReplyTo: msg.GetContextInfo().GetStanzaId(),
Extra: extraContent,
} }
} }
@ -1909,53 +1893,6 @@ func (portal *Portal) convertWebPtoPNG(webpImage []byte) ([]byte, error) {
return pngBuffer.Bytes(), nil return pngBuffer.Bytes(), nil
} }
func (portal *Portal) convertGifToVideo(gif []byte) ([]byte, error) {
dir, err := os.MkdirTemp("", "gif-convert-*")
if err != nil {
return nil, fmt.Errorf("failed to make temp dir: %w", err)
}
defer os.RemoveAll(dir)
inputFile, err := os.OpenFile(filepath.Join(dir, "input.gif"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if err != nil {
return nil, fmt.Errorf("failed open input file: %w", err)
}
_, err = inputFile.Write(gif)
if err != nil {
_ = inputFile.Close()
return nil, fmt.Errorf("failed to write gif to input file: %w", err)
}
_ = inputFile.Close()
outputFileName := filepath.Join(dir, "output.mp4")
cmd := exec.Command("ffmpeg", "-hide_banner", "-loglevel", "warning",
"-f", "gif", "-i", inputFile.Name(),
"-pix_fmt", "yuv420p", "-c:v", "libx264", "-movflags", "+faststart",
"-filter:v", "crop='floor(in_w/2)*2:floor(in_h/2)*2'",
outputFileName)
vcLog := portal.log.Sub("VideoConverter").Writer(log.LevelWarn)
cmd.Stdout = vcLog
cmd.Stderr = vcLog
err = cmd.Run()
if err != nil {
return nil, fmt.Errorf("failed to run ffmpeg: %w", err)
}
outputFile, err := os.OpenFile(filepath.Join(dir, "output.mp4"), os.O_RDONLY, 0)
if err != nil {
return nil, fmt.Errorf("failed to open output file: %w", err)
}
defer func() {
_ = outputFile.Close()
_ = os.Remove(outputFile.Name())
}()
mp4, err := io.ReadAll(outputFile)
if err != nil {
return nil, fmt.Errorf("failed to read mp4 from output file: %w", err)
}
return mp4, nil
}
func (portal *Portal) preprocessMatrixMedia(sender *User, relaybotFormatted bool, content *event.MessageEventContent, eventID id.EventID, mediaType whatsmeow.MediaType) *MediaUpload { func (portal *Portal) preprocessMatrixMedia(sender *User, relaybotFormatted bool, content *event.MessageEventContent, eventID id.EventID, mediaType whatsmeow.MediaType) *MediaUpload {
var caption string var caption string
var mentionedJIDs []string var mentionedJIDs []string
@ -1987,7 +1924,10 @@ func (portal *Portal) preprocessMatrixMedia(sender *User, relaybotFormatted bool
} }
} }
if mediaType == whatsmeow.MediaVideo && content.GetInfo().MimeType == "image/gif" { if mediaType == whatsmeow.MediaVideo && content.GetInfo().MimeType == "image/gif" {
data, err = portal.convertGifToVideo(data) data, err = ffmpeg.ConvertBytes(data, ".mp4", []string{"-f", "gif"}, []string{
"-pix_fmt", "yuv420p", "-c:v", "libx264", "-movflags", "+faststart",
"-filter:v", "crop='floor(in_w/2)*2:floor(in_h/2)*2'",
}, content.GetInfo().MimeType)
if err != nil { if err != nil {
portal.log.Errorfln("Failed to convert gif to mp4 in %s: %v", eventID, err) portal.log.Errorfln("Failed to convert gif to mp4 in %s: %v", eventID, err)
return nil return nil