directmedia: implement

This commit is contained in:
Tulir Asokan 2024-11-06 13:14:12 +01:00
parent 733dc1432a
commit 8dc2701194
13 changed files with 523 additions and 72 deletions

8
go.mod
View file

@ -9,16 +9,16 @@ require (
github.com/gorilla/websocket v1.5.0
github.com/lib/pq v1.10.9
github.com/rs/zerolog v1.33.0
go.mau.fi/util v0.8.1
go.mau.fi/util v0.8.2-0.20241106111346-576742786fe9
go.mau.fi/webp v0.1.0
go.mau.fi/whatsmeow v0.0.0-20241030164414-f98aea1881f6
go.mau.fi/whatsmeow v0.0.0-20241106153717-65ee2390b147
golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c
golang.org/x/image v0.21.0
golang.org/x/net v0.30.0
golang.org/x/sync v0.8.0
google.golang.org/protobuf v1.35.1
gopkg.in/yaml.v3 v3.0.1
maunium.net/go/mautrix v0.21.1
maunium.net/go/mautrix v0.21.2-0.20241106145856-449de115ffad
)
require (
@ -37,7 +37,7 @@ require (
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tidwall/sjson v1.2.5 // indirect
github.com/yuin/goldmark v1.7.7 // indirect
github.com/yuin/goldmark v1.7.8 // indirect
go.mau.fi/libsignal v0.1.1 // indirect
go.mau.fi/zeroconfig v0.1.3 // indirect
golang.org/x/crypto v0.28.0 // indirect

16
go.sum
View file

@ -61,16 +61,16 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
github.com/yuin/goldmark v1.7.7 h1:5m9rrB1sW3JUMToKFQfb+FGt1U7r57IHu5GrYrG2nqU=
github.com/yuin/goldmark v1.7.7/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic=
github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
go.mau.fi/libsignal v0.1.1 h1:m/0PGBh4QKP/I1MQ44ti4C0fMbLMuHb95cmDw01FIpI=
go.mau.fi/libsignal v0.1.1/go.mod h1:QLs89F/OA3ThdSL2Wz2p+o+fi8uuQUz0e1BRa6ExdBw=
go.mau.fi/util v0.8.1 h1:Ga43cz6esQBYqcjZ/onRoVnYWoUwjWbsxVeJg2jOTSo=
go.mau.fi/util v0.8.1/go.mod h1:T1u/rD2rzidVrBLyaUdPpZiJdP/rsyi+aTzn0D+Q6wc=
go.mau.fi/util v0.8.2-0.20241106111346-576742786fe9 h1:zYcb/lTZudowXAjKi6Yc2/2y5xxglPFfy9ZT2pNGsuM=
go.mau.fi/util v0.8.2-0.20241106111346-576742786fe9/go.mod h1:T1u/rD2rzidVrBLyaUdPpZiJdP/rsyi+aTzn0D+Q6wc=
go.mau.fi/webp v0.1.0 h1:BHObH/DcFntT9KYun5pDr0Ot4eUZO8k2C7eP7vF4ueA=
go.mau.fi/webp v0.1.0/go.mod h1:e42Z+VMFrUMS9cpEwGRIor+lQWO8oUAyPyMtcL+NMt8=
go.mau.fi/whatsmeow v0.0.0-20241030164414-f98aea1881f6 h1:ibChSQNQa6WTO+jUuJQz9x7qwCQoeIl/zlNCa/dAtvg=
go.mau.fi/whatsmeow v0.0.0-20241030164414-f98aea1881f6/go.mod h1:UvaXcdb8y5Mryj2LSXAMw7u4/exnWJIXn8Gvpmf6ndI=
go.mau.fi/whatsmeow v0.0.0-20241106153717-65ee2390b147 h1:IWKH0NL34DlR3P10yL6hNpsMnjsQajAvDejIyqrIAp4=
go.mau.fi/whatsmeow v0.0.0-20241106153717-65ee2390b147/go.mod h1:UvaXcdb8y5Mryj2LSXAMw7u4/exnWJIXn8Gvpmf6ndI=
go.mau.fi/zeroconfig v0.1.3 h1:As9wYDKmktjmNZW5i1vn8zvJlmGKHeVxHVIBMXsm4kM=
go.mau.fi/zeroconfig v0.1.3/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70=
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
@ -101,5 +101,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M=
maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA=
maunium.net/go/mautrix v0.21.1 h1:Z+e448jtlY977iC1kokNJTH5kg2WmDpcQCqn+v9oZOA=
maunium.net/go/mautrix v0.21.1/go.mod h1:7F/S6XAdyc/6DW+Q7xyFXRSPb6IjfqMb1OMepQ8C8OE=
maunium.net/go/mautrix v0.21.2-0.20241106145856-449de115ffad h1:+x7KwNbPT50ETQ0mGRkYME8mAX1QVaPdF4E21qBg1HA=
maunium.net/go/mautrix v0.21.2-0.20241106145856-449de115ffad/go.mod h1:UBuBMbPJfh1AqYc1K1Lr0eQclx5vs1k1iiLVO/iMyw4=

View file

@ -42,9 +42,10 @@ func (wa *WhatsAppConnector) LoadUserLogin(_ context.Context, login *bridgev2.Us
Main: wa,
UserLogin: login,
historySyncs: make(chan *waHistorySync.HistorySync, 64),
resyncQueue: make(map[types.JID]resyncQueueItem),
mediaRetryLock: semaphore.NewWeighted(wa.Config.HistorySync.MediaRequests.MaxAsyncHandle),
historySyncs: make(chan *waHistorySync.HistorySync, 64),
resyncQueue: make(map[types.JID]resyncQueueItem),
directMediaRetries: make(map[networkid.MessageID]*directMediaRetry),
mediaRetryLock: semaphore.NewWeighted(wa.Config.HistorySync.MediaRequests.MaxAsyncHandle),
}
login.Client = w
@ -87,12 +88,14 @@ type WhatsAppClient struct {
Device *store.Device
JID types.JID
historySyncs chan *waHistorySync.HistorySync
stopLoops atomic.Pointer[context.CancelFunc]
resyncQueue map[types.JID]resyncQueueItem
resyncQueueLock sync.Mutex
nextResync time.Time
mediaRetryLock *semaphore.Weighted
historySyncs chan *waHistorySync.HistorySync
stopLoops atomic.Pointer[context.CancelFunc]
resyncQueue map[types.JID]resyncQueueItem
resyncQueueLock sync.Mutex
nextResync time.Time
directMediaRetries map[networkid.MessageID]*directMediaRetry
directMediaLock sync.Mutex
mediaRetryLock *semaphore.Weighted
lastPhoneOfflineWarning time.Time
}

View file

@ -0,0 +1,223 @@
// mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge.
// Copyright (C) 2024 Tulir Asokan
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package connector
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"sync"
"time"
"github.com/rs/zerolog"
"go.mau.fi/util/exsync"
"go.mau.fi/whatsmeow"
"go.mau.fi/whatsmeow/proto/waMmsRetry"
"go.mau.fi/whatsmeow/types/events"
"maunium.net/go/mautrix"
"maunium.net/go/mautrix/bridgev2"
"maunium.net/go/mautrix/bridgev2/database"
"maunium.net/go/mautrix/bridgev2/networkid"
"maunium.net/go/mautrix/mediaproxy"
"go.mau.fi/mautrix-whatsapp/pkg/msgconv"
"go.mau.fi/mautrix-whatsapp/pkg/waid"
)
var _ bridgev2.DirectMediableNetwork = (*WhatsAppConnector)(nil)
func (wa *WhatsAppConnector) SetUseDirectMedia() {
wa.MsgConv.DirectMedia = true
}
var ErrReloadNeeded = mautrix.RespError{
ErrCode: "FI.MAU.WHATSAPP_RELOAD_NEEDED",
Err: "Media is no longer available on WhatsApp servers and must be re-requested from your phone",
StatusCode: http.StatusNotFound,
}
func (wa *WhatsAppConnector) Download(ctx context.Context, mediaID networkid.MediaID, params map[string]string) (mediaproxy.GetMediaResponse, error) {
parsedID, receiverID, err := waid.ParseMediaID(mediaID)
if err != nil {
return nil, err
}
msg, err := wa.Bridge.DB.Message.GetFirstPartByID(ctx, receiverID, parsedID.String())
if err != nil {
return nil, fmt.Errorf("failed to get message: %w", err)
} else if msg == nil {
return nil, fmt.Errorf("message not found")
}
dmm := msg.Metadata.(*waid.MessageMetadata).DirectMediaMeta
if dmm == nil {
return nil, fmt.Errorf("message does not have direct media metadata")
}
var keys *msgconv.FailedMediaKeys
err = json.Unmarshal(dmm, &keys)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal media keys: %w", err)
}
var ul *bridgev2.UserLogin
if receiverID != "" {
ul = wa.Bridge.GetCachedUserLoginByID(receiverID)
} else {
logins, err := wa.Bridge.GetUserLoginsInPortal(ctx, msg.Room)
if err != nil {
return nil, fmt.Errorf("failed to get user logins in portal: %w", err)
}
for _, login := range logins {
if login.Client.IsLoggedIn() {
ul = login
break
}
}
}
if ul == nil || !ul.Client.IsLoggedIn() {
return nil, fmt.Errorf("no logged in user found")
}
waClient := ul.Client.(*WhatsAppClient)
if waClient.Client == nil {
return nil, fmt.Errorf("no WhatsApp client found on login")
}
return &mediaproxy.GetMediaResponseFile{
Callback: func(f *os.File) error {
err := waClient.Client.DownloadToFile(keys, f)
if errors.Is(err, whatsmeow.ErrMediaDownloadFailedWith403) || errors.Is(err, whatsmeow.ErrMediaDownloadFailedWith404) || errors.Is(err, whatsmeow.ErrMediaDownloadFailedWith410) {
if _, noReload := params["fi.mau.whatsapp.no_reload_media"]; noReload {
return ErrReloadNeeded
}
err = waClient.requestAndWaitDirectMedia(ctx, msg.ID, keys)
if err != nil {
return err
}
err = waClient.Client.DownloadToFile(keys, f)
}
if errors.Is(err, whatsmeow.ErrFileLengthMismatch) || errors.Is(err, whatsmeow.ErrInvalidMediaSHA256) {
zerolog.Ctx(ctx).Warn().Err(err).Msg("Mismatching media checksums in message. Ignoring because WhatsApp seems to ignore them too")
} else if err != nil {
return err
}
return nil
},
// TODO?
ContentType: "",
}, nil
}
type directMediaRetry struct {
sync.Mutex
resultURL string
wait *exsync.Event
requested bool
resultType waMmsRetry.MediaRetryNotification_ResultType
}
func (wa *WhatsAppClient) getDirectMediaRetryState(msgID networkid.MessageID, create bool) *directMediaRetry {
wa.directMediaLock.Lock()
defer wa.directMediaLock.Unlock()
retry, ok := wa.directMediaRetries[msgID]
if !ok && create {
retry = &directMediaRetry{
wait: exsync.NewEvent(),
}
wa.directMediaRetries[msgID] = retry
}
return retry
}
func (wa *WhatsAppClient) requestAndWaitDirectMedia(ctx context.Context, rawMsgID networkid.MessageID, keys *msgconv.FailedMediaKeys) error {
state, err := wa.requestDirectMedia(rawMsgID, keys.Key)
if err != nil {
return err
}
select {
case <-state.wait.GetChan():
if state.resultURL != "" {
keys.DirectPath = state.resultURL
return nil
}
switch state.resultType {
case waMmsRetry.MediaRetryNotification_NOT_FOUND:
return mautrix.MNotFound.WithMessage("Media not found on phone")
default:
return mautrix.MNotFound.WithMessage("Phone returned error response")
}
case <-time.After(30 * time.Second):
return mautrix.MNotFound.WithMessage("Phone did not respond in time").WithStatus(http.StatusGatewayTimeout)
case <-ctx.Done():
return ctx.Err()
}
}
func (wa *WhatsAppClient) requestDirectMedia(rawMsgID networkid.MessageID, key []byte) (*directMediaRetry, error) {
state := wa.getDirectMediaRetryState(rawMsgID, true)
state.Lock()
defer state.Unlock()
if !state.requested {
err := wa.sendMediaRequestDirect(rawMsgID, key)
if err != nil {
return nil, fmt.Errorf("failed to send media retry request: %w", err)
}
state.requested = true
}
return state, nil
}
func (wa *WhatsAppClient) receiveDirectMediaRetry(ctx context.Context, msg *database.Message, retry *events.MediaRetry) {
state := wa.getDirectMediaRetryState(msg.ID, false)
if state != nil {
state.Lock()
defer func() {
state.wait.Set()
state.Unlock()
}()
}
log := zerolog.Ctx(ctx)
var keys msgconv.FailedMediaKeys
err := json.Unmarshal(msg.Metadata.(*waid.MessageMetadata).DirectMediaMeta, &keys)
if err != nil {
log.Err(err).Msg("Failed to parse direct media metadata for media retry")
return
}
retryData, err := whatsmeow.DecryptMediaRetryNotification(retry, keys.Key)
if err != nil {
log.Warn().Err(err).Msg("Failed to decrypt media retry notification")
return
}
state.resultType = retryData.GetResult()
if retryData.GetResult() != waMmsRetry.MediaRetryNotification_SUCCESS {
errorName := waMmsRetry.MediaRetryNotification_ResultType_name[int32(retryData.GetResult())]
if retryData.GetDirectPath() == "" {
log.Warn().Str("error_name", errorName).Msg("Got error response in media retry notification")
log.Debug().Any("error_content", retryData).Msg("Full error response content")
return
}
log.Debug().Msg("Got error response in media retry notification, but response also contains a new download URL")
}
keys.DirectPath = retryData.GetDirectPath()
msg.Metadata.(*waid.MessageMetadata).DirectMediaMeta, err = json.Marshal(keys)
if err != nil {
log.Err(err).Msg("Failed to marshal updated direct media metadata")
} else if err = wa.Main.Bridge.DB.Message.Update(ctx, msg); err != nil {
log.Err(err).Msg("Failed to update message with new direct media metadata")
}
if state != nil {
state.resultURL = retryData.GetDirectPath()
}
}

View file

@ -386,13 +386,16 @@ func (evt *WAMediaRetry) makeErrorEdit(part *database.Message, meta *msgconv.Pre
func (evt *WAMediaRetry) ConvertEdit(ctx context.Context, portal *bridgev2.Portal, intent bridgev2.MatrixAPI, existing []*database.Message) (*bridgev2.ConvertedEdit, error) {
meta := existing[0].Metadata.(*waid.MessageMetadata)
if meta.Error != waid.MsgErrMediaNotFound {
if meta.DirectMediaMeta != nil {
evt.wa.receiveDirectMediaRetry(ctx, existing[0], evt.MediaRetry)
return nil, fmt.Errorf("%w: direct media retry", bridgev2.ErrIgnoringRemoteEvent)
} else if meta.Error != waid.MsgErrMediaNotFound {
return nil, fmt.Errorf("%w: message doesn't have media error", bridgev2.ErrIgnoringRemoteEvent)
} else if meta.MediaMeta == nil {
} else if meta.FailedMediaMeta == nil {
return nil, fmt.Errorf("%w: message doesn't have media metadata", bridgev2.ErrIgnoringRemoteEvent)
}
var mediaMeta msgconv.PreparedMedia
err := json.Unmarshal(meta.MediaMeta, &mediaMeta)
err := json.Unmarshal(meta.FailedMediaMeta, &mediaMeta)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal media metadata: %w", err)
}

View file

@ -18,6 +18,7 @@ package connector
import (
"context"
"fmt"
"time"
"github.com/rs/zerolog"
@ -115,17 +116,9 @@ func (wa *WhatsAppClient) sendMediaRequests(ctx context.Context) {
}
func (wa *WhatsAppClient) sendMediaRequest(ctx context.Context, req *wadb.MediaRequest) {
msgID, err := waid.ParseMessageID(req.MessageID)
if err != nil {
err = wa.Main.DB.MediaRequest.Delete(ctx, wa.UserLogin.ID, req.MessageID)
if err != nil {
zerolog.Ctx(ctx).Err(err).Str("message_id", string(req.MessageID)).Msg("Failed to delete invalid media request")
}
return
}
log := zerolog.Ctx(ctx).With().Str("action", "send media request").Str("message_id", string(req.MessageID)).Logger()
defer func() {
err = wa.Main.DB.MediaRequest.Put(ctx, req)
err := wa.Main.DB.MediaRequest.Put(ctx, req)
if err != nil {
log.Err(err).Msg("Failed to save media request status")
}
@ -144,15 +137,7 @@ func (wa *WhatsAppClient) sendMediaRequest(ctx context.Context, req *wadb.MediaR
req.Status = wadb.MediaBackfillRequestStatusRequestSkipped
return
}
err = wa.Client.SendMediaRetryReceipt(&types.MessageInfo{
ID: msgID.ID,
MessageSource: types.MessageSource{
IsFromMe: msgID.Sender.User == wa.JID.User,
IsGroup: msgID.Chat.Server != types.DefaultUserServer,
Sender: msgID.Sender,
Chat: msgID.Chat,
},
}, req.MediaKey)
err = wa.sendMediaRequestDirect(req.MessageID, req.MediaKey)
if err != nil {
log.Err(err).Msg("Failed to send media retry request")
req.Status = wadb.MediaBackfillRequestStatusRequestFailed
@ -162,3 +147,19 @@ func (wa *WhatsAppClient) sendMediaRequest(ctx context.Context, req *wadb.MediaR
req.Status = wadb.MediaBackfillRequestStatusRequested
}
}
func (wa *WhatsAppClient) sendMediaRequestDirect(rawMsgID networkid.MessageID, key []byte) error {
msgID, err := waid.ParseMessageID(rawMsgID)
if err != nil {
return fmt.Errorf("failed to parse message ID: %w", err)
}
return wa.Client.SendMediaRetryReceipt(&types.MessageInfo{
ID: msgID.ID,
MessageSource: types.MessageSource{
IsFromMe: msgID.Sender.User == wa.JID.User,
IsGroup: msgID.Chat.Server != types.DefaultUserServer,
Sender: msgID.Sender,
Chat: msgID.Chat,
},
}, key)
}

View file

@ -134,21 +134,21 @@ func (mc *MessageConverter) ToMatrix(
case waMsg.EventMessage != nil:
part, contextInfo = mc.convertEventMessage(ctx, waMsg.EventMessage)
case waMsg.ImageMessage != nil:
part, contextInfo = mc.convertMediaMessage(ctx, waMsg.ImageMessage, "photo", isViewOnce, previouslyConvertedPart)
part, contextInfo = mc.convertMediaMessage(ctx, waMsg.ImageMessage, "photo", info, isViewOnce, previouslyConvertedPart)
case waMsg.StickerMessage != nil:
part, contextInfo = mc.convertMediaMessage(ctx, waMsg.StickerMessage, "sticker", isViewOnce, previouslyConvertedPart)
part, contextInfo = mc.convertMediaMessage(ctx, waMsg.StickerMessage, "sticker", info, isViewOnce, previouslyConvertedPart)
case waMsg.VideoMessage != nil:
part, contextInfo = mc.convertMediaMessage(ctx, waMsg.VideoMessage, "video attachment", isViewOnce, previouslyConvertedPart)
part, contextInfo = mc.convertMediaMessage(ctx, waMsg.VideoMessage, "video attachment", info, isViewOnce, previouslyConvertedPart)
case waMsg.PtvMessage != nil:
part, contextInfo = mc.convertMediaMessage(ctx, waMsg.PtvMessage, "video message", isViewOnce, previouslyConvertedPart)
part, contextInfo = mc.convertMediaMessage(ctx, waMsg.PtvMessage, "video message", info, isViewOnce, previouslyConvertedPart)
case waMsg.AudioMessage != nil:
typeName := "audio attachment"
if waMsg.AudioMessage.GetPTT() {
typeName = "voice message"
}
part, contextInfo = mc.convertMediaMessage(ctx, waMsg.AudioMessage, typeName, isViewOnce, previouslyConvertedPart)
part, contextInfo = mc.convertMediaMessage(ctx, waMsg.AudioMessage, typeName, info, isViewOnce, previouslyConvertedPart)
case waMsg.DocumentMessage != nil:
part, contextInfo = mc.convertMediaMessage(ctx, waMsg.DocumentMessage, "file attachment", isViewOnce, previouslyConvertedPart)
part, contextInfo = mc.convertMediaMessage(ctx, waMsg.DocumentMessage, "file attachment", info, isViewOnce, previouslyConvertedPart)
case waMsg.LocationMessage != nil:
part, contextInfo = mc.convertLocationMessage(ctx, waMsg.LocationMessage)
case waMsg.LiveLocationMessage != nil:

View file

@ -41,6 +41,7 @@ type MessageConverter struct {
FetchURLPreviews bool
ExtEvPolls bool
DisableViewOnce bool
DirectMedia bool
OldMediaSuffix string
}

View file

@ -70,11 +70,11 @@ func (mc *MessageConverter) convertTemplateMessage(ctx context.Context, info *ty
var convertedTitle *bridgev2.ConvertedMessagePart
switch title := tpl.GetTitle().(type) {
case *waE2E.TemplateMessage_HydratedFourRowTemplate_DocumentMessage:
convertedTitle, _ = mc.convertMediaMessage(ctx, title.DocumentMessage, "file attachment", false, nil)
convertedTitle, _ = mc.convertMediaMessage(ctx, title.DocumentMessage, "file attachment", info, false, nil)
case *waE2E.TemplateMessage_HydratedFourRowTemplate_ImageMessage:
convertedTitle, _ = mc.convertMediaMessage(ctx, title.ImageMessage, "photo", false, nil)
convertedTitle, _ = mc.convertMediaMessage(ctx, title.ImageMessage, "photo", info, false, nil)
case *waE2E.TemplateMessage_HydratedFourRowTemplate_VideoMessage:
convertedTitle, _ = mc.convertMediaMessage(ctx, title.VideoMessage, "video attachment", false, nil)
convertedTitle, _ = mc.convertMediaMessage(ctx, title.VideoMessage, "video attachment", info, false, nil)
case *waE2E.TemplateMessage_HydratedFourRowTemplate_LocationMessage:
content = fmt.Sprintf("Unsupported location message\n\n%s", content)
case *waE2E.TemplateMessage_HydratedFourRowTemplate_HydratedTitleText:

View file

@ -37,6 +37,7 @@ import (
"go.mau.fi/util/random"
"go.mau.fi/whatsmeow"
"go.mau.fi/whatsmeow/proto/waE2E"
"go.mau.fi/whatsmeow/types"
"maunium.net/go/mautrix/bridgev2"
"maunium.net/go/mautrix/bridgev2/database"
"maunium.net/go/mautrix/event"
@ -48,6 +49,7 @@ func (mc *MessageConverter) convertMediaMessage(
ctx context.Context,
msg MediaMessage,
typeName string,
messageInfo *types.MessageInfo,
isViewOnce bool,
cachedPart *bridgev2.ConvertedMessagePart,
) (part *bridgev2.ConvertedMessagePart, contextInfo *waE2E.ContextInfo) {
@ -72,15 +74,36 @@ func (mc *MessageConverter) convertMediaMessage(
cachedPart.Content.FormattedBody = preparedMedia.FormattedBody
return cachedPart, contextInfo
}
err := mc.reuploadWhatsAppAttachment(ctx, msg, preparedMedia)
if err != nil {
part = mc.makeMediaFailure(ctx, preparedMedia, &FailedMediaKeys{
Key: msg.GetMediaKey(),
Length: msg.GetFileLength(),
Type: whatsmeow.GetMediaType(msg),
SHA256: msg.GetFileSHA256(),
EncSHA256: msg.GetFileEncSHA256(),
}, err)
mediaKeys := &FailedMediaKeys{
Key: msg.GetMediaKey(),
Length: msg.GetFileLength(),
Type: whatsmeow.GetMediaType(msg),
SHA256: msg.GetFileSHA256(),
EncSHA256: msg.GetFileEncSHA256(),
}
if mc.DirectMedia {
preparedMedia.FillFileName()
var err error
portal := getPortal(ctx)
preparedMedia.URL, err = portal.Bridge.Matrix.GenerateContentURI(ctx, waid.MakeMediaID(messageInfo, portal.Receiver))
if err != nil {
panic(fmt.Errorf("failed to generate content URI: %w", err))
}
mediaKeys.DirectPath = msg.GetDirectPath()
directMediaMeta, err := json.Marshal(mediaKeys)
if err != nil {
panic(err)
}
part = &bridgev2.ConvertedMessagePart{
Type: preparedMedia.Type,
Content: preparedMedia.MessageEventContent,
Extra: preparedMedia.Extra,
DBMetadata: &waid.MessageMetadata{
DirectMediaMeta: directMediaMeta,
},
}
} else if err := mc.reuploadWhatsAppAttachment(ctx, msg, preparedMedia); err != nil {
part = mc.makeMediaFailure(ctx, preparedMedia, mediaKeys, err)
} else {
part = &bridgev2.ConvertedMessagePart{
Type: preparedMedia.Type,
@ -99,7 +122,7 @@ type FailedMediaKeys struct {
Type whatsmeow.MediaType `json:"type"`
SHA256 []byte `json:"sha256"`
EncSHA256 []byte `json:"enc_sha256"`
DirectPath string `json:"-"`
DirectPath string `json:"direct_path,omitempty"`
}
func (f *FailedMediaKeys) GetDirectPath() string {
@ -141,6 +164,13 @@ type PreparedMedia struct {
ContextInfo *waE2E.ContextInfo `json:"-"`
}
func (pm *PreparedMedia) FillFileName() *PreparedMedia {
if pm.FileName == "" {
pm.FileName = strings.TrimPrefix(string(pm.MsgType), "m.") + exmime.ExtensionFromMimetype(pm.Info.MimeType)
}
return pm
}
type MediaMessage interface {
whatsmeow.DownloadableMessage
GetContextInfo() *waE2E.ContextInfo
@ -289,7 +319,9 @@ func (mc *MessageConverter) reuploadWhatsAppAttachment(
var err error
part.URL, part.File, err = intent.UploadMediaStream(ctx, portal.MXID, -1, true, func(file io.Writer) (*bridgev2.FileStreamResult, error) {
err := client.DownloadToFile(message, file.(*os.File))
if err != nil {
if errors.Is(err, whatsmeow.ErrFileLengthMismatch) || errors.Is(err, whatsmeow.ErrInvalidMediaSHA256) {
zerolog.Ctx(ctx).Warn().Err(err).Msg("Mismatching media checksums in message. Ignoring because WhatsApp seems to ignore them too")
} else if err != nil {
return nil, fmt.Errorf("%w: %w", bridgev2.ErrMediaDownloadFailed, err)
}
if part.Info.MimeType == "" {
@ -297,9 +329,7 @@ func (mc *MessageConverter) reuploadWhatsAppAttachment(
n, _ := file.(*os.File).ReadAt(header, 0)
part.Info.MimeType = http.DetectContentType(header[:n])
}
if part.FileName == "" {
part.FileName = strings.TrimPrefix(string(part.MsgType), "m.") + exmime.ExtensionFromMimetype(part.Info.MimeType)
}
part.FillFileName()
return &bridgev2.FileStreamResult{
FileName: part.FileName,
MimeType: part.Info.MimeType,
@ -310,7 +340,9 @@ func (mc *MessageConverter) reuploadWhatsAppAttachment(
}
} else {
data, err := client.Download(message)
if err != nil {
if errors.Is(err, whatsmeow.ErrFileLengthMismatch) || errors.Is(err, whatsmeow.ErrInvalidMediaSHA256) {
zerolog.Ctx(ctx).Warn().Err(err).Msg("Mismatching media checksums in message. Ignoring because WhatsApp seems to ignore them too")
} else if err != nil {
return fmt.Errorf("%w: %w", bridgev2.ErrMediaDownloadFailed, err)
}
if part.Type == event.EventSticker && part.Info.MimeType == "application/was" {
@ -322,9 +354,7 @@ func (mc *MessageConverter) reuploadWhatsAppAttachment(
if part.Info.MimeType == "" {
part.Info.MimeType = http.DetectContentType(data)
}
if part.FileName == "" {
part.FileName = strings.TrimPrefix(string(part.MsgType), "m.") + exmime.ExtensionFromMimetype(part.Info.MimeType)
}
part.FillFileName()
part.URL, part.File, err = intent.UploadMedia(ctx, portal.MXID, data, part.FileName, part.Info.MimeType)
if err != nil {
return fmt.Errorf("%w: %w", bridgev2.ErrMediaReuploadFailed, err)
@ -434,6 +464,7 @@ func (mc *MessageConverter) makeMediaFailure(ctx context.Context, mediaInfo *Pre
errorMsg := fmt.Sprintf("Failed to bridge %s, please view it on the WhatsApp app", mediaInfo.TypeDescription)
if keys != nil && (errors.Is(err, whatsmeow.ErrMediaDownloadFailedWith403) || errors.Is(err, whatsmeow.ErrMediaDownloadFailedWith404) || errors.Is(err, whatsmeow.ErrMediaDownloadFailedWith410)) {
logLevel = zerolog.DebugLevel
keys.DirectPath = ""
mediaInfo.FailedKeys = keys
mediaInfo.MentionedJID = mediaInfo.ContextInfo.GetMentionedJID()
serializedMedia, serializerErr := json.Marshal(mediaInfo)
@ -444,8 +475,8 @@ func (mc *MessageConverter) makeMediaFailure(ctx context.Context, mediaInfo *Pre
FailedMediaField: mediaInfo,
}
dbMeta = &waid.MessageMetadata{
Error: waid.MsgErrMediaNotFound,
MediaMeta: serializedMedia,
Error: waid.MsgErrMediaNotFound,
FailedMediaMeta: serializedMedia,
}
errorMsg = fmt.Sprintf("Old %s. %s", mediaInfo.TypeDescription, mc.OldMediaSuffix)
}

View file

@ -70,7 +70,8 @@ type MessageMetadata struct {
Error MessageErrorType `json:"error,omitempty"`
BroadcastListJID *types.JID `json:"broadcast_list_jid,omitempty"`
GroupInvite *GroupInviteMeta `json:"group_invite,omitempty"`
MediaMeta json.RawMessage `json:"media_meta,omitempty"`
FailedMediaMeta json.RawMessage `json:"media_meta,omitempty"`
DirectMediaMeta json.RawMessage `json:"direct_media_meta,omitempty"`
IsMatrixPoll bool `json:"is_matrix_poll,omitempty"`
}

View file

@ -70,6 +70,10 @@ type ParsedMessageID struct {
ID types.MessageID
}
func (pmi *ParsedMessageID) String() networkid.MessageID {
return MakeMessageID(pmi.Chat, pmi.Sender, pmi.ID)
}
func ParseMessageID(messageID networkid.MessageID) (*ParsedMessageID, error) {
parts := strings.SplitN(string(messageID), ":", 3)
if len(parts) == 3 {

184
pkg/waid/mediaid.go Normal file
View file

@ -0,0 +1,184 @@
// mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge.
// Copyright (C) 2024 Tulir Asokan
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package waid
import (
"bytes"
"encoding/binary"
"encoding/hex"
"fmt"
"io"
"strconv"
"strings"
"go.mau.fi/whatsmeow/types"
"maunium.net/go/mautrix/bridgev2/networkid"
)
func MakeMediaID(messageInfo *types.MessageInfo, receiver networkid.UserLoginID) networkid.MediaID {
compactChat := compactJID(messageInfo.Chat.ToNonAD())
compactSender := compactJID(messageInfo.Sender.ToNonAD())
receiverID := compactJID(ParseUserLoginID(receiver, 0))
compactID := compactMsgID(messageInfo.ID)
mediaID := make([]byte, 0, 3+len(compactChat)+len(compactSender)+len(compactID))
mediaID = append(mediaID, byte(len(compactChat)))
mediaID = append(mediaID, compactChat...)
mediaID = append(mediaID, byte(len(compactSender)))
mediaID = append(mediaID, compactSender...)
mediaID = append(mediaID, byte(len(receiverID)))
mediaID = append(mediaID, receiverID...)
mediaID = append(mediaID, byte(len(compactID)))
mediaID = append(mediaID, compactID...)
return mediaID
}
func ParseMediaID(mediaID networkid.MediaID) (*ParsedMessageID, networkid.UserLoginID, error) {
reader := bytes.NewReader(mediaID)
chatJID, err := readCompact(reader, parseCompactJID)
if err != nil {
return nil, "", fmt.Errorf("failed to parse chat JID: %w", err)
}
senderJID, err := readCompact(reader, parseCompactJID)
if err != nil {
return nil, "", fmt.Errorf("failed to parse sender JID: %w", err)
}
receiverID, err := readCompact(reader, parseCompactJID)
if err != nil {
return nil, "", fmt.Errorf("failed to parse receiver JID: %w", err)
}
id, err := readCompact(reader, parseCompactMsgID)
if err != nil {
return nil, "", fmt.Errorf("failed to parse message ID: %w", err)
}
return &ParsedMessageID{
Chat: chatJID,
Sender: senderJID,
ID: id,
}, MakeUserLoginID(receiverID), nil
}
func isUpperHex(str string) bool {
for _, c := range str {
if (c < '0' || c > '9') && (c < 'A' || c > 'F') {
return false
}
}
return true
}
const (
compactJIDRawString = 0
compactJIDUserInt = 1
compactJIDUserDevice = 2
compactJIDGroupString = 3
compactMsgIDRawString = 0
compactMsgIDUpperHex = 1
)
func compactMsgID(id types.MessageID) []byte {
if isUpperHex(id) && len(id)%2 == 0 {
msgID, err := hex.AppendDecode([]byte{compactMsgIDUpperHex}, []byte(id))
if err == nil {
return msgID
}
}
return append([]byte{compactMsgIDRawString}, []byte(id)...)
}
func parseCompactMsgID(id []byte) (types.MessageID, error) {
if len(id) == 0 {
return "", fmt.Errorf("empty message ID")
}
switch id[0] {
case compactMsgIDRawString:
return types.MessageID(id[1:]), nil
case compactMsgIDUpperHex:
return types.MessageID(strings.ToUpper(hex.EncodeToString(id[1:]))), nil
default:
return "", fmt.Errorf("invalid compact message ID type")
}
}
func compactJID(jid types.JID) []byte {
if jid.Server == types.DefaultUserServer {
userInt, err := strconv.ParseUint(jid.User, 10, 64)
if err == nil {
if jid.Device == 0 {
return binary.BigEndian.AppendUint64([]byte{compactJIDUserInt}, userInt)
} else {
data := make([]byte, 1, 1+8+2)
data[1] = compactJIDUserDevice
data = binary.BigEndian.AppendUint64(data, userInt)
data = binary.BigEndian.AppendUint16(data, jid.Device)
return data
}
}
} else if jid.Server == types.GroupServer {
return append([]byte{compactJIDGroupString}, jid.User...)
}
return append([]byte{compactJIDRawString}, []byte(jid.String())...)
}
func parseCompactJID(jid []byte) (types.JID, error) {
if len(jid) == 0 {
return types.EmptyJID, fmt.Errorf("empty JID")
}
switch jid[0] {
case compactJIDRawString:
parsed, err := types.ParseJID(string(jid[1:]))
return parsed, err
case compactJIDUserInt:
if len(jid) != 9 {
return types.EmptyJID, fmt.Errorf("invalid compact user JID length")
}
return types.JID{
User: strconv.FormatUint(binary.BigEndian.Uint64(jid[1:]), 10),
Server: types.DefaultUserServer,
}, nil
case compactJIDUserDevice:
if len(jid) != 11 {
return types.EmptyJID, fmt.Errorf("invalid compact user device JID length")
}
return types.JID{
User: strconv.FormatUint(binary.BigEndian.Uint64(jid[1:]), 10),
Device: binary.BigEndian.Uint16(jid[9:]),
Server: types.DefaultUserServer,
}, nil
case compactJIDGroupString:
return types.JID{
User: string(jid[1:]),
Server: types.GroupServer,
}, nil
default:
return types.EmptyJID, fmt.Errorf("invalid compact JID type")
}
}
func readCompact[T any](reader *bytes.Reader, fn func(data []byte) (T, error)) (T, error) {
var defVal T
length, err := reader.ReadByte()
if err != nil {
return defVal, err
}
data := make([]byte, length)
_, err = io.ReadFull(reader, data)
if err != nil {
return defVal, err
}
return fn(data)
}