mirror of
https://github.com/tulir/mautrix-whatsapp
synced 2024-12-14 09:23:51 +01:00
Merge pull request #484 from mautrix/sumner/bri-3057
backfill: fixed bug where the media backfill loop would sleep too often
This commit is contained in:
commit
86840dec59
2 changed files with 20 additions and 33 deletions
|
@ -32,30 +32,13 @@ type BackfillQueue struct {
|
||||||
log log.Logger
|
log log.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
func (bq *BackfillQueue) RunLoops(user *User) {
|
// Immediate backfills should happen first, then deferred backfills and lastly
|
||||||
go bq.immediateBackfillLoop(user)
|
// media backfills.
|
||||||
bq.deferredBackfillLoop(user)
|
func (bq *BackfillQueue) RunLoop(user *User) {
|
||||||
}
|
|
||||||
|
|
||||||
func (bq *BackfillQueue) immediateBackfillLoop(user *User) {
|
|
||||||
for {
|
for {
|
||||||
if backfill := bq.BackfillQuery.GetNext(user.MXID, database.BackfillImmediate); backfill != nil {
|
|
||||||
bq.ImmediateBackfillRequests <- backfill
|
|
||||||
backfill.MarkDone()
|
|
||||||
} else {
|
|
||||||
select {
|
|
||||||
case <-bq.ReCheckQueue:
|
|
||||||
case <-time.After(10 * time.Second):
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (bq *BackfillQueue) deferredBackfillLoop(user *User) {
|
|
||||||
for {
|
|
||||||
// Finish all immediate backfills before doing the deferred ones.
|
|
||||||
if immediate := bq.BackfillQuery.GetNext(user.MXID, database.BackfillImmediate); immediate != nil {
|
if immediate := bq.BackfillQuery.GetNext(user.MXID, database.BackfillImmediate); immediate != nil {
|
||||||
time.Sleep(10 * time.Second)
|
bq.ImmediateBackfillRequests <- immediate
|
||||||
|
immediate.MarkDone()
|
||||||
} else if backfill := bq.BackfillQuery.GetNext(user.MXID, database.BackfillDeferred); backfill != nil {
|
} else if backfill := bq.BackfillQuery.GetNext(user.MXID, database.BackfillDeferred); backfill != nil {
|
||||||
bq.DeferredBackfillRequests <- backfill
|
bq.DeferredBackfillRequests <- backfill
|
||||||
backfill.MarkDone()
|
backfill.MarkDone()
|
||||||
|
@ -63,7 +46,10 @@ func (bq *BackfillQueue) deferredBackfillLoop(user *User) {
|
||||||
bq.DeferredBackfillRequests <- mediaBackfill
|
bq.DeferredBackfillRequests <- mediaBackfill
|
||||||
mediaBackfill.MarkDone()
|
mediaBackfill.MarkDone()
|
||||||
} else {
|
} else {
|
||||||
time.Sleep(10 * time.Second)
|
select {
|
||||||
|
case <-bq.ReCheckQueue:
|
||||||
|
case <-time.After(time.Minute):
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -67,7 +67,7 @@ func (user *User) handleHistorySyncsLoop() {
|
||||||
// overload the homeserver. Users can configure their backfill stages
|
// overload the homeserver. Users can configure their backfill stages
|
||||||
// to be more or less aggressive with backfilling at this stage.
|
// to be more or less aggressive with backfilling at this stage.
|
||||||
go user.handleBackfillRequestsLoop(user.BackfillQueue.DeferredBackfillRequests)
|
go user.handleBackfillRequestsLoop(user.BackfillQueue.DeferredBackfillRequests)
|
||||||
go user.BackfillQueue.RunLoops(user)
|
go user.BackfillQueue.RunLoop(user)
|
||||||
|
|
||||||
// Always save the history syncs for the user. If they want to enable
|
// Always save the history syncs for the user. If they want to enable
|
||||||
// backfilling in the future, we will have it in the database.
|
// backfilling in the future, we will have it in the database.
|
||||||
|
@ -78,7 +78,7 @@ func (user *User) handleHistorySyncsLoop() {
|
||||||
|
|
||||||
func (user *User) handleBackfillRequestsLoop(backfillRequests chan *database.Backfill) {
|
func (user *User) handleBackfillRequestsLoop(backfillRequests chan *database.Backfill) {
|
||||||
for req := range backfillRequests {
|
for req := range backfillRequests {
|
||||||
user.log.Debugfln("Handling backfill request %s", req)
|
user.log.Infofln("Handling backfill request %s", req)
|
||||||
conv := user.bridge.DB.HistorySyncQuery.GetConversation(user.MXID, req.Portal)
|
conv := user.bridge.DB.HistorySyncQuery.GetConversation(user.MXID, req.Portal)
|
||||||
if conv == nil {
|
if conv == nil {
|
||||||
user.log.Debugfln("Could not find history sync conversation data for %s", req.Portal.String())
|
user.log.Debugfln("Could not find history sync conversation data for %s", req.Portal.String())
|
||||||
|
@ -96,16 +96,17 @@ func (user *User) handleBackfillRequestsLoop(backfillRequests chan *database.Bac
|
||||||
endTime = *req.TimeEnd
|
endTime = *req.TimeEnd
|
||||||
}
|
}
|
||||||
|
|
||||||
user.log.Debugfln("Backfilling media from %v to %v for %s", startTime, endTime, portal.Key.String())
|
user.log.Infofln("Backfilling media from %v to %v for %s", startTime, endTime, portal.Key.String())
|
||||||
|
|
||||||
// Go through all of the messages in the given time range,
|
// Go through all of the messages in the given time range,
|
||||||
// requesting any media that errored.
|
// requesting any media that errored.
|
||||||
requested := 0
|
requested := 0
|
||||||
for _, msg := range user.bridge.DB.Message.GetMessagesBetween(portal.Key, startTime, endTime) {
|
for _, msg := range user.bridge.DB.Message.GetMessagesBetween(portal.Key, startTime, endTime) {
|
||||||
if requested > 0 && requested%req.MaxBatchEvents == 0 {
|
|
||||||
time.Sleep(time.Duration(req.BatchDelay) * time.Second)
|
|
||||||
}
|
|
||||||
if msg.Error == database.MsgErrMediaNotFound {
|
if msg.Error == database.MsgErrMediaNotFound {
|
||||||
|
if requested > 0 && requested%req.MaxBatchEvents == 0 {
|
||||||
|
time.Sleep(time.Duration(req.BatchDelay) * time.Second)
|
||||||
|
}
|
||||||
|
|
||||||
portal.requestMediaRetry(user, msg.MXID)
|
portal.requestMediaRetry(user, msg.MXID)
|
||||||
requested += 1
|
requested += 1
|
||||||
}
|
}
|
||||||
|
@ -155,7 +156,7 @@ func (user *User) createOrUpdatePortalAndBackfillWithLock(req *database.Backfill
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
user.log.Debugfln("Backfilling %d messages in %s, %d messages at a time", len(allMsgs), portal.Key.JID, req.MaxBatchEvents)
|
user.log.Infofln("Backfilling %d messages in %s, %d messages at a time (queue ID: %d)", len(allMsgs), portal.Key.JID, req.MaxBatchEvents, req.QueueID)
|
||||||
toBackfill := allMsgs[0:]
|
toBackfill := allMsgs[0:]
|
||||||
var insertionEventIds []id.EventID
|
var insertionEventIds []id.EventID
|
||||||
for len(toBackfill) > 0 {
|
for len(toBackfill) > 0 {
|
||||||
|
@ -180,10 +181,10 @@ func (user *User) createOrUpdatePortalAndBackfillWithLock(req *database.Backfill
|
||||||
time.Unix(int64(allMsgs[0].GetMessageTimestamp()), 0),
|
time.Unix(int64(allMsgs[0].GetMessageTimestamp()), 0),
|
||||||
insertionEventIds[0])
|
insertionEventIds[0])
|
||||||
}
|
}
|
||||||
user.log.Debugfln("Deleting %d history sync messages after backfilling", len(allMsgs))
|
user.log.Debugfln("Deleting %d history sync messages after backfilling (queue ID: %d)", len(allMsgs), req.QueueID)
|
||||||
err := user.bridge.DB.HistorySyncQuery.DeleteMessages(user.MXID, conv.ConversationID, allMsgs)
|
err := user.bridge.DB.HistorySyncQuery.DeleteMessages(user.MXID, conv.ConversationID, allMsgs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
user.log.Warnfln("Failed to delete %d history sync messages after backfilling: %v", len(allMsgs), err)
|
user.log.Warnfln("Failed to delete %d history sync messages after backfilling (queue ID: %d): %v", len(allMsgs), req.QueueID, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !conv.MarkedAsUnread && conv.UnreadCount == 0 {
|
if !conv.MarkedAsUnread && conv.UnreadCount == 0 {
|
||||||
|
@ -422,7 +423,7 @@ func (portal *Portal) backfill(source *User, messages []*waProto.WebMessageInfo)
|
||||||
newMinTs = lastMessage.Timestamp
|
newMinTs = lastMessage.Timestamp
|
||||||
}
|
}
|
||||||
|
|
||||||
portal.log.Infofln("Processing history sync with %d messages", len(messages))
|
portal.log.Debugfln("Processing backfill with %d messages", len(messages))
|
||||||
// The messages are ordered newest to oldest, so iterate them in reverse order.
|
// The messages are ordered newest to oldest, so iterate them in reverse order.
|
||||||
for i := len(messages) - 1; i >= 0; i-- {
|
for i := len(messages) - 1; i >= 0; i-- {
|
||||||
webMsg := messages[i]
|
webMsg := messages[i]
|
||||||
|
|
Loading…
Reference in a new issue