tileradotorg/mautrix-whatsapp
1
0
Fork 0
forked from MirrorHub/mautrix-whatsapp
Code Pull requests Activity

backfill queue: don't delete, just mark as complete

Browse source
This commit is contained in:
Sumner Evans 2022-04-05 13:12:57 -06:00
parent 173edfaaf1
commit 005fbb09f8
No known key found for this signature in database
GPG key ID: 8904527AB50022FD
3 changed files with 11 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
backfillqueue.go
View file

@ -41,7 +41,7 @@ func (bq *BackfillQueue) immediateBackfillLoop(user *User) {
for {
if backfill := bq.BackfillQuery.GetNext(user.MXID, database.BackfillImmediate); backfill != nil {
bq.ImmediateBackfillRequests <- backfill
backfill.Delete()
backfill.MarkDone()
} else {
select {
case <-bq.ReCheckQueue:
@ -61,7 +61,7 @@ func (bq *BackfillQueue) deferredBackfillLoop(user *User) {
if backfill := bq.BackfillQuery.GetNext(user.MXID, database.BackfillDeferred); backfill != nil {
bq.DeferredBackfillRequests <- backfill
backfill.Delete()
backfill.MarkDone()
} else {
time.Sleep(10 * time.Second)
}

14
database/backfillqueue.go
View file

@ -67,6 +67,7 @@ const (
FROM backfill_queue
WHERE user_mxid=$1
AND type=$2
AND completed_at IS NULL
ORDER BY priority, queue_id
LIMIT 1
`
@ -106,6 +107,7 @@ type Backfill struct {
MaxBatchEvents int
MaxTotalEvents int
BatchDelay int
CompletedAt *time.Time
}
func (b *Backfill) Scan(row Scannable) *Backfill {
@ -122,10 +124,10 @@ func (b *Backfill) Scan(row Scannable) *Backfill {
func (b *Backfill) Insert() {
rows, err := b.db.Query(`
INSERT INTO backfill_queue
(user_mxid, type, priority, portal_jid, portal_receiver, time_start, time_end, max_batch_events, max_total_events, batch_delay)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
(user_mxid, type, priority, portal_jid, portal_receiver, time_start, time_end, max_batch_events, max_total_events, batch_delay, completed_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
RETURNING queue_id
`, b.UserID, b.BackfillType, b.Priority, b.Portal.JID, b.Portal.Receiver, b.TimeStart, b.TimeEnd, b.MaxBatchEvents, b.MaxTotalEvents, b.BatchDelay)
`, b.UserID, b.BackfillType, b.Priority, b.Portal.JID, b.Portal.Receiver, b.TimeStart, b.TimeEnd, b.MaxBatchEvents, b.MaxTotalEvents, b.BatchDelay, b.CompletedAt)
defer rows.Close()
if err != nil || !rows.Next() {
b.log.Warnfln("Failed to insert %v/%s with priority %d: %v", b.BackfillType, b.Portal.JID, b.Priority, err)
@ -137,13 +139,13 @@ func (b *Backfill) Insert() {
}
}
func (b *Backfill) Delete() {
func (b *Backfill) MarkDone() {
if b.QueueID == 0 {
b.log.Errorf("Cannot delete backfill without queue_id. Maybe it wasn't actually inserted in the database?")
return
}
_, err := b.db.Exec("DELETE FROM backfill_queue WHERE queue_id=$1", b.QueueID)
_, err := b.db.Exec("UPDATE backfill_queue SET completed_at=$1 WHERE queue_id=$2", time.Now(), b.QueueID)
if err != nil {
b.log.Warnfln("Failed to delete %s/%s: %v", b.BackfillType, b.Priority, err)
b.log.Warnfln("Failed to mark %s/%s as complete: %v", b.BackfillType, b.Priority, err)
}
}

1
database/upgrades/2022-03-15-prioritized-backfill.go
View file

@ -17,6 +17,7 @@ func init() {
max_batch_events INTEGER NOT NULL,
max_total_events INTEGER,
batch_delay INTEGER,
completed_at TIMESTAMP,
FOREIGN KEY (user_mxid) REFERENCES "user"(mxid) ON DELETE CASCADE ON UPDATE CASCADE,
FOREIGN KEY (portal_jid, portal_receiver) REFERENCES portal(jid, receiver) ON DELETE CASCADE