MirrorHub/mautrix-whatsapp
0
0
Fork 1
mirror of https://github.com/tulir/mautrix-whatsapp synced 2024-12-14 09:23:51 +01:00
Code Issues Projects Releases Wiki Activity

Fix backfill queue query. Fixes #517

Browse source
This commit is contained in:
Tulir Asokan 2022-06-17 10:33:46 +03:00
parent 7d5aeeaf88
commit 8012368de5
2 changed files with 16 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
database/backfill.go
View file

@ -90,7 +90,7 @@ const (
AND ( AND (
dispatch_time IS NULL dispatch_time IS NULL
OR ( OR (
dispatch_time < current_timestamp - interval '15 minutes' dispatch_time < $2
AND completed_at IS NULL AND completed_at IS NULL
) )
) )
@ -112,13 +112,13 @@ func (bq *BackfillQuery) GetNext(userID id.UserID, backfillTypes []BackfillType)
bq.backfillQueryLock.Lock() bq.backfillQueryLock.Lock()
defer bq.backfillQueryLock.Unlock() defer bq.backfillQueryLock.Unlock()
types := []string{} var types []string
for _, backfillType := range backfillTypes { for _, backfillType := range backfillTypes {
types = append(types, strconv.Itoa(int(backfillType))) types = append(types, strconv.Itoa(int(backfillType)))
} }
rows, err := bq.db.Query(fmt.Sprintf(getNextBackfillQuery, strings.Join(types, ",")), userID) rows, err := bq.db.Query(fmt.Sprintf(getNextBackfillQuery, strings.Join(types, ",")), userID, time.Now().Add(-15*time.Minute))
if err != nil || rows == nil { if err != nil || rows == nil {
bq.log.Error(err) bq.log.Errorfln("Failed to query next backfill queue job: %v", err)
return return
} }
defer rows.Close() defer rows.Close()
@ -142,6 +142,9 @@ func (bq *BackfillQuery) HasUnstartedOrInFlightOfType(userID id.UserID, backfill
} }
rows, err := bq.db.Query(fmt.Sprintf(getUnstartedOrInFlightQuery, strings.Join(types, ",")), userID) rows, err := bq.db.Query(fmt.Sprintf(getUnstartedOrInFlightQuery, strings.Join(types, ",")), userID)
if err != nil || rows == nil { if err != nil || rows == nil {
if err != nil && !errors.Is(err, sql.ErrNoRows) {
bq.log.Warnfln("Failed to query backfill queue jobs: %v", err)
}
// No rows means that there are no unstarted or in flight backfill // No rows means that there are no unstarted or in flight backfill
// requests. // requests.
return false return false
@ -198,13 +201,16 @@ func (b *Backfill) String() string {
} }
func (b *Backfill) Scan(row dbutil.Scannable) *Backfill { func (b *Backfill) Scan(row dbutil.Scannable) *Backfill {
err := row.Scan(&b.QueueID, &b.UserID, &b.BackfillType, &b.Priority, &b.Portal.JID, &b.Portal.Receiver, &b.TimeStart, &b.MaxBatchEvents, &b.MaxTotalEvents, &b.BatchDelay) var maxTotalEvents, batchDelay sql.NullInt32
err := row.Scan(&b.QueueID, &b.UserID, &b.BackfillType, &b.Priority, &b.Portal.JID, &b.Portal.Receiver, &b.TimeStart, &b.MaxBatchEvents, &maxTotalEvents, &batchDelay)
if err != nil { if err != nil {
if !errors.Is(err, sql.ErrNoRows) { if !errors.Is(err, sql.ErrNoRows) {
b.log.Errorln("Database scan failed:", err) b.log.Errorln("Database scan failed:", err)
} }
return nil return nil
} }
b.MaxTotalEvents = int(maxTotalEvents.Int32)
b.BatchDelay = int(batchDelay.Int32)
return b return b
} }
@ -234,7 +240,7 @@ func (b *Backfill) MarkDispatched() {
defer b.db.Backfill.backfillQueryLock.Unlock() defer b.db.Backfill.backfillQueryLock.Unlock()
if b.QueueID == 0 { if b.QueueID == 0 {
b.log.Errorf("Cannot mark backfill as dispatched without queue_id. Maybe it wasn't actually inserted in the database?") b.log.Errorfln("Cannot mark backfill as dispatched without queue_id. Maybe it wasn't actually inserted in the database?")
return return
} }
_, err := b.db.Exec("UPDATE backfill_queue SET dispatch_time=$1 WHERE queue_id=$2", time.Now(), b.QueueID) _, err := b.db.Exec("UPDATE backfill_queue SET dispatch_time=$1 WHERE queue_id=$2", time.Now(), b.QueueID)
@ -248,7 +254,7 @@ func (b *Backfill) MarkDone() {
defer b.db.Backfill.backfillQueryLock.Unlock() defer b.db.Backfill.backfillQueryLock.Unlock()
if b.QueueID == 0 { if b.QueueID == 0 {
b.log.Errorf("Cannot mark backfill done without queue_id. Maybe it wasn't actually inserted in the database?") b.log.Errorfln("Cannot mark backfill done without queue_id. Maybe it wasn't actually inserted in the database?")
return return
} }
_, err := b.db.Exec("UPDATE backfill_queue SET completed_at=$1 WHERE queue_id=$2", time.Now(), b.QueueID) _, err := b.db.Exec("UPDATE backfill_queue SET completed_at=$1 WHERE queue_id=$2", time.Now(), b.QueueID)

3
database/historysync.go
View file

@ -275,6 +275,9 @@ func (hsq *HistorySyncQuery) GetMessagesBetween(userID id.UserID, conversationID
rows, err := hsq.db.Query(fmt.Sprintf(getMessagesBetween, whereClauses, limitClause), args...) rows, err := hsq.db.Query(fmt.Sprintf(getMessagesBetween, whereClauses, limitClause), args...)
defer rows.Close() defer rows.Close()
if err != nil || rows == nil { if err != nil || rows == nil {
if err != nil && !errors.Is(err, sql.ErrNoRows) {
hsq.log.Warnfln("Failed to query messages between range: %v", err)
}
return nil return nil
} }