config: add settings for prioritized backfill

bridge.history_sync.max_initial_conversations:

  This setting determines the maximum number of initial conversations
  that should be backfilled. The data for all the other conversations
  will be stored in the database for backfill at a later time.

bridge.history_sync.immediate:

  These settings are for the initial backfill that should be performed
  to populate each of the initial chats with a few messages so that
  users can continue their conversations without loosing context.

bridge.history_sync.deferred:

  These settings are for backfilling the rest of the chat history that
  was not covered by the immediate backfills. These should generally be
  done at a slower pace to avoid overloading the homeserver.
This commit is contained in:
Sumner Evans 2022-03-24 14:18:32 -06:00
parent 1ecdb71ac3
commit 536d340f20
No known key found for this signature in database
GPG key ID: 8904527AB50022FD
3 changed files with 69 additions and 11 deletions

View file

@ -28,6 +28,12 @@ import (
"maunium.net/go/mautrix/id"
)
type DeferredConfig struct {
StartDaysAgo int `yaml:"start_days_ago"`
MaxBatchEvents int `yaml:"max_batch_events"`
BatchDelay int `yaml:"batch_delay"`
}
type BridgeConfig struct {
UsernameTemplate string `yaml:"username_template"`
DisplaynameTemplate string `yaml:"displayname_template"`
@ -41,10 +47,17 @@ type BridgeConfig struct {
HistorySync struct {
CreatePortals bool `yaml:"create_portals"`
MaxAge int64 `yaml:"max_age"`
Backfill bool `yaml:"backfill"`
DoublePuppetBackfill bool `yaml:"double_puppet_backfill"`
RequestFullSync bool `yaml:"request_full_sync"`
MaxInitialConversations int `yaml:"max_initial_conversations"`
Immediate struct {
WorkerCount int `yaml:"worker_count"`
MaxEvents int `yaml:"max_events"`
} `yaml:"immediate"`
Deferred []DeferredConfig `yaml:"deferred"`
} `yaml:"history_sync"`
UserAvatarSync bool `yaml:"user_avatar_sync"`
BridgeMatrixLeave bool `yaml:"bridge_matrix_leave"`

View file

@ -78,10 +78,13 @@ func (helper *UpgradeHelper) doUpgrade() {
helper.Copy(Bool, "bridge", "call_start_notices")
helper.Copy(Bool, "bridge", "identity_change_notices")
helper.Copy(Bool, "bridge", "history_sync", "create_portals")
helper.Copy(Int, "bridge", "history_sync", "max_age")
helper.Copy(Bool, "bridge", "history_sync", "backfill")
helper.Copy(Bool, "bridge", "history_sync", "double_puppet_backfill")
helper.Copy(Bool, "bridge", "history_sync", "request_full_sync")
helper.Copy(Int, "bridge", "history_sync", "max_initial_conversations")
helper.Copy(Int, "bridge", "history_sync", "immediate", "worker_count")
helper.Copy(Int, "bridge", "history_sync", "immediate", "max_events")
helper.Copy(List, "bridge", "history_sync", "deferred")
helper.Copy(Bool, "bridge", "user_avatar_sync")
helper.Copy(Bool, "bridge", "bridge_matrix_leave")
helper.Copy(Bool, "bridge", "sync_with_custom_puppets")

View file

@ -115,14 +115,10 @@ bridge:
# Should another user's cryptographic identity changing send a message to Matrix?
identity_change_notices: false
portal_message_buffer: 128
# Settings for handling history sync payloads. These settings only apply right after login,
# because the phone only sends the history sync data once, and there's no way to re-request it
# (other than logging out and back in again).
# Settings for handling history sync payloads.
history_sync:
# Should the bridge create portals for chats in the history sync payload?
create_portals: true
# Maximum age of chats in seconds to create portals for. Set to 0 to create portals for all chats in sync payload.
max_age: 604800
# Enable backfilling history sync payloads from WhatsApp using batch sending?
# This requires a server with MSC2716 support, which is currently an experimental feature in synapse.
# It can be enabled by setting experimental_features -> msc2716_enabled to true in homeserver.yaml.
@ -137,6 +133,52 @@ bridge:
# Should the bridge request a full sync from the phone when logging in?
# This bumps the size of history syncs from 3 months to 1 year.
request_full_sync: false
# The maximum number of initial conversations that should be synced.
# Other conversations will be backfilled on demand when the start PM
# provisioning endpoint is used or when a message comes in from that
# chat.
max_initial_conversations: 10
# Settings for immediate backfills. These backfills should generally be
# small and their main purpose is to populate each of the initial chats
# (as configured by max_initial_conversations) with a few messages so
# that you can continue conversations without loosing context.
immediate:
# The number of concurrent backfill workers to create for immediate
# backfills. Note that using more than one worker could cause the
# room list to jump around since there are no guarantees about the
# order in which the backfills will complete.
worker_count: 1
# The maximum number of events to backfill initially.
max_events: 10
# Settings for deferred backfills. The purpose of these backfills are
# to fill in the rest of the chat history that was not covered by the
# immediate backfills. These backfills generally should happen at a
# slower pace so as not to overload the homeserver.
# Each deferred backfill config should define a "stage" of backfill
# (i.e. the last week of messages). The fields are as follows:
# - start_days_ago: the number of days ago to start backfilling from.
# To indicate the start of time, use -1. For example, for a week ago,
# use 7.
# - max_batch_events: the number of events to send per batch.
# - batch_delay: the number of seconds to wait before backfilling each
# batch.
deferred:
# Last Week
- start_days_ago: 7
max_batch_events: 20
batch_delay: 5
# Last Month
- start_days_ago: 30
max_batch_events: 50
batch_delay: 10
# Last 3 months
- start_days_ago: 90
max_batch_events: 100
batch_delay: 10
# The start of time
- start_days_ago: -1
max_batch_events: 500
batch_delay: 10
# Should puppet avatars be fetched from the server even if an avatar is already set?
user_avatar_sync: true
# Should Matrix users leaving groups be bridged to WhatsApp?