tileradotorg/synapse
1
0
Fork 0
forked from MirrorHub/synapse
Code Pull requests Activity

Lower minumum batch size to 1 for background updates (#11422)

Browse source 
Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
This commit is contained in:
Brendan Abolivier 2021-11-24 20:21:44 +01:00 committed by GitHub
parent f25c75d376
commit 7f9841bdec
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 23 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
changelog.d/11422.bugfix Normal file
View file

@ -0,0 +1 @@
Improve performance of various background database schema updates.

2
synapse/storage/background_updates.py
View file

@ -82,7 +82,7 @@ class BackgroundUpdater:
process and autotuning the batch size.
"""
MINIMUM_BACKGROUND_BATCH_SIZE = 100
MINIMUM_BACKGROUND_BATCH_SIZE = 1
DEFAULT_BACKGROUND_BATCH_SIZE = 100
BACKGROUND_UPDATE_INTERVAL_MS = 1000
BACKGROUND_UPDATE_DURATION_MS = 100

25
tests/rest/admin/test_background_updates.py
View file

@ -20,6 +20,7 @@ import synapse.rest.admin
from synapse.api.errors import Codes
from synapse.rest.client import login
from synapse.server import HomeServer
from synapse.storage.background_updates import BackgroundUpdater
from tests import unittest
@ -150,9 +151,11 @@ class BackgroundUpdatesTestCase(unittest.HomeserverTestCase):
"current_updates": {
"master": {
"name": "test_update",
"average_items_per_ms": 0.1,
"average_items_per_ms": 0.001,
"total_duration_ms": 1000.0,
"total_item_count": 100,
"total_item_count": (
BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE
),
}
},
"enabled": True,
@ -203,9 +206,11 @@ class BackgroundUpdatesTestCase(unittest.HomeserverTestCase):
"current_updates": {
"master": {
"name": "test_update",
"average_items_per_ms": 0.1,
"average_items_per_ms": 0.001,
"total_duration_ms": 1000.0,
"total_item_count": 100,
"total_item_count": (
BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE
),
}
},
"enabled": False,
@ -230,9 +235,11 @@ class BackgroundUpdatesTestCase(unittest.HomeserverTestCase):
"current_updates": {
"master": {
"name": "test_update",
"average_items_per_ms": 0.1,
"average_items_per_ms": 0.001,
"total_duration_ms": 1000.0,
"total_item_count": 100,
"total_item_count": (
BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE
),
}
},
"enabled": False,
@ -267,9 +274,11 @@ class BackgroundUpdatesTestCase(unittest.HomeserverTestCase):
"current_updates": {
"master": {
"name": "test_update",
"average_items_per_ms": 0.1,
"average_items_per_ms": 0.001,
"total_duration_ms": 2000.0,
"total_item_count": 200,
"total_item_count": (
2 * BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE
),
}
},
"enabled": True,

8
tests/storage/test_background_update.py
View file

@ -19,11 +19,11 @@ class BackgroundUpdateTestCase(unittest.HomeserverTestCase):
)
def test_do_background_update(self):
# the time we claim each update takes
duration_ms = 42
# the time we claim it takes to update one item when running the update
duration_ms = 4200
# the target runtime for each bg update
target_background_update_duration_ms = 50000
target_background_update_duration_ms = 5000000
store = self.hs.get_datastore()
self.get_success(
@ -57,7 +57,7 @@ class BackgroundUpdateTestCase(unittest.HomeserverTestCase):
# on the first call, we should get run with the default background update size
self.update_handler.assert_called_once_with(
{"my_key": 1}, self.updates.DEFAULT_BACKGROUND_BATCH_SIZE
{"my_key": 1}, self.updates.MINIMUM_BACKGROUND_BATCH_SIZE
)
# second step: complete the update