MirrorHub/synapse
0
0
Fork 1
mirror of https://mau.dev/maunium/synapse.git synced 2024-06-17 01:59:26 +02:00
Code Issues Projects Releases Wiki Activity

Log the details of background update failures (#16212)

Browse source
This commit is contained in:
David Robertson 2023-09-01 12:41:56 +01:00 committed by GitHub
parent ed5e8a77ca
commit 6525fd65ee
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 26 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
changelog.d/16212.misc Normal file
View file

@ -0,0 +1 @@
Log the details of background update failures.

4
synapse/storage/background_updates.py
View file

@ -405,14 +405,14 @@ class BackgroundUpdater:
try:
result = await self.do_next_background_update(sleep)
back_to_back_failures = 0
except Exception:
except Exception as e:
logger.exception("Error doing update: %s", e)
back_to_back_failures += 1
if back_to_back_failures >= 5:
self._aborted = True
raise RuntimeError(
"5 back-to-back background update failures; aborting."
)
logger.exception("Error doing update")
else:
if result:
logger.info(

24
tests/storage/test_background_update.py
View file

@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from unittest.mock import AsyncMock, Mock
import yaml
@ -330,6 +330,28 @@ class BackgroundUpdateTestCase(unittest.HomeserverTestCase):
self.update_handler.side_effect = update_short
self.get_success(self.updates.do_next_background_update(False))
def test_failed_update_logs_exception_details(self) -> None:
needle = "RUH ROH RAGGY"
def failing_update(progress: JsonDict, count: int) -> int:
raise Exception(needle)
self.update_handler.side_effect = failing_update
self.update_handler.reset_mock()
self.get_success(
self.store.db_pool.simple_insert(
"background_updates",
values={"update_name": "test_update", "progress_json": "{}"},
)
)
with self.assertLogs(level=logging.ERROR) as logs:
# Expect a back-to-back RuntimeError to be raised
self.get_failure(self.updates.run_background_updates(False), RuntimeError)
self.assertTrue(any(needle in log for log in logs.output), logs.output)
class BackgroundUpdateControllerTestCase(unittest.HomeserverTestCase):
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: