From c48ab3734ee57fcf0841d814680e50e49948cafe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 Apr 2022 15:48:16 +0100 Subject: [PATCH 1/6] Fix sending opentracing contexts to remote servers (#12555) --- changelog.d/12555.bugfix | 1 + synapse/storage/databases/main/devices.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12555.bugfix diff --git a/changelog.d/12555.bugfix b/changelog.d/12555.bugfix new file mode 100644 index 000000000..3c721da1e --- /dev/null +++ b/changelog.d/12555.bugfix @@ -0,0 +1 @@ +Fix sending opentracing contexts to whitelisted remote servers with device lists updates. Broken in v1.58.0rc1. diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 318e4df37..59d223a90 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -1776,7 +1776,17 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): def get_uncoverted_outbound_room_pokes_txn(txn): txn.execute(sql, (limit,)) - return txn.fetchall() + + return [ + ( + user_id, + device_id, + room_id, + stream_id, + db_to_json(opentracing_context), + ) + for user_id, device_id, room_id, stream_id, opentracing_context in txn + ] return await self.db_pool.runInteraction( "get_uncoverted_outbound_room_pokes", get_uncoverted_outbound_room_pokes_txn From 6d89f1239ce5ebe83b085a2b8939b2b6ce461265 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 15:53:06 +0100 Subject: [PATCH 2/6] Comment out dodgy log-kv (#12554) --- changelog.d/12554.bugfix | 1 + synapse/handlers/device.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 changelog.d/12554.bugfix diff --git a/changelog.d/12554.bugfix b/changelog.d/12554.bugfix new file mode 100644 index 000000000..48c1146db --- /dev/null +++ b/changelog.d/12554.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse 1.58.0rc1 where the main process could consume excessive amounts of CPU and memory handling sentry logging failures. diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 3c0fc756d..319836da2 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -683,9 +683,12 @@ class DeviceHandler(DeviceWorkerHandler): self.federation_sender.send_device_messages( host, immediate=False ) - log_kv( - {"message": "sent device update to host", "host": host} - ) + # TODO: when called, this isn't in a logging context. + # This leads to log spam, sentry event spam, and massive + # memory usage. See #12552. + # log_kv( + # {"message": "sent device update to host", "host": host} + # ) if current_stream_id != stream_id: # Clear the set of hosts we've already sent to as we're From f59e3f4c900b3961e2b96592a67cd2f0821b00db Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 Apr 2022 17:07:21 +0100 Subject: [PATCH 3/6] Mark remote device list updates as already handled (#12557) --- changelog.d/12557.misc | 1 + synapse/handlers/device.py | 5 +++-- synapse/storage/databases/main/devices.py | 3 ++- tests/storage/test_devices.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 changelog.d/12557.misc diff --git a/changelog.d/12557.misc b/changelog.d/12557.misc new file mode 100644 index 000000000..e4eb895ef --- /dev/null +++ b/changelog.d/12557.misc @@ -0,0 +1 @@ +Reduce unnecessary work when handling remote device list updates. diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 319836da2..a91b1ee4d 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -505,8 +505,9 @@ class DeviceHandler(DeviceWorkerHandler): "device_list_key", position, users={user_id}, rooms=room_ids ) - # We may need to do some processing asynchronously. - self._handle_new_device_update_async() + # We may need to do some processing asynchronously for local user IDs. + if self.hs.is_mine_id(user_id): + self._handle_new_device_update_async() async def notify_user_signature_update( self, from_user_id: str, user_ids: List[str] diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 59d223a90..483dd8040 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -1748,7 +1748,8 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): device_id, room_id, stream_id, - False, + # We only need to calculate outbound pokes for local users + not self.hs.is_mine_id(user_id), encoded_context, ) for room_id in room_ids diff --git a/tests/storage/test_devices.py b/tests/storage/test_devices.py index ccc389386..bbf079b25 100644 --- a/tests/storage/test_devices.py +++ b/tests/storage/test_devices.py @@ -29,7 +29,7 @@ class DeviceStoreTestCase(HomeserverTestCase): for device_id in device_ids: stream_id = self.get_success( self.store.add_device_change_to_streams( - "user_id", [device_id], ["!some:room"] + user_id, [device_id], ["!some:room"] ) ) From 6b64ee9ec7c09a322c38a2e537ad95a32f5ca5e2 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 17:16:43 +0100 Subject: [PATCH 4/6] 1.58.0rc2 --- CHANGES.md | 18 ++++++++++++++++++ changelog.d/12554.bugfix | 1 - changelog.d/12555.bugfix | 1 - changelog.d/12557.misc | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 6 files changed, 25 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/12554.bugfix delete mode 100644 changelog.d/12555.bugfix delete mode 100644 changelog.d/12557.misc diff --git a/CHANGES.md b/CHANGES.md index d8ecb11e4..b65cf00f4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,21 @@ +Synapse 1.58.0rc2 (2022-04-26) +============================== + +This release candidate fixes bugs related to Synapse 1.58.0rc1's logic for handling device list updates. + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.58.0rc1 where the main process could consume excessive amounts of CPU and memory handling sentry logging failures. ([\#12554](https://github.com/matrix-org/synapse/issues/12554)) +- Fix a bug introduced in Synapse 1.58.0rc1 where opentracing contexts were not correctly sent to whitelisted remote servers with device lists updates. ([\#12555](https://github.com/matrix-org/synapse/issues/12555)) + + +Internal Changes +---------------- + +- Reduce unnecessary work when handling remote device list updates. ([\#12557](https://github.com/matrix-org/synapse/issues/12557)) + + Synapse 1.58.0rc1 (2022-04-26) ============================== diff --git a/changelog.d/12554.bugfix b/changelog.d/12554.bugfix deleted file mode 100644 index 48c1146db..000000000 --- a/changelog.d/12554.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.58.0rc1 where the main process could consume excessive amounts of CPU and memory handling sentry logging failures. diff --git a/changelog.d/12555.bugfix b/changelog.d/12555.bugfix deleted file mode 100644 index 3c721da1e..000000000 --- a/changelog.d/12555.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix sending opentracing contexts to whitelisted remote servers with device lists updates. Broken in v1.58.0rc1. diff --git a/changelog.d/12557.misc b/changelog.d/12557.misc deleted file mode 100644 index e4eb895ef..000000000 --- a/changelog.d/12557.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce unnecessary work when handling remote device list updates. diff --git a/debian/changelog b/debian/changelog index 20e756f6d..5f1bf872b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.58.0~rc2) stable; urgency=medium + + * New Synapse release 1.58.0rc2. + + -- Synapse Packaging team Tue, 26 Apr 2022 17:14:56 +0100 + matrix-synapse-py3 (1.58.0~rc1) stable; urgency=medium * Use poetry to manage the bundled virtualenv included with this package. diff --git a/pyproject.toml b/pyproject.toml index cc4908fa2..bdded7843 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.58.0rc1" +version = "1.58.0rc2" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From 56c9c6c4654f4c356fd266396e97f426c2afd8c6 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 17:17:56 +0100 Subject: [PATCH 5/6] Credit Tulir's contribution in 1.58.0rc1 to Beeper, too --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index b65cf00f4..b9b10bb0d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -27,7 +27,7 @@ Features - Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly. ([\#11398](https://github.com/matrix-org/synapse/issues/11398)) - Docker images and Debian packages from matrix.org now contain a locked set of Python dependencies, greatly improving build reproducibility. ([Board](https://github.com/orgs/matrix-org/projects/54), [\#11537](https://github.com/matrix-org/synapse/issues/11537)) - Enable processing of device list updates asynchronously. ([\#12365](https://github.com/matrix-org/synapse/issues/12365), [\#12465](https://github.com/matrix-org/synapse/issues/12465)) -- Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir. ([\#12427](https://github.com/matrix-org/synapse/issues/12427)) +- Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir @ Beeper. ([\#12427](https://github.com/matrix-org/synapse/issues/12427)) - Build Debian packages for Ubuntu 22.04 "Jammy Jellyfish". ([\#12543](https://github.com/matrix-org/synapse/issues/12543)) From 9cfecd2dc01e2e5c6ccb7b87f7beae83a98650f3 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 17:22:12 +0100 Subject: [PATCH 6/6] Adjust changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index b9b10bb0d..1fbe0815d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,7 +6,7 @@ This release candidate fixes bugs related to Synapse 1.58.0rc1's logic for handl Bugfixes -------- -- Fix a bug introduced in Synapse 1.58.0rc1 where the main process could consume excessive amounts of CPU and memory handling sentry logging failures. ([\#12554](https://github.com/matrix-org/synapse/issues/12554)) +- Fix a bug introduced in Synapse 1.58.0rc1 where the main process could consume excessive amounts of CPU and memory while handling sentry logging failures. ([\#12554](https://github.com/matrix-org/synapse/issues/12554)) - Fix a bug introduced in Synapse 1.58.0rc1 where opentracing contexts were not correctly sent to whitelisted remote servers with device lists updates. ([\#12555](https://github.com/matrix-org/synapse/issues/12555))