From 84cf3e47a0318aba51d9f830d5e724182c5d93c4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 28 May 2021 16:28:01 +0100 Subject: [PATCH 1/9] Allow response of `/send_join` to be larger. (#10093) Fixes #10087. --- changelog.d/10093.bugfix | 1 + synapse/federation/transport/client.py | 7 +++++++ synapse/http/matrixfederationclient.py | 14 +++++++++++++- 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 changelog.d/10093.bugfix diff --git a/changelog.d/10093.bugfix b/changelog.d/10093.bugfix new file mode 100644 index 000000000..e50de4b2e --- /dev/null +++ b/changelog.d/10093.bugfix @@ -0,0 +1 @@ +Fix HTTP response size limit to allow joining very large rooms over federation. diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index e93ab83f7..5b4f5d17f 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -35,6 +35,11 @@ from synapse.types import JsonDict logger = logging.getLogger(__name__) +# Send join responses can be huge, so we set a separate limit here. The response +# is parsed in a streaming manner, which helps alleviate the issue of memory +# usage a bit. +MAX_RESPONSE_SIZE_SEND_JOIN = 500 * 1024 * 1024 + class TransportLayerClient: """Sends federation HTTP requests to other servers""" @@ -261,6 +266,7 @@ class TransportLayerClient: path=path, data=content, parser=SendJoinParser(room_version, v1_api=True), + max_response_size=MAX_RESPONSE_SIZE_SEND_JOIN, ) return response @@ -276,6 +282,7 @@ class TransportLayerClient: path=path, data=content, parser=SendJoinParser(room_version, v1_api=False), + max_response_size=MAX_RESPONSE_SIZE_SEND_JOIN, ) return response diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index f5503b394..1998990a1 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -205,6 +205,7 @@ async def _handle_response( response: IResponse, start_ms: int, parser: ByteParser[T], + max_response_size: Optional[int] = None, ) -> T: """ Reads the body of a response with a timeout and sends it to a parser @@ -216,15 +217,20 @@ async def _handle_response( response: response to the request start_ms: Timestamp when request was made parser: The parser for the response + max_response_size: The maximum size to read from the response, if None + uses the default. Returns: The parsed response """ + if max_response_size is None: + max_response_size = MAX_RESPONSE_SIZE + try: check_content_type_is(response.headers, parser.CONTENT_TYPE) - d = read_body_with_max_size(response, parser, MAX_RESPONSE_SIZE) + d = read_body_with_max_size(response, parser, max_response_size) d = timeout_deferred(d, timeout=timeout_sec, reactor=reactor) length = await make_deferred_yieldable(d) @@ -735,6 +741,7 @@ class MatrixFederationHttpClient: backoff_on_404: bool = False, try_trailing_slash_on_400: bool = False, parser: Literal[None] = None, + max_response_size: Optional[int] = None, ) -> Union[JsonDict, list]: ... @@ -752,6 +759,7 @@ class MatrixFederationHttpClient: backoff_on_404: bool = False, try_trailing_slash_on_400: bool = False, parser: Optional[ByteParser[T]] = None, + max_response_size: Optional[int] = None, ) -> T: ... @@ -768,6 +776,7 @@ class MatrixFederationHttpClient: backoff_on_404: bool = False, try_trailing_slash_on_400: bool = False, parser: Optional[ByteParser] = None, + max_response_size: Optional[int] = None, ): """Sends the specified json data using PUT @@ -803,6 +812,8 @@ class MatrixFederationHttpClient: enabled. parser: The parser to use to decode the response. Defaults to parsing as JSON. + max_response_size: The maximum size to read from the response, if None + uses the default. Returns: Succeeds when we get a 2xx HTTP response. The @@ -853,6 +864,7 @@ class MatrixFederationHttpClient: response, start_ms, parser=parser, + max_response_size=max_response_size, ) return body From 1641c5c707fe9cac5f68589863082409c8979da6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 28 May 2021 15:57:53 +0100 Subject: [PATCH 2/9] Log method and path when dropping request due to size limit (#10091) --- changelog.d/10091.misc | 1 + synapse/http/site.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changelog.d/10091.misc diff --git a/changelog.d/10091.misc b/changelog.d/10091.misc new file mode 100644 index 000000000..dbe310fd1 --- /dev/null +++ b/changelog.d/10091.misc @@ -0,0 +1 @@ +Log method and path when dropping request due to size limit. diff --git a/synapse/http/site.py b/synapse/http/site.py index 671fd3fbc..40754b7be 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -105,8 +105,10 @@ class SynapseRequest(Request): assert self.content, "handleContentChunk() called before gotLength()" if self.content.tell() + len(data) > self._max_request_body_size: logger.warning( - "Aborting connection from %s because the request exceeds maximum size", + "Aborting connection from %s because the request exceeds maximum size: %s %s", self.client, + self.get_method(), + self.get_redacted_uri(), ) self.transport.abortConnection() return From 9408b86f5c3616e8cfaa2c183e787780a3a64f95 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Thu, 27 May 2021 18:10:58 +0200 Subject: [PATCH 3/9] Limit the number of events sent over replication when persisting events. (#10082) --- changelog.d/10082.bugfix | 1 + synapse/handlers/federation.py | 17 ++++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 changelog.d/10082.bugfix diff --git a/changelog.d/10082.bugfix b/changelog.d/10082.bugfix new file mode 100644 index 000000000..b4f8bcc4f --- /dev/null +++ b/changelog.d/10082.bugfix @@ -0,0 +1 @@ +Fixed a bug causing replication requests to fail when receiving a lot of events via federation. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 678f6b770..bf1131525 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -91,6 +91,7 @@ from synapse.types import ( get_domain_from_id, ) from synapse.util.async_helpers import Linearizer, concurrently_execute +from synapse.util.iterutils import batch_iter from synapse.util.retryutils import NotRetryingDestination from synapse.util.stringutils import shortstr from synapse.visibility import filter_events_for_server @@ -3053,13 +3054,15 @@ class FederationHandler(BaseHandler): """ instance = self.config.worker.events_shard_config.get_instance(room_id) if instance != self._instance_name: - result = await self._send_events( - instance_name=instance, - store=self.store, - room_id=room_id, - event_and_contexts=event_and_contexts, - backfilled=backfilled, - ) + # Limit the number of events sent over federation. + for batch in batch_iter(event_and_contexts, 1000): + result = await self._send_events( + instance_name=instance, + store=self.store, + room_id=room_id, + event_and_contexts=batch, + backfilled=backfilled, + ) return result["max_stream_id"] else: assert self.storage.persistence From 258a9a9e8bea851493fc2275ac6b81639c997afb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 28 May 2021 17:06:05 +0100 Subject: [PATCH 4/9] 1.35.0rc3 --- CHANGES.md | 16 ++++++++++++++++ changelog.d/10082.bugfix | 1 - changelog.d/10091.misc | 1 - changelog.d/10093.bugfix | 1 - synapse/__init__.py | 2 +- 5 files changed, 17 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/10082.bugfix delete mode 100644 changelog.d/10091.misc delete mode 100644 changelog.d/10093.bugfix diff --git a/CHANGES.md b/CHANGES.md index 1fac16580..8bd05c318 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,19 @@ +Synapse 1.35.0rc3 (2021-05-28) +============================== + +Bugfixes +-------- + +- Fixed a bug causing replication requests to fail when receiving a lot of events via federation. ([\#10082](https://github.com/matrix-org/synapse/issues/10082)) +- Fix HTTP response size limit to allow joining very large rooms over federation. ([\#10093](https://github.com/matrix-org/synapse/issues/10093)) + + +Internal Changes +---------------- + +- Log method and path when dropping request due to size limit. ([\#10091](https://github.com/matrix-org/synapse/issues/10091)) + + Synapse 1.35.0rc2 (2021-05-27) ============================== diff --git a/changelog.d/10082.bugfix b/changelog.d/10082.bugfix deleted file mode 100644 index b4f8bcc4f..000000000 --- a/changelog.d/10082.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed a bug causing replication requests to fail when receiving a lot of events via federation. diff --git a/changelog.d/10091.misc b/changelog.d/10091.misc deleted file mode 100644 index dbe310fd1..000000000 --- a/changelog.d/10091.misc +++ /dev/null @@ -1 +0,0 @@ -Log method and path when dropping request due to size limit. diff --git a/changelog.d/10093.bugfix b/changelog.d/10093.bugfix deleted file mode 100644 index e50de4b2e..000000000 --- a/changelog.d/10093.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix HTTP response size limit to allow joining very large rooms over federation. diff --git a/synapse/__init__.py b/synapse/__init__.py index 6faf31dbb..4591246bd 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ try: except ImportError: pass -__version__ = "1.35.0rc2" +__version__ = "1.35.0rc3" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From 4f41b711d8b37da3403ce67c88d62133f732a459 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 28 May 2021 17:13:57 +0100 Subject: [PATCH 5/9] CHANGELOG --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8bd05c318..7e6f478d4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,8 +4,8 @@ Synapse 1.35.0rc3 (2021-05-28) Bugfixes -------- -- Fixed a bug causing replication requests to fail when receiving a lot of events via federation. ([\#10082](https://github.com/matrix-org/synapse/issues/10082)) -- Fix HTTP response size limit to allow joining very large rooms over federation. ([\#10093](https://github.com/matrix-org/synapse/issues/10093)) +- Fixed a bug causing replication requests to fail when receiving a lot of events via federation. Introduced in v1.33.0. ([\#10082](https://github.com/matrix-org/synapse/issues/10082)) +- Fix HTTP response size limit to allow joining very large rooms over federation. Introduced in v1.33.0. ([\#10093](https://github.com/matrix-org/synapse/issues/10093)) Internal Changes From 408ecf8ece397bcf08564031379b461d5c9b0de5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 1 Jun 2021 13:19:50 +0100 Subject: [PATCH 6/9] Announce deprecation of experimental `msc2858_enabled` option. (#10101) c.f. https://github.com/matrix-org/synapse/pull/9617 and https://github.com/matrix-org/matrix-doc/blob/master/proposals/2858-Multiple-SSO-Identity-Providers.md Fixes #9627. --- changelog.d/10101.removal | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/10101.removal diff --git a/changelog.d/10101.removal b/changelog.d/10101.removal new file mode 100644 index 000000000..f2020e9dd --- /dev/null +++ b/changelog.d/10101.removal @@ -0,0 +1 @@ +The core Synapse development team plan to drop support for the [unstable API of MSC2858](https://github.com/matrix-org/matrix-doc/blob/master/proposals/2858-Multiple-SSO-Identity-Providers.md#unstable-prefix), including the undocumented `experimental.msc2858_enabled` config option, in August 2021. Client authors should ensure that their clients are updated to use the stable API (which has been supported since Synapse 1.30) well before that time, to give their users time to upgrade. From a8372ad591e07fa76e194a22732a5301d9e55b6f Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 1 Jun 2021 13:23:55 +0100 Subject: [PATCH 7/9] 1.35.0 --- CHANGES.md | 9 +++++++++ changelog.d/10101.removal | 1 - debian/changelog | 6 ++++++ synapse/__init__.py | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/10101.removal diff --git a/CHANGES.md b/CHANGES.md index 7e6f478d4..09f0be8e1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +Synapse 1.35.0 (2021-06-01) +=========================== + +Deprecations and Removals +------------------------- + +- The core Synapse development team plan to drop support for the [unstable API of MSC2858](https://github.com/matrix-org/matrix-doc/blob/master/proposals/2858-Multiple-SSO-Identity-Providers.md#unstable-prefix), including the undocumented `experimental.msc2858_enabled` config option, in August 2021. Client authors should ensure that their clients are updated to use the stable API (which has been supported since Synapse 1.30) well before that time, to give their users time to upgrade. ([\#10101](https://github.com/matrix-org/synapse/issues/10101)) + + Synapse 1.35.0rc3 (2021-05-28) ============================== diff --git a/changelog.d/10101.removal b/changelog.d/10101.removal deleted file mode 100644 index f2020e9dd..000000000 --- a/changelog.d/10101.removal +++ /dev/null @@ -1 +0,0 @@ -The core Synapse development team plan to drop support for the [unstable API of MSC2858](https://github.com/matrix-org/matrix-doc/blob/master/proposals/2858-Multiple-SSO-Identity-Providers.md#unstable-prefix), including the undocumented `experimental.msc2858_enabled` config option, in August 2021. Client authors should ensure that their clients are updated to use the stable API (which has been supported since Synapse 1.30) well before that time, to give their users time to upgrade. diff --git a/debian/changelog b/debian/changelog index bf99ae772..d5efb8ccb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.35.0) stable; urgency=medium + + * New synapse release 1.35.0. + + -- Synapse Packaging team Tue, 01 Jun 2021 13:23:35 +0100 + matrix-synapse-py3 (1.34.0) stable; urgency=medium * New synapse release 1.34.0. diff --git a/synapse/__init__.py b/synapse/__init__.py index 4591246bd..d9843a170 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ try: except ImportError: pass -__version__ = "1.35.0rc3" +__version__ = "1.35.0" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From 08e54345b1332889cd9e88a778bc13caca7b556f Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 1 Jun 2021 13:25:18 +0100 Subject: [PATCH 8/9] Indicate that there were no functional changes since v1.35.0rc3 --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 09f0be8e1..c969fe1eb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,8 @@ Synapse 1.35.0 (2021-06-01) =========================== +No changes since v1.35.0rc3. + Deprecations and Removals ------------------------- From 3fdaf4df55f52ccf283cf6b0ca73a3f98cd5e8f0 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 1 Jun 2021 13:40:46 +0100 Subject: [PATCH 9/9] Merge v1.35.0rc3 into v1.35.0 due to incorrect tagging --- CHANGES.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c969fe1eb..f03a53aff 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,17 +1,13 @@ Synapse 1.35.0 (2021-06-01) =========================== -No changes since v1.35.0rc3. +Note that [the tag](https://github.com/matrix-org/synapse/releases/tag/v1.35.0rc3) and [docker images](https://hub.docker.com/layers/matrixdotorg/synapse/v1.35.0rc3/images/sha256-34ccc87bd99a17e2cbc0902e678b5937d16bdc1991ead097eee6096481ecf2c4?context=explore) for `v1.35.0rc3` were incorrectly built. If you are experiencing issues with either, it is recommended to upgrade to the equivalent tag or docker image for the `v1.35.0` release. Deprecations and Removals ------------------------- - The core Synapse development team plan to drop support for the [unstable API of MSC2858](https://github.com/matrix-org/matrix-doc/blob/master/proposals/2858-Multiple-SSO-Identity-Providers.md#unstable-prefix), including the undocumented `experimental.msc2858_enabled` config option, in August 2021. Client authors should ensure that their clients are updated to use the stable API (which has been supported since Synapse 1.30) well before that time, to give their users time to upgrade. ([\#10101](https://github.com/matrix-org/synapse/issues/10101)) - -Synapse 1.35.0rc3 (2021-05-28) -============================== - Bugfixes --------