From c2a173a0f2241d8e66f9725bd2ab0f7eb61cd67e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 15 Aug 2023 14:31:09 +0300 Subject: [PATCH 01/14] Add meow dockerfile N.B. requires requirements.txt to be generated in repo root beforehand --- .dockerignore | 1 + .gitlab-ci.yml | 19 ++++++++++++++++ Dockerfile | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 .gitlab-ci.yml create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore index 0b51345cb..c7d713713 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,6 +8,7 @@ !README.rst !pyproject.toml !poetry.lock +!requirements.txt !Cargo.lock !Cargo.toml !build_rust.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..8e7d10e12 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,19 @@ +image: docker:stable + +stages: +- build + +build amd64: + stage: build + tags: + - amd64 + only: + - master + before_script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + script: + - synversion=$(cat pyproject.toml | grep '^version =' | sed -E 's/^version = "(.+)"$/\1/') + - docker build --tag $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$synversion . + - docker push $CI_REGISTRY_IMAGE:latest + - docker push $CI_REGISTRY_IMAGE:$synversion + - docker rmi $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$synversion diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..fad6c2ebf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,61 @@ +ARG PYTHON_VERSION=3.11 + +FROM docker.io/python:${PYTHON_VERSION}-slim as builder + +RUN apt-get update && apt-get install -y \ + build-essential \ + libffi-dev \ + libjpeg-dev \ + libpq-dev \ + libssl-dev \ + libwebp-dev \ + libxml++2.6-dev \ + libxslt1-dev \ + zlib1g-dev \ + openssl \ + git \ + curl \ + && rm -rf /var/lib/apt/lists/* + +ENV RUSTUP_HOME=/rust +ENV CARGO_HOME=/cargo +ENV PATH=/cargo/bin:/rust/bin:$PATH +RUN mkdir /rust /cargo + +RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable + +COPY synapse /synapse/synapse/ +COPY rust /synapse/rust/ +COPY README.rst pyproject.toml requirements.txt build_rust.py /synapse/ + +RUN pip install --prefix="/install" --no-warn-script-location --ignore-installed \ + --no-deps -r /synapse/requirements.txt \ + && pip install --prefix="/install" --no-warn-script-location \ + --no-deps \ + 'git+https://github.com/maunium/synapse-simple-antispam#egg=synapse-simple-antispam' \ + 'git+https://github.com/devture/matrix-synapse-shared-secret-auth@2.0.3#egg=shared_secret_authenticator' \ + && pip install --prefix="/install" --no-warn-script-location \ + --no-deps /synapse + +FROM docker.io/python:${PYTHON_VERSION}-slim + +RUN apt-get update && apt-get install -y \ + curl \ + libjpeg62-turbo \ + libpq5 \ + libwebp7 \ + xmlsec1 \ + libjemalloc2 \ + openssl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /install /usr/local + +VOLUME ["/data"] +ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libjemalloc.so.2" + +ENTRYPOINT ["python3", "-m", "synapse.app.homeserver"] +CMD ["--keys-directory", "/data", "-c", "/data/homeserver.yaml"] + +HEALTHCHECK --start-period=5s --interval=1m --timeout=5s \ + CMD curl -fSs http://localhost:8008/health || exit 1 From e21fcdca4f8ea0d021828641466c286a0256c908 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 14:28:52 +0200 Subject: [PATCH 02/14] Add meow readme and config extension --- README.md | 63 ++++++++++++++++++++++++++++++++++++ synapse/config/_base.pyi | 2 ++ synapse/config/homeserver.py | 2 ++ synapse/config/meow.py | 33 +++++++++++++++++++ 4 files changed, 100 insertions(+) create mode 100644 README.md create mode 100644 synapse/config/meow.py diff --git a/README.md b/README.md new file mode 100644 index 000000000..9d4829c93 --- /dev/null +++ b/README.md @@ -0,0 +1,63 @@ +# Maunium Synapse +This is a fork of [Synapse] to remove dumb limits and fix bugs that the +upstream devs don't want to fix. + +The only official distribution is the docker image in the [GitLab container +registry], but you can also install from source ([upstream instructions]). + +The master branch and `:latest` docker tag are upgraded to each upstream +release candidate very soon after release (usually within 10 minutes†). There +are also docker tags for each release, e.g. `:1.75.0`. If you don't want RCs, +use the specific release tags. + +†If there are merge conflicts, the update may be delayed for up to a few days +after the full release. + +[Synapse]: https://github.com/matrix-org/synapse +[GitLab container registry]: https://mau.dev/maunium/synapse/container_registry +[upstream instructions]: https://github.com/matrix-org/synapse/blob/develop/INSTALL.md#installing-from-source + +## List of changes +* Default power level for room creator is 9001 instead of 100. +* Room creator can specify a custom room ID with the `room_id` param in the + request body. If the room ID is already in use, it will return `M_CONFLICT`. +* ~~URL previewer user agent includes `Bot` so Twitter previews work properly.~~ + Upstreamed after over 2 years 🎉 +* ~~Local event creation concurrency is disabled to avoid unnecessary state + resolution.~~ Upstreamed after over 3 years 🎉 +* Register admin API can register invalid user IDs. +* Docker image with jemalloc enabled by default. +* Config option to allow specific users to send events without unnecessary + validation. +* Config option to allow specific users to receive events that are usually + filtered away (e.g. `org.matrix.dummy_event` and `m.room.aliases`). +* Config option to allow specific users to use timestamp massaging without + being appservice users. +* Removed bad pusher URL validation. +* webp images are thumbnailed to webp instead of jpeg to avoid losing + transparency. +* Media repo `Cache-Control` header says `immutable` and 1 year for all media + that exists, as media IDs in Matrix are immutable. +* Allowed sending custom data with read receipts. + +You can view the full list of changes on the [meow-patchset] branch. +Additionally, historical patch sets are saved as `meow-patchset-vX` [tags]. + +[meow-patchset]: https://mau.dev/maunium/synapse/-/compare/patchset-base...meow-patchset +[tags]: https://mau.dev/maunium/synapse/-/tags?search=meow-patchset&sort=updated_desc + +## Configuration reference +```yaml +meow: + # List of users who aren't subject to unnecessary validation in the C-S API. + validation_override: + - "@you:example.com" + # List of users who will get org.matrix.dummy_event and m.room.aliases events down /sync + filter_override: + - "@you:example.com" + # Whether or not the admin API should be able to register invalid user IDs. + admin_api_register_invalid: true + # List of users who can use timestamp massaging without being appservices + timestamp_override: + - "@you:example.com" +``` diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi index d9cb0da38..c3120e665 100644 --- a/synapse/config/_base.pyi +++ b/synapse/config/_base.pyi @@ -36,6 +36,7 @@ from synapse.config import ( # noqa: F401 jwt, key, logger, + meow, metrics, modules, oembed, @@ -92,6 +93,7 @@ class RootConfig: voip: voip.VoipConfig registration: registration.RegistrationConfig account_validity: account_validity.AccountValidityConfig + meow: meow.MeowConfig metrics: metrics.MetricsConfig api: api.ApiConfig appservice: appservice.AppServiceConfig diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index e36c0bd6a..822e25f68 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -19,6 +19,7 @@ # # from ._base import RootConfig +from .meow import MeowConfig from .account_validity import AccountValidityConfig from .api import ApiConfig from .appservice import AppServiceConfig @@ -65,6 +66,7 @@ from .workers import WorkerConfig class HomeServerConfig(RootConfig): config_classes = [ + MeowConfig, ModulesConfig, ServerConfig, RetentionConfig, diff --git a/synapse/config/meow.py b/synapse/config/meow.py new file mode 100644 index 000000000..a7492bf66 --- /dev/null +++ b/synapse/config/meow.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Maunium +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config + + +class MeowConfig(Config): + """Meow Configuration + Configuration for disabling dumb limits in Synapse + """ + + section = "meow" + + def read_config(self, config, **kwargs): + meow_config = config.get("meow", {}) + self.validation_override = set(meow_config.get("validation_override", [])) + self.filter_override = set(meow_config.get("filter_override", [])) + self.timestamp_override = set(meow_config.get("timestamp_override", [])) + self.admin_api_register_invalid = meow_config.get( + "admin_api_register_invalid", True + ) From d3229d1b5e9d938fb9db96feae71753ee9636a33 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 14:30:50 +0200 Subject: [PATCH 03/14] Fix default power level for room creator --- synapse/handlers/room.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 386375d64..b38b64f47 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -1190,7 +1190,7 @@ class RoomCreationHandler: # Please update the docs for `default_power_level_content_override` when # updating the `events` dict below power_level_content: JsonDict = { - "users": {creator_id: 100}, + "users": {creator_id: 9001}, "users_default": 0, "events": { EventTypes.Name: 50, From 75620e118e502c42669d7fdc3062ead27ee39072 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 14:35:30 +0200 Subject: [PATCH 04/14] Allow specifying room ID when creating room --- synapse/handlers/room.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index b38b64f47..c4f4bd6e5 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -894,11 +894,23 @@ class RoomCreationHandler: self._validate_room_config(config, visibility) - room_id = await self._generate_and_create_room_id( - creator_id=user_id, - is_public=is_public, - room_version=room_version, - ) + if "room_id" in config: + room_id = config["room_id"] + try: + await self.store.store_room( + room_id=room_id, + room_creator_user_id=user_id, + is_public=is_public, + room_version=room_version, + ) + except StoreError: + raise SynapseError(409, "Room ID already in use", errcode="M_CONFLICT") + else: + room_id = await self._generate_and_create_room_id( + creator_id=user_id, + is_public=is_public, + room_version=room_version, + ) # Check whether this visibility value is blocked by a third party module allowed_by_third_party_rules = await ( From 4db99ed0197873e293723a062a35f604e9f44cee Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 14:37:27 +0200 Subject: [PATCH 05/14] Allow registering invalid user IDs with admin API --- synapse/handlers/register.py | 50 +++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index c200e2956..779b4625a 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -148,22 +148,25 @@ class RegistrationHandler: localpart: str, guest_access_token: Optional[str] = None, assigned_user_id: Optional[str] = None, + allow_invalid: bool = False, inhibit_user_in_use_error: bool = False, ) -> None: - if types.contains_invalid_mxid_characters(localpart): - raise SynapseError( - 400, - "User ID can only contain characters a-z, 0-9, or '=_-./+'", - Codes.INVALID_USERNAME, - ) + # meow: allow admins to register invalid user ids + if not allow_invalid: + if types.contains_invalid_mxid_characters(localpart): + raise SynapseError( + 400, + "User ID can only contain characters a-z, 0-9, or '=_-./+'", + Codes.INVALID_USERNAME, + ) - if not localpart: - raise SynapseError(400, "User ID cannot be empty", Codes.INVALID_USERNAME) + if not localpart: + raise SynapseError(400, "User ID cannot be empty", Codes.INVALID_USERNAME) - if localpart[0] == "_": - raise SynapseError( - 400, "User ID may not begin with _", Codes.INVALID_USERNAME - ) + if localpart[0] == "_": + raise SynapseError( + 400, "User ID may not begin with _", Codes.INVALID_USERNAME + ) user = UserID(localpart, self.hs.hostname) user_id = user.to_string() @@ -177,14 +180,16 @@ class RegistrationHandler: "A different user ID has already been registered for this session", ) - self.check_user_id_not_appservice_exclusive(user_id) + # meow: allow admins to register reserved user ids and long user ids + if not allow_invalid: + self.check_user_id_not_appservice_exclusive(user_id) - if len(user_id) > MAX_USERID_LENGTH: - raise SynapseError( - 400, - "User ID may not be longer than %s characters" % (MAX_USERID_LENGTH,), - Codes.INVALID_USERNAME, - ) + if len(user_id) > MAX_USERID_LENGTH: + raise SynapseError( + 400, + "User ID may not be longer than %s characters" % (MAX_USERID_LENGTH,), + Codes.INVALID_USERNAME, + ) users = await self.store.get_users_by_id_case_insensitive(user_id) if users: @@ -290,7 +295,12 @@ class RegistrationHandler: await self.auth_blocking.check_auth_blocking(threepid=threepid) if localpart is not None: - await self.check_username(localpart, guest_access_token=guest_access_token) + allow_invalid = by_admin and self.hs.config.meow.admin_api_register_invalid + await self.check_username( + localpart, + guest_access_token=guest_access_token, + allow_invalid=allow_invalid, + ) was_guest = guest_access_token is not None From d50743e1d88ae7b0a7b647c6b13e1e875dec6158 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 14:39:07 +0200 Subject: [PATCH 06/14] Thumbnail webp images as webp to avoid losing transparency --- synapse/config/repository.py | 10 ++++++---- synapse/media/thumbnailer.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 97ce6de52..b532a2ea3 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -54,10 +54,8 @@ THUMBNAIL_SIZE_YAML = """\ THUMBNAIL_SUPPORTED_MEDIA_FORMAT_MAP = { "image/jpeg": "jpeg", "image/jpg": "jpeg", - "image/webp": "jpeg", - # Thumbnails can only be jpeg or png. We choose png thumbnails for gif - # because it can have transparency. - "image/gif": "png", + "image/webp": "webp", + "image/gif": "webp", "image/png": "png", } @@ -109,6 +107,10 @@ def parse_thumbnail_requirements( requirement.append( ThumbnailRequirement(width, height, method, "image/png") ) + elif thumbnail_format == "webp": + requirement.append( + ThumbnailRequirement(width, height, method, "image/webp") + ) else: raise Exception( "Unknown thumbnail mapping from %s to %s. This is a Synapse problem, please report!" diff --git a/synapse/media/thumbnailer.py b/synapse/media/thumbnailer.py index ee1118a53..a83b209bd 100644 --- a/synapse/media/thumbnailer.py +++ b/synapse/media/thumbnailer.py @@ -65,7 +65,7 @@ class ThumbnailError(Exception): class Thumbnailer: - FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG"} + FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG", "image/webp": "WEBP"} @staticmethod def set_limits(max_image_pixels: int) -> None: From abf856ae6f08c0ddb6242e5c434f3dc8b33ab229 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 14:39:50 +0200 Subject: [PATCH 07/14] Set immutable cache-control header for media downloads --- synapse/media/_base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/media/_base.py b/synapse/media/_base.py index 7877df62f..4f423c0ae 100644 --- a/synapse/media/_base.py +++ b/synapse/media/_base.py @@ -229,7 +229,9 @@ def add_file_headers( # recommend caching as it's sensitive or private - or at least # select private. don't bother setting Expires as all our # clients are smart enough to be happy with Cache-Control - request.setHeader(b"Cache-Control", b"public,max-age=86400,s-maxage=86400") + request.setHeader( + b"Cache-Control", b"public,immutable,max-age=86400,s-maxage=86400" + ) if file_size is not None: request.setHeader(b"Content-Length", b"%d" % (file_size,)) From 5994cbd4a025656777b07be5d3594cb9d4281d18 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 14:51:56 +0200 Subject: [PATCH 08/14] Allow bypassing unnecessary validation in C-S API --- synapse/events/validator.py | 16 ++++++++++++++-- synapse/handlers/directory.py | 13 +++++++++---- synapse/handlers/federation.py | 4 ++-- synapse/handlers/message.py | 9 ++++++--- synapse/handlers/room_member.py | 20 -------------------- 5 files changed, 31 insertions(+), 31 deletions(-) diff --git a/synapse/events/validator.py b/synapse/events/validator.py index 8aa8d7e01..6ba69042b 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -58,7 +58,7 @@ class EventValidator: event: The event to validate. config: The homeserver's configuration. """ - self.validate_builder(event) + self.validate_builder(event, config) if event.format_version == EventFormatVersions.ROOM_V1_V2: EventID.from_string(event.event_id) @@ -89,6 +89,12 @@ class EventValidator: # Note that only the client controlled portion of the event is # checked, since we trust the portions of the event we created. validate_canonicaljson(event.content) + if not 0 < event.origin_server_ts < 2**53: + raise SynapseError(400, "Event timestamp is out of range") + + # meow: allow specific users to send potentially dangerous events. + if event.sender in config.meow.validation_override: + return if event.type == EventTypes.Aliases: if "aliases" in event.content: @@ -187,7 +193,9 @@ class EventValidator: errcode=Codes.BAD_JSON, ) - def validate_builder(self, event: Union[EventBase, EventBuilder]) -> None: + def validate_builder( + self, event: Union[EventBase, EventBuilder], config: HomeServerConfig + ) -> None: """Validates that the builder/event has roughly the right format. Only checks values that we expect a proto event to have, rather than all the fields an event would have @@ -205,6 +213,10 @@ class EventValidator: RoomID.from_string(event.room_id) UserID.from_string(event.sender) + # meow: allow specific users to send so-called invalid events + if event.sender in config.meow.validation_override: + return + if event.type == EventTypes.Message: strings = ["body", "msgtype"] diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 62ce16794..a2874b070 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -80,9 +80,11 @@ class DirectoryHandler: ) -> None: # general association creation for both human users and app services - for wchar in string.whitespace: - if wchar in room_alias.localpart: - raise SynapseError(400, "Invalid characters in room alias") + # meow: allow specific users to include anything in room aliases + if creator not in self.config.meow.validation_override: + for wchar in string.whitespace: + if wchar in room_alias.localpart: + raise SynapseError(400, "Invalid characters in room alias") if ":" in room_alias.localpart: raise SynapseError(400, "Invalid character in room alias localpart: ':'.") @@ -127,7 +129,10 @@ class DirectoryHandler: user_id = requester.user.to_string() room_alias_str = room_alias.to_string() - if len(room_alias_str) > MAX_ALIAS_LENGTH: + if ( + user_id not in self.hs.config.meow.validation_override + and len(room_alias_str) > MAX_ALIAS_LENGTH + ): raise SynapseError( 400, "Can't create aliases longer than %s characters" % MAX_ALIAS_LENGTH, diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 2b7aad5b5..520e30fd1 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1452,7 +1452,7 @@ class FederationHandler: room_version_obj, event_dict ) - EventValidator().validate_builder(builder) + EventValidator().validate_builder(builder, self.hs.config) # Try several times, it could fail with PartialStateConflictError # in send_membership_event, cf comment in except block. @@ -1617,7 +1617,7 @@ class FederationHandler: builder = self.event_builder_factory.for_room_version( room_version_obj, event_dict ) - EventValidator().validate_builder(builder) + EventValidator().validate_builder(builder, self.hs.config) ( event, diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 204965afe..b9e5b1580 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -672,7 +672,7 @@ class EventCreationHandler: room_version_obj, event_dict ) - self.validator.validate_builder(builder) + self.validator.validate_builder(builder, self.config) is_exempt = await self._is_exempt_from_privacy_policy(builder, requester) if require_consent and not is_exempt: @@ -1330,6 +1330,8 @@ class EventCreationHandler: Raises: SynapseError if the event is invalid. """ + if event.sender in self.config.meow.validation_override: + return relation = relation_from_event(event) if not relation: @@ -1749,7 +1751,8 @@ class EventCreationHandler: await self._maybe_kick_guest_users(event, context) - if event.type == EventTypes.CanonicalAlias: + validation_override = event.sender in self.config.meow.validation_override + if event.type == EventTypes.CanonicalAlias and not validation_override: # Validate a newly added alias or newly added alt_aliases. original_alias = None @@ -2104,7 +2107,7 @@ class EventCreationHandler: builder = self.event_builder_factory.for_room_version( original_event.room_version, third_party_result ) - self.validator.validate_builder(builder) + self.validator.validate_builder(builder, self.config) except SynapseError as e: raise Exception( "Third party rules module created an invalid event: " + e.msg, diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 75c60e3c3..54d4902be 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -797,26 +797,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): content.pop("displayname", None) content.pop("avatar_url", None) - if len(content.get("displayname") or "") > MAX_DISPLAYNAME_LEN: - raise SynapseError( - 400, - f"Displayname is too long (max {MAX_DISPLAYNAME_LEN})", - errcode=Codes.BAD_JSON, - ) - - if len(content.get("avatar_url") or "") > MAX_AVATAR_URL_LEN: - raise SynapseError( - 400, - f"Avatar URL is too long (max {MAX_AVATAR_URL_LEN})", - errcode=Codes.BAD_JSON, - ) - - if "avatar_url" in content and content.get("avatar_url") is not None: - if not await self.profile_handler.check_avatar_size_and_mime_type( - content["avatar_url"], - ): - raise SynapseError(403, "This avatar is not allowed", Codes.FORBIDDEN) - # The event content should *not* include the authorising user as # it won't be properly signed. Strip it out since it might come # back from a client updating a display name / avatar. From 9bff00c396a1b363308df9c26aba9670b0fdfa0b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 14:54:24 +0200 Subject: [PATCH 09/14] Allow unhiding events that the C-S API filters away by default --- synapse/handlers/sync.py | 1 - synapse/storage/controllers/__init__.py | 1 + synapse/visibility.py | 12 +++++++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index f4ea90fbd..941932dba 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1310,7 +1310,6 @@ class SyncHandler: for e in await sync_config.filter_collection.filter_room_state( list(state.values()) ) - if e.type != EventTypes.Aliases # until MSC2261 or alternative solution } async def _compute_state_delta_for_full_sync( diff --git a/synapse/storage/controllers/__init__.py b/synapse/storage/controllers/__init__.py index 5ba848efb..17449b7fc 100644 --- a/synapse/storage/controllers/__init__.py +++ b/synapse/storage/controllers/__init__.py @@ -45,6 +45,7 @@ class StorageControllers: # rewrite all the existing code to split it into high vs low level # interfaces. self.main = stores.main + self.hs = hs self.purge_events = PurgeEventsStorageController(hs, stores) self.state = StateStorageController(hs, stores) diff --git a/synapse/visibility.py b/synapse/visibility.py index 3a2782bad..f5d7b1b0d 100644 --- a/synapse/visibility.py +++ b/synapse/visibility.py @@ -139,6 +139,10 @@ async def filter_events_for_client( room_id ] = await storage.main.get_retention_policy_for_room(room_id) + # meow: let admins see secret events like org.matrix.dummy_event, m.room.aliases + # and events expired by the retention policy. + filter_override = user_id in storage.hs.config.meow.filter_override + def allowed(event: EventBase) -> Optional[EventBase]: state_after_event = event_id_to_state.get(event.event_id) filtered = _check_client_allowed_to_see_event( @@ -152,6 +156,7 @@ async def filter_events_for_client( state=state_after_event, is_peeking=is_peeking, sender_erased=erased_senders.get(event.sender, False), + filter_override=filter_override, ) if filtered is None: return None @@ -328,6 +333,7 @@ def _check_client_allowed_to_see_event( retention_policy: RetentionPolicy, state: Optional[StateMap[EventBase]], sender_erased: bool, + filter_override: bool, ) -> Optional[EventBase]: """Check with the given user is allowed to see the given event @@ -344,6 +350,7 @@ def _check_client_allowed_to_see_event( retention_policy: The retention policy of the room state: The state at the event, unless its an outlier sender_erased: Whether the event sender has been marked as "erased" + filter_override: meow Returns: None if the user cannot see this event at all @@ -357,7 +364,7 @@ def _check_client_allowed_to_see_event( # because, if this is not the case, we're probably only checking if the users can # see events in the room at that point in the DAG, and that shouldn't be decided # on those checks. - if filter_send_to_client: + if filter_send_to_client and not filter_override: if ( _check_filter_send_to_client(event, clock, retention_policy, sender_ignored) == _CheckFilter.DENIED @@ -367,6 +374,9 @@ def _check_client_allowed_to_see_event( event.event_id, ) return None + # meow: even with filter_override, we want to filter ignored users + elif filter_send_to_client and not event.is_state() and sender_ignored: + return None if event.event_id in always_include_ids: return event From e48d921119d5a6332f473933900886e869f653da Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 15:02:13 +0200 Subject: [PATCH 10/14] Allow custom content in read receipts --- synapse/handlers/read_marker.py | 11 ++++++++--- synapse/handlers/receipts.py | 3 ++- synapse/rest/client/read_marker.py | 5 +++++ synapse/rest/client/receipts.py | 4 +++- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py index fb39c8e04..5f5240b96 100644 --- a/synapse/handlers/read_marker.py +++ b/synapse/handlers/read_marker.py @@ -20,11 +20,12 @@ # import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from synapse.api.constants import ReceiptTypes from synapse.api.errors import SynapseError from synapse.util.async_helpers import Linearizer +from synapse.types import JsonDict if TYPE_CHECKING: from synapse.server import HomeServer @@ -39,7 +40,11 @@ class ReadMarkerHandler: self.read_marker_linearizer = Linearizer(name="read_marker") async def received_client_read_marker( - self, room_id: str, user_id: str, event_id: str + self, + room_id: str, + user_id: str, + event_id: str, + extra_content: Optional[JsonDict] = None, ) -> None: """Updates the read marker for a given user in a given room if the event ID given is ahead in the stream relative to the current read marker. @@ -71,7 +76,7 @@ class ReadMarkerHandler: should_update = event_ordering > old_event_ordering if should_update: - content = {"event_id": event_id} + content = {"event_id": event_id, **(extra_content or {})} await self.account_data_handler.add_account_data_to_room( user_id, room_id, ReceiptTypes.FULLY_READ, content ) diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index c776654d1..78d2ef7d1 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -181,6 +181,7 @@ class ReceiptsHandler: user_id: UserID, event_id: str, thread_id: Optional[str], + extra_content: Optional[JsonDict] = None, ) -> None: """Called when a client tells us a local user has read up to the given event_id in the room. @@ -197,7 +198,7 @@ class ReceiptsHandler: user_id=user_id.to_string(), event_ids=[event_id], thread_id=thread_id, - data={"ts": int(self.clock.time_msec())}, + data={"ts": int(self.clock.time_msec()), **(extra_content or {})}, ) is_new = await self._handle_new_receipts([receipt]) diff --git a/synapse/rest/client/read_marker.py b/synapse/rest/client/read_marker.py index d3d3c7c41..dce29e493 100644 --- a/synapse/rest/client/read_marker.py +++ b/synapse/rest/client/read_marker.py @@ -80,12 +80,16 @@ class ReadMarkerRestServlet(RestServlet): # TODO Add validation to reject non-string event IDs. if not event_id: continue + extra_content = body.get( + receipt_type.replace("m.", "com.beeper.") + ".extra", None + ) if receipt_type == ReceiptTypes.FULLY_READ: await self.read_marker_handler.received_client_read_marker( room_id, user_id=requester.user.to_string(), event_id=event_id, + extra_content=extra_content, ) else: await self.receipts_handler.received_client_receipt( @@ -95,6 +99,7 @@ class ReadMarkerRestServlet(RestServlet): event_id=event_id, # Setting the thread ID is not possible with the /read_markers endpoint. thread_id=None, + extra_content=extra_content, ) return 200, {} diff --git a/synapse/rest/client/receipts.py b/synapse/rest/client/receipts.py index 89203dc45..d83baaa1c 100644 --- a/synapse/rest/client/receipts.py +++ b/synapse/rest/client/receipts.py @@ -73,7 +73,7 @@ class ReceiptRestServlet(RestServlet): f"Receipt type must be {', '.join(self._known_receipt_types)}", ) - body = parse_json_object_from_request(request) + body = parse_json_object_from_request(request, allow_empty_body=False) # Pull the thread ID, if one exists. thread_id = None @@ -110,6 +110,7 @@ class ReceiptRestServlet(RestServlet): room_id, user_id=requester.user.to_string(), event_id=event_id, + extra_content=body, ) else: await self.receipts_handler.received_client_receipt( @@ -118,6 +119,7 @@ class ReceiptRestServlet(RestServlet): user_id=requester.user, event_id=event_id, thread_id=thread_id, + extra_content=body, ) return 200, {} From 0583a5967b1b4748d73db97943a1e00d1b368904 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 15:02:25 +0200 Subject: [PATCH 11/14] Allow specific users to use timestamp massaging without being appservices --- synapse/rest/client/room.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 8883cd6bc..bffd36832 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -362,6 +362,7 @@ class RoomSendEventRestServlet(TransactionRestServlet): self.delayed_events_handler = hs.get_delayed_events_handler() self.auth = hs.get_auth() self._max_event_delay_ms = hs.config.server.max_event_delay_ms + self.hs = hs def register(self, http_server: HttpServer) -> None: # /rooms/$roomid/send/$event_type[/$txn_id] @@ -379,7 +380,10 @@ class RoomSendEventRestServlet(TransactionRestServlet): content = parse_json_object_from_request(request) origin_server_ts = None - if requester.app_service: + if ( + requester.app_service + or requester.user.to_string() in self.hs.config.meow.timestamp_override + ): origin_server_ts = parse_integer(request, "ts") delay = _parse_request_delay(request, self._max_event_delay_ms) From ca4e3165b3beff3b1a61614ef9dbb9d605ca56ec Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 15:02:44 +0200 Subject: [PATCH 12/14] Remove unnecessary pusher URL validation --- synapse/push/httppusher.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index dd9b64d6e..b40112541 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -140,13 +140,6 @@ class HttpPusher(Pusher): url = self.data["url"] if not isinstance(url, str): raise PusherConfigException("'url' must be a string") - url_parts = urllib.parse.urlparse(url) - # Note that the specification also says the scheme must be HTTPS, but - # it isn't up to the homeserver to verify that. - if url_parts.path != "/_matrix/push/v1/notify": - raise PusherConfigException( - "'url' must have a path of '/_matrix/push/v1/notify'" - ) self.url = url self.http_client = hs.get_proxied_blocklisted_http_client() From 245d82f44e4b8e5daf6304294faaff46b59e4ed0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 10 Oct 2023 15:29:12 +0300 Subject: [PATCH 13/14] Allow pdf inline --- synapse/media/_base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/media/_base.py b/synapse/media/_base.py index 4f423c0ae..104f1edf4 100644 --- a/synapse/media/_base.py +++ b/synapse/media/_base.py @@ -86,6 +86,7 @@ INLINE_CONTENT_TYPES = [ "text/csv", "application/json", "application/ld+json", + "application/pdf", # We allow some media files deemed as safe, which comes from the matrix-react-sdk. # https://github.com/matrix-org/matrix-react-sdk/blob/a70fcfd0bcf7f8c85986da18001ea11597989a7c/src/utils/blobs.ts#L51 # SVGs are *intentionally* omitted. From 9572748b56b2264748525b55cf3f080408a3d697 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 12 Dec 2023 14:34:01 +0200 Subject: [PATCH 14/14] Don't apply alias rules to admins --- synapse/handlers/directory.py | 4 ++-- synapse/handlers/room.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index a2874b070..3fce6dcff 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -174,7 +174,7 @@ class DirectoryHandler: if not self.config.roomdirectory.is_alias_creation_allowed( user_id, room_id, room_alias_str - ): + ) and not is_admin: # Let's just return a generic message, as there may be all sorts of # reasons why we said no. TODO: Allow configurable error messages # per alias creation rule? @@ -510,7 +510,7 @@ class DirectoryHandler: if not self.config.roomdirectory.is_publishing_room_allowed( user_id, room_id, room_aliases - ): + ) and not await self.auth.is_server_admin(requester): # Let's just return a generic message, as there may be all sorts of # reasons why we said no. TODO: Allow configurable error messages # per alias creation rule? diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c4f4bd6e5..cfa7f6eb0 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -927,7 +927,7 @@ class RoomCreationHandler: room_aliases.append(room_alias.to_string()) if not self.config.roomdirectory.is_publishing_room_allowed( user_id, room_id, room_aliases - ): + ) and not is_requester_admin: # allow room creation to continue but do not publish room await self.store.set_room_is_public(room_id, False)