From e95889bab30159ecd1576f074aecac11c2c79220 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 15 Aug 2023 14:31:09 +0300 Subject: [PATCH 01/14] Add meow dockerfile N.B. requires requirements.txt to be generated in repo root beforehand --- .dockerignore | 1 + .gitlab-ci.yml | 19 ++++++++++++++++ Dockerfile | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 .gitlab-ci.yml create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore index 0b51345cb..c7d713713 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,6 +8,7 @@ !README.rst !pyproject.toml !poetry.lock +!requirements.txt !Cargo.lock !Cargo.toml !build_rust.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..8e7d10e12 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,19 @@ +image: docker:stable + +stages: +- build + +build amd64: + stage: build + tags: + - amd64 + only: + - master + before_script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + script: + - synversion=$(cat pyproject.toml | grep '^version =' | sed -E 's/^version = "(.+)"$/\1/') + - docker build --tag $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$synversion . + - docker push $CI_REGISTRY_IMAGE:latest + - docker push $CI_REGISTRY_IMAGE:$synversion + - docker rmi $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$synversion diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..fad6c2ebf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,61 @@ +ARG PYTHON_VERSION=3.11 + +FROM docker.io/python:${PYTHON_VERSION}-slim as builder + +RUN apt-get update && apt-get install -y \ + build-essential \ + libffi-dev \ + libjpeg-dev \ + libpq-dev \ + libssl-dev \ + libwebp-dev \ + libxml++2.6-dev \ + libxslt1-dev \ + zlib1g-dev \ + openssl \ + git \ + curl \ + && rm -rf /var/lib/apt/lists/* + +ENV RUSTUP_HOME=/rust +ENV CARGO_HOME=/cargo +ENV PATH=/cargo/bin:/rust/bin:$PATH +RUN mkdir /rust /cargo + +RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable + +COPY synapse /synapse/synapse/ +COPY rust /synapse/rust/ +COPY README.rst pyproject.toml requirements.txt build_rust.py /synapse/ + +RUN pip install --prefix="/install" --no-warn-script-location --ignore-installed \ + --no-deps -r /synapse/requirements.txt \ + && pip install --prefix="/install" --no-warn-script-location \ + --no-deps \ + 'git+https://github.com/maunium/synapse-simple-antispam#egg=synapse-simple-antispam' \ + 'git+https://github.com/devture/matrix-synapse-shared-secret-auth@2.0.3#egg=shared_secret_authenticator' \ + && pip install --prefix="/install" --no-warn-script-location \ + --no-deps /synapse + +FROM docker.io/python:${PYTHON_VERSION}-slim + +RUN apt-get update && apt-get install -y \ + curl \ + libjpeg62-turbo \ + libpq5 \ + libwebp7 \ + xmlsec1 \ + libjemalloc2 \ + openssl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /install /usr/local + +VOLUME ["/data"] +ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libjemalloc.so.2" + +ENTRYPOINT ["python3", "-m", "synapse.app.homeserver"] +CMD ["--keys-directory", "/data", "-c", "/data/homeserver.yaml"] + +HEALTHCHECK --start-period=5s --interval=1m --timeout=5s \ + CMD curl -fSs http://localhost:8008/health || exit 1 From ce380461240afb3a6c9f70d42bdf8553741b8243 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 14:28:52 +0200 Subject: [PATCH 02/14] Add meow readme and config extension --- README.md | 63 ++++++++++++++++++++++++++++++++++++ synapse/config/_base.pyi | 2 ++ synapse/config/homeserver.py | 2 ++ synapse/config/meow.py | 33 +++++++++++++++++++ 4 files changed, 100 insertions(+) create mode 100644 README.md create mode 100644 synapse/config/meow.py diff --git a/README.md b/README.md new file mode 100644 index 000000000..9d4829c93 --- /dev/null +++ b/README.md @@ -0,0 +1,63 @@ +# Maunium Synapse +This is a fork of [Synapse] to remove dumb limits and fix bugs that the +upstream devs don't want to fix. + +The only official distribution is the docker image in the [GitLab container +registry], but you can also install from source ([upstream instructions]). + +The master branch and `:latest` docker tag are upgraded to each upstream +release candidate very soon after release (usually within 10 minutes†). There +are also docker tags for each release, e.g. `:1.75.0`. If you don't want RCs, +use the specific release tags. + +†If there are merge conflicts, the update may be delayed for up to a few days +after the full release. + +[Synapse]: https://github.com/matrix-org/synapse +[GitLab container registry]: https://mau.dev/maunium/synapse/container_registry +[upstream instructions]: https://github.com/matrix-org/synapse/blob/develop/INSTALL.md#installing-from-source + +## List of changes +* Default power level for room creator is 9001 instead of 100. +* Room creator can specify a custom room ID with the `room_id` param in the + request body. If the room ID is already in use, it will return `M_CONFLICT`. +* ~~URL previewer user agent includes `Bot` so Twitter previews work properly.~~ + Upstreamed after over 2 years 🎉 +* ~~Local event creation concurrency is disabled to avoid unnecessary state + resolution.~~ Upstreamed after over 3 years 🎉 +* Register admin API can register invalid user IDs. +* Docker image with jemalloc enabled by default. +* Config option to allow specific users to send events without unnecessary + validation. +* Config option to allow specific users to receive events that are usually + filtered away (e.g. `org.matrix.dummy_event` and `m.room.aliases`). +* Config option to allow specific users to use timestamp massaging without + being appservice users. +* Removed bad pusher URL validation. +* webp images are thumbnailed to webp instead of jpeg to avoid losing + transparency. +* Media repo `Cache-Control` header says `immutable` and 1 year for all media + that exists, as media IDs in Matrix are immutable. +* Allowed sending custom data with read receipts. + +You can view the full list of changes on the [meow-patchset] branch. +Additionally, historical patch sets are saved as `meow-patchset-vX` [tags]. + +[meow-patchset]: https://mau.dev/maunium/synapse/-/compare/patchset-base...meow-patchset +[tags]: https://mau.dev/maunium/synapse/-/tags?search=meow-patchset&sort=updated_desc + +## Configuration reference +```yaml +meow: + # List of users who aren't subject to unnecessary validation in the C-S API. + validation_override: + - "@you:example.com" + # List of users who will get org.matrix.dummy_event and m.room.aliases events down /sync + filter_override: + - "@you:example.com" + # Whether or not the admin API should be able to register invalid user IDs. + admin_api_register_invalid: true + # List of users who can use timestamp massaging without being appservices + timestamp_override: + - "@you:example.com" +``` diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi index fc51aed23..e6d4428ee 100644 --- a/synapse/config/_base.pyi +++ b/synapse/config/_base.pyi @@ -35,6 +35,7 @@ from synapse.config import ( # noqa: F401 jwt, key, logger, + meow, metrics, modules, oembed, @@ -91,6 +92,7 @@ class RootConfig: voip: voip.VoipConfig registration: registration.RegistrationConfig account_validity: account_validity.AccountValidityConfig + meow: meow.MeowConfig metrics: metrics.MetricsConfig api: api.ApiConfig appservice: appservice.AppServiceConfig diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 72e93ed04..03f987ab6 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -19,6 +19,7 @@ # # from ._base import RootConfig +from .meow import MeowConfig from .account_validity import AccountValidityConfig from .api import ApiConfig from .appservice import AppServiceConfig @@ -64,6 +65,7 @@ from .workers import WorkerConfig class HomeServerConfig(RootConfig): config_classes = [ + MeowConfig, ModulesConfig, ServerConfig, RetentionConfig, diff --git a/synapse/config/meow.py b/synapse/config/meow.py new file mode 100644 index 000000000..a7492bf66 --- /dev/null +++ b/synapse/config/meow.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Maunium +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config + + +class MeowConfig(Config): + """Meow Configuration + Configuration for disabling dumb limits in Synapse + """ + + section = "meow" + + def read_config(self, config, **kwargs): + meow_config = config.get("meow", {}) + self.validation_override = set(meow_config.get("validation_override", [])) + self.filter_override = set(meow_config.get("filter_override", [])) + self.timestamp_override = set(meow_config.get("timestamp_override", [])) + self.admin_api_register_invalid = meow_config.get( + "admin_api_register_invalid", True + ) From 78584d476cdd790c1ab198c90d3c457d35fb74e8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 14:30:50 +0200 Subject: [PATCH 03/14] Fix default power level for room creator --- synapse/handlers/room.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 6b116dce8..ff6927cec 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -1188,7 +1188,7 @@ class RoomCreationHandler: events_to_send.append((power_event, power_context)) else: power_level_content: JsonDict = { - "users": {creator_id: 100}, + "users": {creator_id: 9001}, "users_default": 0, "events": { EventTypes.Name: 50, From 83f9a6cdd5acbe207e1f5b4806c74bbf027ad4cc Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 14:35:30 +0200 Subject: [PATCH 04/14] Allow specifying room ID when creating room --- synapse/handlers/room.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index ff6927cec..2bd8e58da 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -893,11 +893,23 @@ class RoomCreationHandler: self._validate_room_config(config, visibility) - room_id = await self._generate_and_create_room_id( - creator_id=user_id, - is_public=is_public, - room_version=room_version, - ) + if "room_id" in config: + room_id = config["room_id"] + try: + await self.store.store_room( + room_id=room_id, + room_creator_user_id=user_id, + is_public=is_public, + room_version=room_version, + ) + except StoreError: + raise SynapseError(409, "Room ID already in use", errcode="M_CONFLICT") + else: + room_id = await self._generate_and_create_room_id( + creator_id=user_id, + is_public=is_public, + room_version=room_version, + ) # Check whether this visibility value is blocked by a third party module allowed_by_third_party_rules = ( From 0de822af4d8fab30bee234ae68d4b3823a51b974 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 14:37:27 +0200 Subject: [PATCH 05/14] Allow registering invalid user IDs with admin API --- synapse/handlers/register.py | 50 +++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index e48e70db0..b64dadf64 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -148,22 +148,25 @@ class RegistrationHandler: localpart: str, guest_access_token: Optional[str] = None, assigned_user_id: Optional[str] = None, + allow_invalid: bool = False, inhibit_user_in_use_error: bool = False, ) -> None: - if types.contains_invalid_mxid_characters(localpart): - raise SynapseError( - 400, - "User ID can only contain characters a-z, 0-9, or '=_-./+'", - Codes.INVALID_USERNAME, - ) + # meow: allow admins to register invalid user ids + if not allow_invalid: + if types.contains_invalid_mxid_characters(localpart): + raise SynapseError( + 400, + "User ID can only contain characters a-z, 0-9, or '=_-./+'", + Codes.INVALID_USERNAME, + ) - if not localpart: - raise SynapseError(400, "User ID cannot be empty", Codes.INVALID_USERNAME) + if not localpart: + raise SynapseError(400, "User ID cannot be empty", Codes.INVALID_USERNAME) - if localpart[0] == "_": - raise SynapseError( - 400, "User ID may not begin with _", Codes.INVALID_USERNAME - ) + if localpart[0] == "_": + raise SynapseError( + 400, "User ID may not begin with _", Codes.INVALID_USERNAME + ) user = UserID(localpart, self.hs.hostname) user_id = user.to_string() @@ -177,14 +180,16 @@ class RegistrationHandler: "A different user ID has already been registered for this session", ) - self.check_user_id_not_appservice_exclusive(user_id) + # meow: allow admins to register reserved user ids and long user ids + if not allow_invalid: + self.check_user_id_not_appservice_exclusive(user_id) - if len(user_id) > MAX_USERID_LENGTH: - raise SynapseError( - 400, - "User ID may not be longer than %s characters" % (MAX_USERID_LENGTH,), - Codes.INVALID_USERNAME, - ) + if len(user_id) > MAX_USERID_LENGTH: + raise SynapseError( + 400, + "User ID may not be longer than %s characters" % (MAX_USERID_LENGTH,), + Codes.INVALID_USERNAME, + ) users = await self.store.get_users_by_id_case_insensitive(user_id) if users: @@ -290,7 +295,12 @@ class RegistrationHandler: await self.auth_blocking.check_auth_blocking(threepid=threepid) if localpart is not None: - await self.check_username(localpart, guest_access_token=guest_access_token) + allow_invalid = by_admin and self.hs.config.meow.admin_api_register_invalid + await self.check_username( + localpart, + guest_access_token=guest_access_token, + allow_invalid=allow_invalid, + ) was_guest = guest_access_token is not None From 5e7ff45534c15f4b3c60dfde1c9d7644def2258d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 14:39:07 +0200 Subject: [PATCH 06/14] Thumbnail webp images as webp to avoid losing transparency --- synapse/config/repository.py | 10 ++++++---- synapse/media/thumbnailer.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 4655882b4..05b4be638 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -54,10 +54,8 @@ THUMBNAIL_SIZE_YAML = """\ THUMBNAIL_SUPPORTED_MEDIA_FORMAT_MAP = { "image/jpeg": "jpeg", "image/jpg": "jpeg", - "image/webp": "jpeg", - # Thumbnails can only be jpeg or png. We choose png thumbnails for gif - # because it can have transparency. - "image/gif": "png", + "image/webp": "webp", + "image/gif": "webp", "image/png": "png", } @@ -109,6 +107,10 @@ def parse_thumbnail_requirements( requirement.append( ThumbnailRequirement(width, height, method, "image/png") ) + elif thumbnail_format == "webp": + requirement.append( + ThumbnailRequirement(width, height, method, "image/webp") + ) else: raise Exception( "Unknown thumbnail mapping from %s to %s. This is a Synapse problem, please report!" diff --git a/synapse/media/thumbnailer.py b/synapse/media/thumbnailer.py index 5538020be..a071dbf55 100644 --- a/synapse/media/thumbnailer.py +++ b/synapse/media/thumbnailer.py @@ -47,7 +47,7 @@ class ThumbnailError(Exception): class Thumbnailer: - FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG"} + FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG", "image/webp": "WEBP"} @staticmethod def set_limits(max_image_pixels: int) -> None: From ab635c80a7a67a04f738b9d6f54d786e21f44b8d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 14:39:50 +0200 Subject: [PATCH 07/14] Set immutable cache-control header for media downloads --- synapse/media/_base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/media/_base.py b/synapse/media/_base.py index 3fbed6062..fc53c357e 100644 --- a/synapse/media/_base.py +++ b/synapse/media/_base.py @@ -206,7 +206,9 @@ def add_file_headers( # recommend caching as it's sensitive or private - or at least # select private. don't bother setting Expires as all our # clients are smart enough to be happy with Cache-Control - request.setHeader(b"Cache-Control", b"public,max-age=86400,s-maxage=86400") + request.setHeader( + b"Cache-Control", b"public,immutable,max-age=86400,s-maxage=86400" + ) if file_size is not None: request.setHeader(b"Content-Length", b"%d" % (file_size,)) From 9eb9372eb40b8a2713632085b9b27e843228590a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 14:51:56 +0200 Subject: [PATCH 08/14] Allow bypassing unnecessary validation in C-S API --- synapse/events/validator.py | 16 ++++++++++++++-- synapse/handlers/directory.py | 13 +++++++++---- synapse/handlers/federation.py | 4 ++-- synapse/handlers/message.py | 9 ++++++--- synapse/handlers/room_member.py | 20 -------------------- 5 files changed, 31 insertions(+), 31 deletions(-) diff --git a/synapse/events/validator.py b/synapse/events/validator.py index 62f0b67db..697522407 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -64,7 +64,7 @@ class EventValidator: event: The event to validate. config: The homeserver's configuration. """ - self.validate_builder(event) + self.validate_builder(event, config) if event.format_version == EventFormatVersions.ROOM_V1_V2: EventID.from_string(event.event_id) @@ -95,6 +95,12 @@ class EventValidator: # Note that only the client controlled portion of the event is # checked, since we trust the portions of the event we created. validate_canonicaljson(event.content) + if not 0 < event.origin_server_ts < 2**53: + raise SynapseError(400, "Event timestamp is out of range") + + # meow: allow specific users to send potentially dangerous events. + if event.sender in config.meow.validation_override: + return if event.type == EventTypes.Aliases: if "aliases" in event.content: @@ -193,7 +199,9 @@ class EventValidator: errcode=Codes.BAD_JSON, ) - def validate_builder(self, event: Union[EventBase, EventBuilder]) -> None: + def validate_builder( + self, event: Union[EventBase, EventBuilder], config: HomeServerConfig + ) -> None: """Validates that the builder/event has roughly the right format. Only checks values that we expect a proto event to have, rather than all the fields an event would have @@ -211,6 +219,10 @@ class EventValidator: RoomID.from_string(event.room_id) UserID.from_string(event.sender) + # meow: allow specific users to send so-called invalid events + if event.sender in config.meow.validation_override: + return + if event.type == EventTypes.Message: strings = ["body", "msgtype"] diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 5f3dc30b6..6fc3379b6 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -80,9 +80,11 @@ class DirectoryHandler: ) -> None: # general association creation for both human users and app services - for wchar in string.whitespace: - if wchar in room_alias.localpart: - raise SynapseError(400, "Invalid characters in room alias") + # meow: allow specific users to include anything in room aliases + if creator not in self.config.meow.validation_override: + for wchar in string.whitespace: + if wchar in room_alias.localpart: + raise SynapseError(400, "Invalid characters in room alias") if ":" in room_alias.localpart: raise SynapseError(400, "Invalid character in room alias localpart: ':'.") @@ -127,7 +129,10 @@ class DirectoryHandler: user_id = requester.user.to_string() room_alias_str = room_alias.to_string() - if len(room_alias_str) > MAX_ALIAS_LENGTH: + if ( + user_id not in self.hs.config.meow.validation_override + and len(room_alias_str) > MAX_ALIAS_LENGTH + ): raise SynapseError( 400, "Can't create aliases longer than %s characters" % MAX_ALIAS_LENGTH, diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 2b7aad5b5..520e30fd1 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1452,7 +1452,7 @@ class FederationHandler: room_version_obj, event_dict ) - EventValidator().validate_builder(builder) + EventValidator().validate_builder(builder, self.hs.config) # Try several times, it could fail with PartialStateConflictError # in send_membership_event, cf comment in except block. @@ -1617,7 +1617,7 @@ class FederationHandler: builder = self.event_builder_factory.for_room_version( room_version_obj, event_dict ) - EventValidator().validate_builder(builder) + EventValidator().validate_builder(builder, self.hs.config) ( event, diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7e5bb97f2..f4113466a 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -669,7 +669,7 @@ class EventCreationHandler: room_version_obj, event_dict ) - self.validator.validate_builder(builder) + self.validator.validate_builder(builder, self.config) if builder.type == EventTypes.Member: membership = builder.content.get("membership", None) @@ -1339,6 +1339,8 @@ class EventCreationHandler: Raises: SynapseError if the event is invalid. """ + if event.sender in self.config.meow.validation_override: + return relation = relation_from_event(event) if not relation: @@ -1757,7 +1759,8 @@ class EventCreationHandler: await self._maybe_kick_guest_users(event, context) - if event.type == EventTypes.CanonicalAlias: + validation_override = event.sender in self.config.meow.validation_override + if event.type == EventTypes.CanonicalAlias and not validation_override: # Validate a newly added alias or newly added alt_aliases. original_alias = None @@ -2112,7 +2115,7 @@ class EventCreationHandler: builder = self.event_builder_factory.for_room_version( original_event.room_version, third_party_result ) - self.validator.validate_builder(builder) + self.validator.validate_builder(builder, self.config) except SynapseError as e: raise Exception( "Third party rules module created an invalid event: " + e.msg, diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index d238c40bc..029a1c73d 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -754,26 +754,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): content.pop("displayname", None) content.pop("avatar_url", None) - if len(content.get("displayname") or "") > MAX_DISPLAYNAME_LEN: - raise SynapseError( - 400, - f"Displayname is too long (max {MAX_DISPLAYNAME_LEN})", - errcode=Codes.BAD_JSON, - ) - - if len(content.get("avatar_url") or "") > MAX_AVATAR_URL_LEN: - raise SynapseError( - 400, - f"Avatar URL is too long (max {MAX_AVATAR_URL_LEN})", - errcode=Codes.BAD_JSON, - ) - - if "avatar_url" in content and content.get("avatar_url") is not None: - if not await self.profile_handler.check_avatar_size_and_mime_type( - content["avatar_url"], - ): - raise SynapseError(403, "This avatar is not allowed", Codes.FORBIDDEN) - # The event content should *not* include the authorising user as # it won't be properly signed. Strip it out since it might come # back from a client updating a display name / avatar. From b07561405cc1722774401f8c8a485839b90289e8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 14:54:24 +0200 Subject: [PATCH 09/14] Allow unhiding events that the C-S API filters away by default --- synapse/handlers/sync.py | 1 - synapse/storage/controllers/__init__.py | 1 + synapse/visibility.py | 12 +++++++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9122a79b4..7e214f919 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1242,7 +1242,6 @@ class SyncHandler: for e in await sync_config.filter_collection.filter_room_state( list(state.values()) ) - if e.type != EventTypes.Aliases # until MSC2261 or alternative solution } async def _find_missing_partial_state_memberships( diff --git a/synapse/storage/controllers/__init__.py b/synapse/storage/controllers/__init__.py index 5ba848efb..17449b7fc 100644 --- a/synapse/storage/controllers/__init__.py +++ b/synapse/storage/controllers/__init__.py @@ -45,6 +45,7 @@ class StorageControllers: # rewrite all the existing code to split it into high vs low level # interfaces. self.main = stores.main + self.hs = hs self.purge_events = PurgeEventsStorageController(hs, stores) self.state = StateStorageController(hs, stores) diff --git a/synapse/visibility.py b/synapse/visibility.py index e58f649aa..6e1f93d2c 100644 --- a/synapse/visibility.py +++ b/synapse/visibility.py @@ -133,6 +133,10 @@ async def filter_events_for_client( room_id ] = await storage.main.get_retention_policy_for_room(room_id) + # meow: let admins see secret events like org.matrix.dummy_event, m.room.aliases + # and events expired by the retention policy. + filter_override = user_id in storage.hs.config.meow.filter_override + def allowed(event: EventBase) -> Optional[EventBase]: return _check_client_allowed_to_see_event( user_id=user_id, @@ -145,6 +149,7 @@ async def filter_events_for_client( state=event_id_to_state.get(event.event_id), is_peeking=is_peeking, sender_erased=erased_senders.get(event.sender, False), + filter_override=filter_override, ) # Check each event: gives an iterable of None or (a potentially modified) @@ -292,6 +297,7 @@ def _check_client_allowed_to_see_event( retention_policy: RetentionPolicy, state: Optional[StateMap[EventBase]], sender_erased: bool, + filter_override: bool, ) -> Optional[EventBase]: """Check with the given user is allowed to see the given event @@ -308,6 +314,7 @@ def _check_client_allowed_to_see_event( retention_policy: The retention policy of the room state: The state at the event, unless its an outlier sender_erased: Whether the event sender has been marked as "erased" + filter_override: meow Returns: None if the user cannot see this event at all @@ -321,7 +328,7 @@ def _check_client_allowed_to_see_event( # because, if this is not the case, we're probably only checking if the users can # see events in the room at that point in the DAG, and that shouldn't be decided # on those checks. - if filter_send_to_client: + if filter_send_to_client and not filter_override: if ( _check_filter_send_to_client(event, clock, retention_policy, sender_ignored) == _CheckFilter.DENIED @@ -331,6 +338,9 @@ def _check_client_allowed_to_see_event( event.event_id, ) return None + # meow: even with filter_override, we want to filter ignored users + elif filter_send_to_client and not event.is_state() and sender_ignored: + return None if event.event_id in always_include_ids: return event From 3108b67232a23479f6ca5158b85635b043816ae2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 15:02:13 +0200 Subject: [PATCH 10/14] Allow custom content in read receipts --- synapse/handlers/read_marker.py | 11 ++++++++--- synapse/handlers/receipts.py | 3 ++- synapse/rest/client/read_marker.py | 5 +++++ synapse/rest/client/receipts.py | 4 +++- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py index 135a66226..f39732f2c 100644 --- a/synapse/handlers/read_marker.py +++ b/synapse/handlers/read_marker.py @@ -20,11 +20,12 @@ # import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from synapse.api.constants import ReceiptTypes from synapse.api.errors import SynapseError from synapse.util.async_helpers import Linearizer +from synapse.types import JsonDict if TYPE_CHECKING: from synapse.server import HomeServer @@ -39,7 +40,11 @@ class ReadMarkerHandler: self.read_marker_linearizer = Linearizer(name="read_marker") async def received_client_read_marker( - self, room_id: str, user_id: str, event_id: str + self, + room_id: str, + user_id: str, + event_id: str, + extra_content: Optional[JsonDict] = None, ) -> None: """Updates the read marker for a given user in a given room if the event ID given is ahead in the stream relative to the current read marker. @@ -71,7 +76,7 @@ class ReadMarkerHandler: should_update = event_ordering > old_event_ordering if should_update: - content = {"event_id": event_id} + content = {"event_id": event_id, **(extra_content or {})} await self.account_data_handler.add_account_data_to_room( user_id, room_id, ReceiptTypes.FULLY_READ, content ) diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index 8674a8fcd..cc2f49efd 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -181,6 +181,7 @@ class ReceiptsHandler: user_id: UserID, event_id: str, thread_id: Optional[str], + extra_content: Optional[JsonDict] = None, ) -> None: """Called when a client tells us a local user has read up to the given event_id in the room. @@ -197,7 +198,7 @@ class ReceiptsHandler: user_id=user_id.to_string(), event_ids=[event_id], thread_id=thread_id, - data={"ts": int(self.clock.time_msec())}, + data={"ts": int(self.clock.time_msec()), **(extra_content or {})}, ) is_new = await self._handle_new_receipts([receipt]) diff --git a/synapse/rest/client/read_marker.py b/synapse/rest/client/read_marker.py index d3d3c7c41..dce29e493 100644 --- a/synapse/rest/client/read_marker.py +++ b/synapse/rest/client/read_marker.py @@ -80,12 +80,16 @@ class ReadMarkerRestServlet(RestServlet): # TODO Add validation to reject non-string event IDs. if not event_id: continue + extra_content = body.get( + receipt_type.replace("m.", "com.beeper.") + ".extra", None + ) if receipt_type == ReceiptTypes.FULLY_READ: await self.read_marker_handler.received_client_read_marker( room_id, user_id=requester.user.to_string(), event_id=event_id, + extra_content=extra_content, ) else: await self.receipts_handler.received_client_receipt( @@ -95,6 +99,7 @@ class ReadMarkerRestServlet(RestServlet): event_id=event_id, # Setting the thread ID is not possible with the /read_markers endpoint. thread_id=None, + extra_content=extra_content, ) return 200, {} diff --git a/synapse/rest/client/receipts.py b/synapse/rest/client/receipts.py index 89203dc45..d83baaa1c 100644 --- a/synapse/rest/client/receipts.py +++ b/synapse/rest/client/receipts.py @@ -73,7 +73,7 @@ class ReceiptRestServlet(RestServlet): f"Receipt type must be {', '.join(self._known_receipt_types)}", ) - body = parse_json_object_from_request(request) + body = parse_json_object_from_request(request, allow_empty_body=False) # Pull the thread ID, if one exists. thread_id = None @@ -110,6 +110,7 @@ class ReceiptRestServlet(RestServlet): room_id, user_id=requester.user.to_string(), event_id=event_id, + extra_content=body, ) else: await self.receipts_handler.received_client_receipt( @@ -118,6 +119,7 @@ class ReceiptRestServlet(RestServlet): user_id=requester.user, event_id=event_id, thread_id=thread_id, + extra_content=body, ) return 200, {} From de89885d15c4a3c6ca3618d912390e67eaf47e7c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 15:02:25 +0200 Subject: [PATCH 11/14] Allow specific users to use timestamp massaging without being appservices --- synapse/rest/client/room.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 65dedb8b9..91b7da8fb 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -337,6 +337,7 @@ class RoomSendEventRestServlet(TransactionRestServlet): super().__init__(hs) self.event_creation_handler = hs.get_event_creation_handler() self.auth = hs.get_auth() + self.hs = hs def register(self, http_server: HttpServer) -> None: # /rooms/$roomid/send/$event_type[/$txn_id] @@ -360,7 +361,10 @@ class RoomSendEventRestServlet(TransactionRestServlet): "sender": requester.user.to_string(), } - if requester.app_service: + if ( + requester.app_service + or requester.user.to_string() in self.hs.config.meow.timestamp_override + ): origin_server_ts = parse_integer(request, "ts") if origin_server_ts is not None: event_dict["origin_server_ts"] = origin_server_ts From f4f711f28b41ecc0dcdcc42a7c66af71777e5899 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 15:02:44 +0200 Subject: [PATCH 12/14] Remove unnecessary pusher URL validation --- synapse/push/httppusher.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index dd9b64d6e..b40112541 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -140,13 +140,6 @@ class HttpPusher(Pusher): url = self.data["url"] if not isinstance(url, str): raise PusherConfigException("'url' must be a string") - url_parts = urllib.parse.urlparse(url) - # Note that the specification also says the scheme must be HTTPS, but - # it isn't up to the homeserver to verify that. - if url_parts.path != "/_matrix/push/v1/notify": - raise PusherConfigException( - "'url' must have a path of '/_matrix/push/v1/notify'" - ) self.url = url self.http_client = hs.get_proxied_blocklisted_http_client() From 1b784b06d48b76a0dbedb2d61695bd6ea69d70fc Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 10 Oct 2023 15:29:12 +0300 Subject: [PATCH 13/14] Allow pdf inline --- synapse/media/_base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/media/_base.py b/synapse/media/_base.py index fc53c357e..6e73da2ef 100644 --- a/synapse/media/_base.py +++ b/synapse/media/_base.py @@ -64,6 +64,7 @@ INLINE_CONTENT_TYPES = [ "text/csv", "application/json", "application/ld+json", + "application/pdf", # We allow some media files deemed as safe, which comes from the matrix-react-sdk. # https://github.com/matrix-org/matrix-react-sdk/blob/a70fcfd0bcf7f8c85986da18001ea11597989a7c/src/utils/blobs.ts#L51 # SVGs are *intentionally* omitted. From 0f5e09524d130fa81dee570810b16a3c9f6c19f0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 12 Dec 2023 14:34:01 +0200 Subject: [PATCH 14/14] Don't apply alias rules to admins --- synapse/handlers/directory.py | 4 ++-- synapse/handlers/room.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 6fc3379b6..021584fdc 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -174,7 +174,7 @@ class DirectoryHandler: if not self.config.roomdirectory.is_alias_creation_allowed( user_id, room_id, room_alias_str - ): + ) and not is_admin: # Let's just return a generic message, as there may be all sorts of # reasons why we said no. TODO: Allow configurable error messages # per alias creation rule? @@ -510,7 +510,7 @@ class DirectoryHandler: if not self.config.roomdirectory.is_publishing_room_allowed( user_id, room_id, room_aliases - ): + ) and not await self.auth.is_server_admin(requester): # Let's just return a generic message, as there may be all sorts of # reasons why we said no. TODO: Allow configurable error messages # per alias creation rule? diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 2bd8e58da..04b394bbc 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -928,7 +928,7 @@ class RoomCreationHandler: room_aliases.append(room_alias.to_string()) if not self.config.roomdirectory.is_publishing_room_allowed( user_id, room_id, room_aliases - ): + ) and not is_requester_admin: # allow room creation to continue but do not publish room await self.store.set_room_is_public(room_id, False)