Merge tag 'meow-patchset-v1.128.0'

.dockerignore

@@ -8,6 +8,7 @@
 !README.rst
 !pyproject.toml
 !poetry.lock
+!requirements.txt
 !Cargo.lock
 !Cargo.toml
 !build_rust.py

.gitlab-ci.yml

@@ -0,0 +1,19 @@
+image: docker:stable
+
+stages:
+- build
+
+build amd64:
+  stage: build
+  tags:
+  - amd64
+  only:
+  - master
+  before_script:
+  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+  script:
+  - synversion=$(cat pyproject.toml | grep '^version =' | sed -E 's/^version = "(.+)"$/\1/')
+  - docker build --tag $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$synversion .
+  - docker push $CI_REGISTRY_IMAGE:latest
+  - docker push $CI_REGISTRY_IMAGE:$synversion
+  - docker rmi $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$synversion

Dockerfile

@@ -0,0 +1,62 @@
+ARG PYTHON_VERSION=3.13
+
+FROM docker.io/python:${PYTHON_VERSION}-slim as builder
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    libffi-dev \
+    libjpeg-dev \
+    libpq-dev \
+    libssl-dev \
+    libwebp-dev \
+    libxml++2.6-dev \
+    libxslt1-dev \
+    zlib1g-dev \
+    openssl \
+    git \
+    curl \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV RUSTUP_HOME=/rust
+ENV CARGO_HOME=/cargo
+ENV PATH=/cargo/bin:/rust/bin:$PATH
+RUN mkdir /rust /cargo
+
+RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable
+
+COPY synapse /synapse/synapse/
+COPY rust /synapse/rust/
+COPY README.rst pyproject.toml requirements.txt build_rust.py /synapse/
+
+RUN pip install --prefix="/install" --no-warn-script-location --ignore-installed \
+        --no-deps -r /synapse/requirements.txt \
+ && pip install --prefix="/install" --no-warn-script-location \
+        --no-deps \
+	'synapse-simple-antispam @ git+https://github.com/maunium/synapse-simple-antispam' \
+	'synapse-http-antispam @ git+https://github.com/maunium/synapse-http-antispam' \
+	'shared_secret_authenticator @ git+https://github.com/devture/matrix-synapse-shared-secret-auth@2.0.3' \
+ && pip install --prefix="/install" --no-warn-script-location \
+        --no-deps /synapse
+
+FROM docker.io/python:${PYTHON_VERSION}-slim
+
+RUN apt-get update && apt-get install -y \
+    curl \
+    libjpeg62-turbo \
+    libpq5 \
+    libwebp7 \
+    xmlsec1 \
+    libjemalloc2 \
+    openssl \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /install /usr/local
+
+VOLUME ["/data"]
+ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libjemalloc.so.2"
+
+ENTRYPOINT ["python3", "-m", "synapse.app.homeserver"]
+CMD ["--keys-directory", "/data", "-c", "/data/homeserver.yaml"]
+
+HEALTHCHECK --start-period=5s --interval=1m --timeout=5s \
+  CMD curl -fSs http://localhost:8008/health || exit 1

README.md

@@ -0,0 +1,63 @@
+# Maunium Synapse
+This is a fork of [Synapse] to remove dumb limits and fix bugs that the
+upstream devs don't want to fix.
+
+The only official distribution is the docker image in the [GitLab container
+registry], but you can also install from source ([upstream instructions]).
+
+The master branch and `:latest` docker tag are upgraded to each upstream
+release candidate very soon after release (usually within 10 minutes†). There
+are also docker tags for each release, e.g. `:1.75.0`. If you don't want RCs,
+use the specific release tags.
+
+†If there are merge conflicts, the update may be delayed for up to a few days
+after the full release.
+
+[Synapse]: https://github.com/matrix-org/synapse
+[GitLab container registry]: https://mau.dev/maunium/synapse/container_registry
+[upstream instructions]: https://github.com/matrix-org/synapse/blob/develop/INSTALL.md#installing-from-source
+
+## List of changes
+* Default power level for room creator is 9001 instead of 100.
+* Room creator can specify a custom room ID with the `room_id` param in the
+  request body. If the room ID is already in use, it will return `M_CONFLICT`.
+* ~~URL previewer user agent includes `Bot` so Twitter previews work properly.~~
+  Upstreamed after over 2 years 🎉
+* ~~Local event creation concurrency is disabled to avoid unnecessary state
+  resolution.~~ Upstreamed after over 3 years 🎉
+* Register admin API can register invalid user IDs.
+* Docker image with jemalloc enabled by default.
+* Config option to allow specific users to send events without unnecessary
+  validation.
+* Config option to allow specific users to receive events that are usually
+  filtered away (e.g. `org.matrix.dummy_event` and `m.room.aliases`).
+* Config option to allow specific users to use timestamp massaging without
+  being appservice users.
+* Removed bad pusher URL validation.
+* webp images are thumbnailed to webp instead of jpeg to avoid losing
+  transparency.
+* Media repo `Cache-Control` header says `immutable` and 1 year for all media
+  that exists, as media IDs in Matrix are immutable.
+* Allowed sending custom data with read receipts.
+
+You can view the full list of changes on the [meow-patchset] branch.
+Additionally, historical patch sets are saved as `meow-patchset-vX` [tags].
+
+[meow-patchset]: https://mau.dev/maunium/synapse/-/compare/patchset-base...meow-patchset
+[tags]: https://mau.dev/maunium/synapse/-/tags?search=meow-patchset&sort=updated_desc
+
+## Configuration reference
+```yaml
+meow:
+   # List of users who aren't subject to unnecessary validation in the C-S API.
+   validation_override:
+   - "@you:example.com"
+   # List of users who will get org.matrix.dummy_event and m.room.aliases events down /sync
+   filter_override:
+   - "@you:example.com"
+   # Whether or not the admin API should be able to register invalid user IDs.
+   admin_api_register_invalid: true
+   # List of users who can use timestamp massaging without being appservices
+   timestamp_override:
+   - "@you:example.com"
+```

synapse/api/constants.py

@@ -83,6 +83,8 @@ class RestrictedJoinRuleTypes:
 
     ROOM_MEMBERSHIP: Final = "m.room_membership"
 
+    MAU_SPAM_CHECKER: Final = "fi.mau.spam_checker"
+
 
 class LoginType:
     PASSWORD: Final = "m.login.password"

synapse/config/_base.pyi

@@ -36,6 +36,7 @@ from synapse.config import (  # noqa: F401
     jwt,
     key,
     logger,
+    meow,
     metrics,
     modules,
     oembed,
@@ -92,6 +93,7 @@ class RootConfig:
     voip: voip.VoipConfig
     registration: registration.RegistrationConfig
     account_validity: account_validity.AccountValidityConfig
+    meow: meow.MeowConfig
     metrics: metrics.MetricsConfig
     api: api.ApiConfig
     appservice: appservice.AppServiceConfig

synapse/config/homeserver.py

@@ -19,6 +19,7 @@
 #
 #
 from ._base import RootConfig
+from .meow import MeowConfig
 from .account_validity import AccountValidityConfig
 from .api import ApiConfig
 from .appservice import AppServiceConfig
@@ -65,6 +66,7 @@ from .workers import WorkerConfig
 
 class HomeServerConfig(RootConfig):
     config_classes = [
+        MeowConfig,
         ModulesConfig,
         ServerConfig,
         RetentionConfig,

synapse/config/meow.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+# Copyright 2020 Maunium
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._base import Config
+
+
+class MeowConfig(Config):
+    """Meow Configuration
+    Configuration for disabling dumb limits in Synapse
+    """
+
+    section = "meow"
+
+    def read_config(self, config, **kwargs):
+        meow_config = config.get("meow", {})
+        self.validation_override = set(meow_config.get("validation_override", []))
+        self.filter_override = set(meow_config.get("filter_override", []))
+        self.timestamp_override = set(meow_config.get("timestamp_override", []))
+        self.admin_api_register_invalid = meow_config.get(
+            "admin_api_register_invalid", True
+        )

synapse/config/repository.py

@@ -54,10 +54,8 @@ THUMBNAIL_SIZE_YAML = """\
 THUMBNAIL_SUPPORTED_MEDIA_FORMAT_MAP = {
     "image/jpeg": "jpeg",
     "image/jpg": "jpeg",
-    "image/webp": "jpeg",
-    # Thumbnails can only be jpeg or png. We choose png thumbnails for gif
-    # because it can have transparency.
-    "image/gif": "png",
+    "image/webp": "webp",
+    "image/gif": "webp",
     "image/png": "png",
 }
 
@@ -109,6 +107,10 @@ def parse_thumbnail_requirements(
                 requirement.append(
                     ThumbnailRequirement(width, height, method, "image/png")
                 )
+            elif thumbnail_format == "webp":
+                requirement.append(
+                    ThumbnailRequirement(width, height, method, "image/webp")
+                )
             else:
                 raise Exception(
                     "Unknown thumbnail mapping from %s to %s. This is a Synapse problem, please report!"

synapse/events/validator.py

@@ -58,7 +58,7 @@ class EventValidator:
             event: The event to validate.
             config: The homeserver's configuration.
         """
-        self.validate_builder(event)
+        self.validate_builder(event, config)
 
         if event.format_version == EventFormatVersions.ROOM_V1_V2:
             EventID.from_string(event.event_id)
@@ -88,6 +88,13 @@ class EventValidator:
         if event.room_version.strict_canonicaljson:
             validate_canonicaljson(event.get_pdu_json())
 
+        if not 0 < event.origin_server_ts < 2**53:
+            raise SynapseError(400, "Event timestamp is out of range")
+
+        # meow: allow specific users to send potentially dangerous events.
+        if event.sender in config.meow.validation_override:
+            return
+
         if event.type == EventTypes.Aliases:
             if "aliases" in event.content:
                 for alias in event.content["aliases"]:
@@ -185,7 +192,9 @@ class EventValidator:
                 errcode=Codes.BAD_JSON,
             )
 
-    def validate_builder(self, event: Union[EventBase, EventBuilder]) -> None:
+    def validate_builder(
+        self, event: Union[EventBase, EventBuilder], config: HomeServerConfig
+    ) -> None:
         """Validates that the builder/event has roughly the right format. Only
         checks values that we expect a proto event to have, rather than all the
         fields an event would have
@@ -203,6 +212,10 @@ class EventValidator:
         RoomID.from_string(event.room_id)
         UserID.from_string(event.sender)
 
+        # meow: allow specific users to send so-called invalid events
+        if event.sender in config.meow.validation_override:
+            return
+
         if event.type == EventTypes.Message:
             strings = ["body", "msgtype"]
 

synapse/handlers/delayed_events.py

@@ -270,7 +270,8 @@ class DelayedEventsHandler:
                     "sender": str(requester.user),
                     **({"state_key": state_key} if state_key is not None else {}),
                 },
-            )
+            ),
+            self._config,
         )
 
         creation_ts = self._get_current_ts()

synapse/handlers/directory.py

@@ -78,9 +78,11 @@ class DirectoryHandler:
     ) -> None:
         # general association creation for both human users and app services
 
-        for wchar in string.whitespace:
-            if wchar in room_alias.localpart:
-                raise SynapseError(400, "Invalid characters in room alias")
+        # meow: allow specific users to include anything in room aliases
+        if creator not in self.config.meow.validation_override:
+            for wchar in string.whitespace:
+                if wchar in room_alias.localpart:
+                    raise SynapseError(400, "Invalid characters in room alias")
 
         if ":" in room_alias.localpart:
             raise SynapseError(400, "Invalid character in room alias localpart: ':'.")
@@ -125,7 +127,10 @@ class DirectoryHandler:
         user_id = requester.user.to_string()
         room_alias_str = room_alias.to_string()
 
-        if len(room_alias_str) > MAX_ALIAS_LENGTH:
+        if (
+            user_id not in self.hs.config.meow.validation_override
+            and len(room_alias_str) > MAX_ALIAS_LENGTH
+        ):
             raise SynapseError(
                 400,
                 "Can't create aliases longer than %s characters" % MAX_ALIAS_LENGTH,
@@ -167,7 +172,7 @@ class DirectoryHandler:
 
             if not self.config.roomdirectory.is_alias_creation_allowed(
                 user_id, room_id, room_alias_str
-            ):
+            ) and not is_admin:
                 # Let's just return a generic message, as there may be all sorts of
                 # reasons why we said no. TODO: Allow configurable error messages
                 # per alias creation rule?
@@ -503,7 +508,7 @@ class DirectoryHandler:
 
             if not self.config.roomdirectory.is_publishing_room_allowed(
                 user_id, room_id, room_aliases
-            ):
+            ) and not await self.auth.is_server_admin(requester):
                 # Let's just return a generic message, as there may be all sorts of
                 # reasons why we said no. TODO: Allow configurable error messages
                 # per alias creation rule?

synapse/handlers/event_auth.py

@@ -55,6 +55,7 @@ class EventAuthHandler:
         self._state_storage_controller = hs.get_storage_controllers().state
         self._server_name = hs.hostname
         self._is_mine_id = hs.is_mine_id
+        self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker
 
     async def check_auth_rules_from_context(
         self,
@@ -220,6 +221,7 @@ class EventAuthHandler:
         room_version: RoomVersion,
         user_id: str,
         prev_membership: Optional[str],
+        room_id: str,
     ) -> None:
         """
         Check whether a user can join a room without an invite due to restricted join rules.
@@ -251,8 +253,27 @@ class EventAuthHandler:
 
         # Get the rooms which allow access to this room and check if the user is
         # in any of them.
-        allowed_rooms = await self.get_rooms_that_allow_join(state_ids)
-        if not await self.is_user_in_rooms(allowed_rooms, user_id):
+        allowed_rooms, ask_spam_checker = await self.get_rooms_that_allow_join(
+            state_ids
+        )
+
+        if await self.is_user_in_rooms(allowed_rooms, user_id):
+            # If the user is in allowed rooms, allow the join without asking the spam checker
+            pass
+        elif ask_spam_checker:
+            spam_check = (
+                await self._spam_checker_module_callbacks.accept_make_join_callback(
+                    user_id, room_id
+                )
+            )
+            if spam_check != self._spam_checker_module_callbacks.NOT_SPAM:
+                raise AuthError(
+                    403,
+                    "The spam checker rejected your join",
+                    errcode=spam_check[0],
+                    additional_fields=spam_check[1],
+                )
+        else:
             # If this is a remote request, the user might be in an allowed room
             # that we do not know about.
             if not self._is_mine_id(user_id):
@@ -306,7 +327,7 @@ class EventAuthHandler:
 
     async def get_rooms_that_allow_join(
         self, state_ids: StateMap[str]
-    ) -> StrCollection:
+    ) -> tuple[StrCollection, bool]:
         """
         Generate a list of rooms in which membership allows access to a room.
 
@@ -319,7 +340,7 @@ class EventAuthHandler:
         # If there's no join rule, then it defaults to invite (so this doesn't apply).
         join_rules_event_id = state_ids.get((EventTypes.JoinRules, ""), None)
         if not join_rules_event_id:
-            return ()
+            return (), False
 
         # If the join rule is not restricted, this doesn't apply.
         join_rules_event = await self._store.get_event(join_rules_event_id)
@@ -327,14 +348,19 @@ class EventAuthHandler:
         # If allowed is of the wrong form, then only allow invited users.
         allow_list = join_rules_event.content.get("allow", [])
         if not isinstance(allow_list, list):
-            return ()
+            return (), False
 
         # Pull out the other room IDs, invalid data gets filtered.
         result = []
+        ask_spam_checker = False
         for allow in allow_list:
             if not isinstance(allow, dict):
                 continue
 
+            if allow.get("type") == RestrictedJoinRuleTypes.MAU_SPAM_CHECKER:
+                ask_spam_checker = True
+                continue
+
             # If the type is unexpected, skip it.
             if allow.get("type") != RestrictedJoinRuleTypes.ROOM_MEMBERSHIP:
                 continue
@@ -345,7 +371,7 @@ class EventAuthHandler:
 
             result.append(room_id)
 
-        return result
+        return result, ask_spam_checker
 
     async def is_user_in_rooms(self, room_ids: StrCollection, user_id: str) -> bool:
         """

synapse/handlers/federation.py

@@ -1455,7 +1455,7 @@ class FederationHandler:
                 room_version_obj, event_dict
             )
 
-            EventValidator().validate_builder(builder)
+            EventValidator().validate_builder(builder, self.hs.config)
 
             # Try several times, it could fail with PartialStateConflictError
             # in send_membership_event, cf comment in except block.
@@ -1620,7 +1620,7 @@ class FederationHandler:
         builder = self.event_builder_factory.for_room_version(
             room_version_obj, event_dict
         )
-        EventValidator().validate_builder(builder)
+        EventValidator().validate_builder(builder, self.hs.config)
 
         (
             event,

synapse/handlers/federation_event.py

@@ -467,6 +467,7 @@ class FederationEventHandler:
             event.room_version,
             user_id,
             prev_membership,
+            event.room_id,
         )
 
     @trace

synapse/handlers/message.py

@@ -696,7 +696,7 @@ class EventCreationHandler:
             room_version_obj, event_dict
         )
 
-        self.validator.validate_builder(builder)
+        self.validator.validate_builder(builder, self.config)
 
         is_exempt = await self._is_exempt_from_privacy_policy(builder, requester)
         if require_consent and not is_exempt:
@@ -1354,6 +1354,8 @@ class EventCreationHandler:
         Raises:
             SynapseError if the event is invalid.
         """
+        if event.sender in self.config.meow.validation_override:
+            return
 
         relation = relation_from_event(event)
         if not relation:
@@ -1779,7 +1781,8 @@ class EventCreationHandler:
 
             await self._maybe_kick_guest_users(event, context)
 
-            if event.type == EventTypes.CanonicalAlias:
+            validation_override = event.sender in self.config.meow.validation_override
+            if event.type == EventTypes.CanonicalAlias and not validation_override:
                 # Validate a newly added alias or newly added alt_aliases.
 
                 original_alias = None
@@ -2134,7 +2137,7 @@ class EventCreationHandler:
             builder = self.event_builder_factory.for_room_version(
                 original_event.room_version, third_party_result
             )
-            self.validator.validate_builder(builder)
+            self.validator.validate_builder(builder, self.config)
         except SynapseError as e:
             raise Exception(
                 "Third party rules module created an invalid event: " + e.msg,

synapse/handlers/read_marker.py

@@ -20,11 +20,12 @@
 #
 
 import logging
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 from synapse.api.constants import ReceiptTypes
 from synapse.api.errors import SynapseError
 from synapse.util.async_helpers import Linearizer
+from synapse.types import JsonDict
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -39,7 +40,11 @@ class ReadMarkerHandler:
         self.read_marker_linearizer = Linearizer(name="read_marker")
 
     async def received_client_read_marker(
-        self, room_id: str, user_id: str, event_id: str
+        self,
+        room_id: str,
+        user_id: str,
+        event_id: str,
+        extra_content: Optional[JsonDict] = None,
     ) -> None:
         """Updates the read marker for a given user in a given room if the event ID given
         is ahead in the stream relative to the current read marker.
@@ -71,7 +76,7 @@ class ReadMarkerHandler:
                     should_update = event_ordering > old_event_ordering
 
             if should_update:
-                content = {"event_id": event_id}
+                content = {"event_id": event_id, **(extra_content or {})}
                 await self.account_data_handler.add_account_data_to_room(
                     user_id, room_id, ReceiptTypes.FULLY_READ, content
                 )

synapse/handlers/receipts.py

@@ -181,6 +181,7 @@ class ReceiptsHandler:
         user_id: UserID,
         event_id: str,
         thread_id: Optional[str],
+        extra_content: Optional[JsonDict] = None,
     ) -> None:
         """Called when a client tells us a local user has read up to the given
         event_id in the room.
@@ -197,7 +198,7 @@ class ReceiptsHandler:
             user_id=user_id.to_string(),
             event_ids=[event_id],
             thread_id=thread_id,
-            data={"ts": int(self.clock.time_msec())},
+            data={"ts": int(self.clock.time_msec()), **(extra_content or {})},
         )
 
         is_new = await self._handle_new_receipts([receipt])

synapse/handlers/register.py

@@ -147,22 +147,25 @@ class RegistrationHandler:
         localpart: str,
         guest_access_token: Optional[str] = None,
         assigned_user_id: Optional[str] = None,
+        allow_invalid: bool = False,
         inhibit_user_in_use_error: bool = False,
     ) -> None:
-        if types.contains_invalid_mxid_characters(localpart):
-            raise SynapseError(
-                400,
-                "User ID can only contain characters a-z, 0-9, or '=_-./+'",
-                Codes.INVALID_USERNAME,
-            )
+        # meow: allow admins to register invalid user ids
+        if not allow_invalid:
+            if types.contains_invalid_mxid_characters(localpart):
+                raise SynapseError(
+                    400,
+                    "User ID can only contain characters a-z, 0-9, or '=_-./+'",
+                    Codes.INVALID_USERNAME,
+                )
 
-        if not localpart:
-            raise SynapseError(400, "User ID cannot be empty", Codes.INVALID_USERNAME)
+            if not localpart:
+                raise SynapseError(400, "User ID cannot be empty", Codes.INVALID_USERNAME)
 
-        if localpart[0] == "_":
-            raise SynapseError(
-                400, "User ID may not begin with _", Codes.INVALID_USERNAME
-            )
+            if localpart[0] == "_":
+                raise SynapseError(
+                    400, "User ID may not begin with _", Codes.INVALID_USERNAME
+                )
 
         user = UserID(localpart, self.hs.hostname)
         user_id = user.to_string()
@@ -176,14 +179,16 @@ class RegistrationHandler:
                     "A different user ID has already been registered for this session",
                 )
 
-        self.check_user_id_not_appservice_exclusive(user_id)
+        # meow: allow admins to register reserved user ids and long user ids
+        if not allow_invalid:
+            self.check_user_id_not_appservice_exclusive(user_id)
 
-        if len(user_id) > MAX_USERID_LENGTH:
-            raise SynapseError(
-                400,
-                "User ID may not be longer than %s characters" % (MAX_USERID_LENGTH,),
-                Codes.INVALID_USERNAME,
-            )
+            if len(user_id) > MAX_USERID_LENGTH:
+                raise SynapseError(
+                    400,
+                    "User ID may not be longer than %s characters" % (MAX_USERID_LENGTH,),
+                    Codes.INVALID_USERNAME,
+                )
 
         users = await self.store.get_users_by_id_case_insensitive(user_id)
         if users:
@@ -289,7 +294,12 @@ class RegistrationHandler:
             await self.auth_blocking.check_auth_blocking(threepid=threepid)
 
         if localpart is not None:
-            await self.check_username(localpart, guest_access_token=guest_access_token)
+            allow_invalid = by_admin and self.hs.config.meow.admin_api_register_invalid
+            await self.check_username(
+                localpart,
+                guest_access_token=guest_access_token,
+                allow_invalid=allow_invalid,
+            )
 
             was_guest = guest_access_token is not None
 

synapse/handlers/room.py

@@ -894,11 +894,23 @@ class RoomCreationHandler:
 
         self._validate_room_config(config, visibility)
 
-        room_id = await self._generate_and_create_room_id(
-            creator_id=user_id,
-            is_public=is_public,
-            room_version=room_version,
-        )
+        if "room_id" in config or "fi.mau.room_id" in config:
+            room_id = config.get("fi.mau.room_id") or config["room_id"]
+            try:
+                await self.store.store_room(
+                    room_id=room_id,
+                    room_creator_user_id=user_id,
+                    is_public=is_public,
+                    room_version=room_version,
+                )
+            except StoreError:
+                raise SynapseError(409, "Room ID already in use", errcode="M_CONFLICT")
+        else:
+            room_id = await self._generate_and_create_room_id(
+                creator_id=user_id,
+                is_public=is_public,
+                room_version=room_version,
+            )
 
         # Check whether this visibility value is blocked by a third party module
         allowed_by_third_party_rules = await (
@@ -915,7 +927,7 @@ class RoomCreationHandler:
                 room_aliases.append(room_alias.to_string())
             if not self.config.roomdirectory.is_publishing_room_allowed(
                 user_id, room_id, room_aliases
-            ):
+            ) and not is_requester_admin:
                 # allow room creation to continue but do not publish room
                 await self.store.set_room_is_public(room_id, False)
 
@@ -1190,7 +1202,7 @@ class RoomCreationHandler:
             # Please update the docs for `default_power_level_content_override` when
             # updating the `events` dict below
             power_level_content: JsonDict = {
-                "users": {creator_id: 100},
+                "users": {creator_id: 9001},
                 "users_default": 0,
                 "events": {
                     EventTypes.Name: 50,

synapse/handlers/room_member.py

@@ -797,26 +797,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
             content.pop("displayname", None)
             content.pop("avatar_url", None)
 
-        if len(content.get("displayname") or "") > MAX_DISPLAYNAME_LEN:
-            raise SynapseError(
-                400,
-                f"Displayname is too long (max {MAX_DISPLAYNAME_LEN})",
-                errcode=Codes.BAD_JSON,
-            )
-
-        if len(content.get("avatar_url") or "") > MAX_AVATAR_URL_LEN:
-            raise SynapseError(
-                400,
-                f"Avatar URL is too long (max {MAX_AVATAR_URL_LEN})",
-                errcode=Codes.BAD_JSON,
-            )
-
-        if "avatar_url" in content and content.get("avatar_url") is not None:
-            if not await self.profile_handler.check_avatar_size_and_mime_type(
-                content["avatar_url"],
-            ):
-                raise SynapseError(403, "This avatar is not allowed", Codes.FORBIDDEN)
-
         # The event content should *not* include the authorising user as
         # it won't be properly signed. Strip it out since it might come
         # back from a client updating a display name / avatar.
@@ -1021,7 +1001,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
                 bypass_spam_checker = True
 
             else:
-                bypass_spam_checker = await self.auth.is_server_admin(requester)
+                bypass_spam_checker = False
 
             inviter = await self._get_inviter(target.to_string(), room_id)
             if (
@@ -1316,7 +1296,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
 
         # Ensure the member should be allowed access via membership in a room.
         await self.event_auth_handler.check_restricted_join_rules(
-            state_before_join, room_version, user_id, previous_membership
+            state_before_join, room_version, user_id, previous_membership, room_id
         )
 
         # If this is going to be a local join, additional information must

synapse/handlers/room_summary.py

@@ -638,7 +638,7 @@ class RoomSummaryHandler:
             if await self._event_auth_handler.has_restricted_join_rules(
                 state_ids, room_version
             ):
-                allowed_rooms = (
+                allowed_rooms, _ = (
                     await self._event_auth_handler.get_rooms_that_allow_join(state_ids)
                 )
                 if await self._event_auth_handler.is_user_in_rooms(
@@ -660,7 +660,7 @@ class RoomSummaryHandler:
             if await self._event_auth_handler.has_restricted_join_rules(
                 state_ids, room_version
             ):
-                allowed_rooms = (
+                allowed_rooms, _ = (
                     await self._event_auth_handler.get_rooms_that_allow_join(state_ids)
                 )
                 for space_id in allowed_rooms:
@@ -771,7 +771,7 @@ class RoomSummaryHandler:
             if await self._event_auth_handler.has_restricted_join_rules(
                 current_state_ids, room_version
             ):
-                allowed_rooms = (
+                allowed_rooms, _ = (
                     await self._event_auth_handler.get_rooms_that_allow_join(
                         current_state_ids
                     )

synapse/handlers/sync.py

@@ -1315,7 +1315,6 @@ class SyncHandler:
             for e in await sync_config.filter_collection.filter_room_state(
                 list(state.values())
             )
-            if e.type != EventTypes.Aliases  # until MSC2261 or alternative solution
         }
 
     async def _compute_state_delta_for_full_sync(

synapse/media/thumbnailer.py

@@ -66,7 +66,7 @@ class ThumbnailError(Exception):
 
 
 class Thumbnailer:
-    FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG"}
+    FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG", "image/webp": "WEBP"}
 
     # Which image formats we allow Pillow to open.
     # This should intentionally be kept restrictive, because the decoder of any

synapse/module_api/__init__.py

@@ -101,6 +101,7 @@ from synapse.module_api.callbacks.spamchecker_callbacks import (
     USER_MAY_CREATE_ROOM_CALLBACK,
     USER_MAY_INVITE_CALLBACK,
     USER_MAY_JOIN_ROOM_CALLBACK,
+    ACCEPT_MAKE_JOIN_CALLBACK,
     USER_MAY_PUBLISH_ROOM_CALLBACK,
     USER_MAY_SEND_3PID_INVITE_CALLBACK,
     SpamCheckerModuleApiCallbacks,
@@ -304,6 +305,7 @@ class ModuleApi:
             SHOULD_DROP_FEDERATED_EVENT_CALLBACK
         ] = None,
         user_may_join_room: Optional[USER_MAY_JOIN_ROOM_CALLBACK] = None,
+        accept_make_join: Optional[ACCEPT_MAKE_JOIN_CALLBACK] = None,
         user_may_invite: Optional[USER_MAY_INVITE_CALLBACK] = None,
         user_may_send_3pid_invite: Optional[USER_MAY_SEND_3PID_INVITE_CALLBACK] = None,
         user_may_create_room: Optional[USER_MAY_CREATE_ROOM_CALLBACK] = None,
@@ -326,6 +328,7 @@ class ModuleApi:
             check_event_for_spam=check_event_for_spam,
             should_drop_federated_event=should_drop_federated_event,
             user_may_join_room=user_may_join_room,
+            accept_make_join=accept_make_join,
             user_may_invite=user_may_invite,
             user_may_send_3pid_invite=user_may_send_3pid_invite,
             user_may_create_room=user_may_create_room,

synapse/module_api/callbacks/spamchecker_callbacks.py

@@ -88,6 +88,16 @@ USER_MAY_JOIN_ROOM_CALLBACK = Callable[
         ]
     ],
 ]
+ACCEPT_MAKE_JOIN_CALLBACK = Callable[
+    [str, str],
+    Awaitable[
+        Union[
+            Literal["NOT_SPAM"],
+            Codes,
+            Tuple[Codes, JsonDict],
+        ]
+    ],
+]
 USER_MAY_INVITE_CALLBACK = Callable[
     [str, str, str],
     Awaitable[
@@ -327,6 +337,7 @@ class SpamCheckerModuleApiCallbacks:
             SHOULD_DROP_FEDERATED_EVENT_CALLBACK
         ] = []
         self._user_may_join_room_callbacks: List[USER_MAY_JOIN_ROOM_CALLBACK] = []
+        self._accept_make_join_callbacks: List[ACCEPT_MAKE_JOIN_CALLBACK] = []
         self._user_may_invite_callbacks: List[USER_MAY_INVITE_CALLBACK] = []
         self._user_may_send_3pid_invite_callbacks: List[
             USER_MAY_SEND_3PID_INVITE_CALLBACK
@@ -354,6 +365,7 @@ class SpamCheckerModuleApiCallbacks:
             SHOULD_DROP_FEDERATED_EVENT_CALLBACK
         ] = None,
         user_may_join_room: Optional[USER_MAY_JOIN_ROOM_CALLBACK] = None,
+        accept_make_join: Optional[ACCEPT_MAKE_JOIN_CALLBACK] = None,
         user_may_invite: Optional[USER_MAY_INVITE_CALLBACK] = None,
         user_may_send_3pid_invite: Optional[USER_MAY_SEND_3PID_INVITE_CALLBACK] = None,
         user_may_create_room: Optional[USER_MAY_CREATE_ROOM_CALLBACK] = None,
@@ -380,6 +392,9 @@ class SpamCheckerModuleApiCallbacks:
         if user_may_join_room is not None:
             self._user_may_join_room_callbacks.append(user_may_join_room)
 
+        if accept_make_join is not None:
+            self._accept_make_join_callbacks.append(accept_make_join)
+
         if user_may_invite is not None:
             self._user_may_invite_callbacks.append(user_may_invite)
 
@@ -536,6 +551,35 @@ class SpamCheckerModuleApiCallbacks:
         # No spam-checker has rejected the request, let it pass.
         return self.NOT_SPAM
 
+    async def accept_make_join_callback(
+        self, user_id: str, room_id: str
+    ) -> Union[Tuple[Codes, JsonDict], Literal["NOT_SPAM"]]:
+        for callback in self._accept_make_join_callbacks:
+            with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"):
+                res = await delay_cancellation(callback(user_id, room_id))
+                # Normalize return values to `Codes` or `"NOT_SPAM"`.
+                if res is True or res is self.NOT_SPAM:
+                    continue
+                elif res is False:
+                    return synapse.api.errors.Codes.FORBIDDEN, {}
+                elif isinstance(res, synapse.api.errors.Codes):
+                    return res, {}
+                elif (
+                    isinstance(res, tuple)
+                    and len(res) == 2
+                    and isinstance(res[0], synapse.api.errors.Codes)
+                    and isinstance(res[1], dict)
+                ):
+                    return res
+                else:
+                    logger.warning(
+                        "Module returned invalid value, rejecting join as spam"
+                    )
+                    return synapse.api.errors.Codes.FORBIDDEN, {}
+
+        # No spam-checker has rejected the request, let it pass.
+        return self.NOT_SPAM
+
     async def user_may_invite(
         self, inviter_userid: str, invitee_userid: str, room_id: str
     ) -> Union[Tuple[Codes, dict], Literal["NOT_SPAM"]]:

synapse/push/httppusher.py

@@ -145,13 +145,6 @@ class HttpPusher(Pusher):
         url = self.data["url"]
         if not isinstance(url, str):
             raise PusherConfigException("'url' must be a string")
-        url_parts = urllib.parse.urlparse(url)
-        # Note that the specification also says the scheme must be HTTPS, but
-        # it isn't up to the homeserver to verify that.
-        if url_parts.path != "/_matrix/push/v1/notify":
-            raise PusherConfigException(
-                "'url' must have a path of '/_matrix/push/v1/notify'"
-            )
 
         self.url = url
         self.http_client = hs.get_proxied_blocklisted_http_client()

synapse/rest/client/read_marker.py

@@ -80,12 +80,16 @@ class ReadMarkerRestServlet(RestServlet):
             # TODO Add validation to reject non-string event IDs.
             if not event_id:
                 continue
+            extra_content = body.get(
+                receipt_type.replace("m.", "com.beeper.") + ".extra", None
+            )
 
             if receipt_type == ReceiptTypes.FULLY_READ:
                 await self.read_marker_handler.received_client_read_marker(
                     room_id,
                     user_id=requester.user.to_string(),
                     event_id=event_id,
+                    extra_content=extra_content,
                 )
             else:
                 await self.receipts_handler.received_client_receipt(
@@ -95,6 +99,7 @@ class ReadMarkerRestServlet(RestServlet):
                     event_id=event_id,
                     # Setting the thread ID is not possible with the /read_markers endpoint.
                     thread_id=None,
+                    extra_content=extra_content,
                 )
 
         return 200, {}

synapse/rest/client/receipts.py

@@ -73,7 +73,7 @@ class ReceiptRestServlet(RestServlet):
                 f"Receipt type must be {', '.join(self._known_receipt_types)}",
             )
 
-        body = parse_json_object_from_request(request)
+        body = parse_json_object_from_request(request, allow_empty_body=False)
 
         # Pull the thread ID, if one exists.
         thread_id = None
@@ -110,6 +110,7 @@ class ReceiptRestServlet(RestServlet):
                 room_id,
                 user_id=requester.user.to_string(),
                 event_id=event_id,
+                extra_content=body,
             )
         else:
             await self.receipts_handler.received_client_receipt(
@@ -118,6 +119,7 @@ class ReceiptRestServlet(RestServlet):
                 user_id=requester.user,
                 event_id=event_id,
                 thread_id=thread_id,
+                extra_content=body,
             )
 
         return 200, {}

synapse/rest/client/room.py

@@ -362,6 +362,7 @@ class RoomSendEventRestServlet(TransactionRestServlet):
         self.delayed_events_handler = hs.get_delayed_events_handler()
         self.auth = hs.get_auth()
         self._max_event_delay_ms = hs.config.server.max_event_delay_ms
+        self.hs = hs
 
     def register(self, http_server: HttpServer) -> None:
         # /rooms/$roomid/send/$event_type[/$txn_id]
@@ -379,7 +380,10 @@ class RoomSendEventRestServlet(TransactionRestServlet):
         content = parse_json_object_from_request(request)
 
         origin_server_ts = None
-        if requester.app_service:
+        if (
+            requester.app_service
+            or requester.user.to_string() in self.hs.config.meow.timestamp_override
+        ):
             origin_server_ts = parse_integer(request, "ts")
 
         delay = _parse_request_delay(request, self._max_event_delay_ms)

synapse/storage/controllers/__init__.py

@@ -45,6 +45,7 @@ class StorageControllers:
         # rewrite all the existing code to split it into high vs low level
         # interfaces.
         self.main = stores.main
+        self.hs = hs
 
         self.purge_events = PurgeEventsStorageController(hs, stores)
         self.state = StateStorageController(hs, stores)

synapse/visibility.py

@@ -139,6 +139,10 @@ async def filter_events_for_client(
                 room_id
             ] = await storage.main.get_retention_policy_for_room(room_id)
 
+    # meow: let admins see secret events like org.matrix.dummy_event, m.room.aliases
+    #       and events expired by the retention policy.
+    filter_override = user_id in storage.hs.config.meow.filter_override
+
     def allowed(event: EventBase) -> Optional[EventBase]:
         state_after_event = event_id_to_state.get(event.event_id)
         filtered = _check_client_allowed_to_see_event(
@@ -152,6 +156,7 @@ async def filter_events_for_client(
             state=state_after_event,
             is_peeking=is_peeking,
             sender_erased=erased_senders.get(event.sender, False),
+            filter_override=filter_override,
         )
         if filtered is None:
             return None
@@ -328,6 +333,7 @@ def _check_client_allowed_to_see_event(
     retention_policy: RetentionPolicy,
     state: Optional[StateMap[EventBase]],
     sender_erased: bool,
+    filter_override: bool,
 ) -> Optional[EventBase]:
     """Check with the given user is allowed to see the given event
 
@@ -344,6 +350,7 @@ def _check_client_allowed_to_see_event(
         retention_policy: The retention policy of the room
         state: The state at the event, unless its an outlier
         sender_erased: Whether the event sender has been marked as "erased"
+        filter_override: meow
 
     Returns:
         None if the user cannot see this event at all
@@ -357,7 +364,7 @@ def _check_client_allowed_to_see_event(
     # because, if this is not the case, we're probably only checking if the users can
     # see events in the room at that point in the DAG, and that shouldn't be decided
     # on those checks.
-    if filter_send_to_client:
+    if filter_send_to_client and not filter_override:
         if (
             _check_filter_send_to_client(event, clock, retention_policy, sender_ignored)
             == _CheckFilter.DENIED
@@ -367,6 +374,9 @@ def _check_client_allowed_to_see_event(
                 event.event_id,
             )
             return None
+    # meow: even with filter_override, we want to filter ignored users
+    elif filter_send_to_client and not event.is_state() and sender_ignored:
+        return None
 
     if event.event_id in always_include_ids:
         return event